diff --git a/Makefile b/Makefile index 9280da5a18d6a..9c342ea873150 100644 --- a/Makefile +++ b/Makefile @@ -49,6 +49,12 @@ # % cd matrixone # % MO_CL_CUDA=1 make +# Go toolchain (override with `make GO=/path/to/go ...`); defaults to `go`. +# Requires Go 1.26+ for the arch-specific SIMD kernels (built by default on x86_64). +ifeq ($(GO),) + GO=go +endif + # where am I ROOT_DIR = $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) BIN_NAME := mo-service @@ -212,6 +218,28 @@ DEBUG_OPT := CGO_DEBUG_OPT := TAGS := +# Env-var prefix for the build command. On x86_64 the arch-specific SIMD kernels in +# pkg/vectorindex/metric are compiled by default (ARCHSIMD=1): GOAMD64 defaults to v3 +# (Haswell baseline -- AVX2/FMA/BMI, required by the Go simd experiment) and +# GOEXPERIMENT defaults to simd (enables the goexperiment.simd build tag on Go 1.26+). +# Disable the SIMD kernels with: +# make ARCHSIMD=0 build # plain x86 build, no SIMD kernels +# Either default can still be overridden individually, e.g. `make GOAMD64=v4 build`. +GOEXPERIMENT_OPT ?= +ifeq ("$(UNAME_M)", "x86_64") + ARCHSIMD ?= 1 + ifeq ($(ARCHSIMD),1) + GOAMD64 ?= v3 + GOEXPERIMENT_SIMD ?= simd + endif + ifneq ($(GOAMD64),) + GOEXPERIMENT_OPT += GOAMD64=$(GOAMD64) + endif + ifneq ($(GOEXPERIMENT_SIMD),) + GOEXPERIMENT_OPT += GOEXPERIMENT=$(GOEXPERIMENT_SIMD) + endif +endif + ifeq ($(MO_CL_CUDA),1) ifeq ($(CONDA_PREFIX),) $(error CONDA_PREFIX env variable not found.) @@ -262,7 +290,7 @@ jieba-dict: .PHONY: build build: config cgo thirdparties jieba-dict $(info [Build binary]) - $(CGO_OPTS) go build $(GO_MODULE_MODE) $(TAGS) $(RACE_OPT) $(GOLDFLAGS) $(DEBUG_OPT) $(GOBUILD_OPT) -o $(BIN_NAME) ./cmd/mo-service + $(GOEXPERIMENT_OPT) $(CGO_OPTS) $(GO) build $(GO_MODULE_MODE) $(TAGS) $(RACE_OPT) $(GOLDFLAGS) $(DEBUG_OPT) $(GOBUILD_OPT) -o $(BIN_NAME) ./cmd/mo-service # https://wiki.musl-libc.org/getting-started.html # https://musl.cc/ @@ -292,13 +320,13 @@ musl: override TAGS := -tags musl musl: musl-install musl-cgo config musl-thirdparties jieba-dict musl: $(info [Build binary(musl)]) - $(CGO_OPTS) go build $(GO_MODULE_MODE) $(TAGS) $(RACE_OPT) $(GOLDFLAGS) $(DEBUG_OPT) $(GOBUILD_OPT) -o $(BIN_NAME) ./cmd/mo-service + $(GOEXPERIMENT_OPT) $(CGO_OPTS) $(GO) build $(GO_MODULE_MODE) $(TAGS) $(RACE_OPT) $(GOLDFLAGS) $(DEBUG_OPT) $(GOBUILD_OPT) -o $(BIN_NAME) ./cmd/mo-service # build mo-tool .PHONY: mo-tool mo-tool: config cgo thirdparties $(info [Build mo-tool tool]) - $(CGO_OPTS) go build $(GO_MODULE_MODE) $(GOLDFLAGS) -o mo-tool ./cmd/mo-tool + $(GOEXPERIMENT_OPT) $(CGO_OPTS) $(GO) build $(GO_MODULE_MODE) $(GOLDFLAGS) -o mo-tool ./cmd/mo-tool # build mo-service binary for debugging with go's race detector enabled # produced executable is 10x slower and consumes much more memory diff --git a/cgo/cuvs/Makefile b/cgo/cuvs/Makefile index 5386d81ea2b25..cee1b339eb15f 100644 --- a/cgo/cuvs/Makefile +++ b/cgo/cuvs/Makefile @@ -128,6 +128,19 @@ test_kmeans: obj/test/test_kmeans.o $(OBJS) @echo "Linking $@" $(NVCC) $(LDFLAGS) $^ $(LIBS) -o $@ +# Standalone reproducer for the uint8 quantization recall collapse +# (f32->uint8 and f16->uint8). Has its own main(); runs in isolation +# rather than as part of the full test_cuvs_worker suite. +uint8_quant_bug: obj/test/uint8_quant_bug.o $(OBJS) + @echo "Linking $@" + $(NVCC) $(LDFLAGS) $^ $(LIBS) -o $@ + +# Real-dataset (wiki_all_1M) version of the uint8 collapse reproducer. Loads the +# .fbin base/queries + .ibin ground truth and grades recall@k at 1M scale. +wiki1m_uint8_bug: obj/test/wiki1m_uint8_bug.o $(OBJS) + @echo "Linking $@" + $(NVCC) $(LDFLAGS) $^ $(LIBS) -o $@ + # Standalone reproducer for the cuvs::neighbors::dynamic_batching deadlock # (conservative_dispatch=true). Intentionally depends on nothing in this # project — links only against the cuVS / RAFT / RMM libraries we already diff --git a/cgo/cuvs/brute_force.hpp b/cgo/cuvs/brute_force.hpp index f0046c3304a1e..7afc03ce3309f 100644 --- a/cgo/cuvs/brute_force.hpp +++ b/cgo/cuvs/brute_force.hpp @@ -102,7 +102,7 @@ namespace matrixone { // is_loaded_=true, then clears flattened_host_dataset to free memory. // The built index holds a device pointer to the dataset via // dataset_device_ptr_ (shared_ptr kept alive by index_). -// 4. search() / search_float() — dispatched via submit() (round-robin, but with +// 4. search() / search_quantize() — dispatched via submit() (round-robin, but with // SINGLE_GPU there is only one device). // 5. Destructor calls destroy() which stops the worker and resets index_. // @@ -125,7 +125,7 @@ namespace matrixone { // search_internal() holds a shared_lock during the GPU search call (read-only // access to index_). This is fine because brute_force is SINGLE_GPU and has no // concurrent extend path. -// search_float_internal() converts float queries to T on the device before +// search_quantize_internal() converts base (B) queries to T on the device before // searching (quantize for 1-byte T, half-cast for T=half, direct for T=float). // // SOFT-DELETE BITSET @@ -160,15 +160,21 @@ struct brute_force_search_result_t { /** * @brief gpu_brute_force_t implements a Brute Force index that can run on a single GPU. */ -template -class gpu_brute_force_t : public gpu_index_base_t { +// [B,Q] design (B = base/query element type, T = storage element type), mirroring +// gpu_cagra_t / gpu_ivf_pq_t. For the unquantized cases B==T; the overflow of a +// quantized index stores T (e.g. half) while the base/query is B (e.g. float), +// so search_quantize() converts B -> T (cast for f32->f16, learned SQ for 1-byte). +template +class gpu_brute_force_t : public gpu_index_base_t { public: + using base_type = B; + using storage_type = T; // We force DistT=float for all our indices to avoid template bloat and satisfy cuVS using brute_force_index = cuvs::neighbors::brute_force::index; using search_result_t = brute_force_search_result_t; // Inherited dependent type — bring into scope so search_internal can take a // const host_mask_bundle_t* parameter without `typename Base::...` everywhere. - using host_mask_bundle_t = typename gpu_index_base_t::host_mask_bundle_t; + using host_mask_bundle_t = typename gpu_index_base_t::host_mask_bundle_t; // Internal index storage std::unique_ptr index_; @@ -536,71 +542,73 @@ class gpu_brute_force_t : public gpu_index_base_tsearch_float_async(queries_data, num_queries, query_dimension, limit, sp); + // Sync quantize entry — wraps search_quantize_async + search_wait. The query + // is the BASE type B; search_quantize_internal converts it to storage T. + search_result_t search_quantize(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const brute_force_search_params_t& sp) { + uint64_t job_id = this->search_quantize_async(queries_data, num_queries, query_dimension, limit, sp); return this->search_wait(job_id); } - uint64_t search_float_async(const float* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const brute_force_search_params_t& sp) { - if constexpr (std::is_same_v) return search_async(queries_data, num_queries, query_dimension, limit, sp); + uint64_t search_quantize_async(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const brute_force_search_params_t& sp) { + if constexpr (std::is_same_v) return search_async(queries_data, num_queries, query_dimension, limit, sp); if (!queries_data) throw std::invalid_argument("search_async: queries_data is null"); if (num_queries == 0) throw std::invalid_argument("search_async: num_queries is 0"); if (this->dimension == 0) throw std::runtime_error("search_async: index dimension is 0"); // Reject a mismatched caller dim instead of silently coercing to - // this->dimension. search_float_internal sizes its H2D extent by + // this->dimension. search_quantize_internal sizes its H2D extent by // this->dimension; if caller's query_dimension differed we'd either // OOB-read or under-copy host queries. Fail loudly so the caller bug // surfaces here rather than as wrong search results. if (query_dimension != this->dimension) { throw std::invalid_argument( - "search_float_async: query_dimension (" + std::to_string(query_dimension) + + "search_quantize_async: query_dimension (" + std::to_string(query_dimension) + ") does not match index dimension (" + std::to_string(this->dimension) + ")"); } if (!this->is_loaded_ || !index_) throw std::runtime_error("search_async: index not loaded"); - auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); + auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); auto task = [this, num_queries, query_dimension, limit, sp, queries_copy](raft_handle_wrapper_t& handle) -> std::any { - return this->search_float_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp); + return this->search_quantize_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp); }; return this->worker->submit(task); } - // Sync float filtered entry — wraps search_float_with_filter_async + search_wait. - search_result_t search_float_with_filter(const float* queries_data, uint64_t num_queries, + // Sync quantize filtered entry — wraps search_quantize_with_filter_async + search_wait. + search_result_t search_quantize_with_filter(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const brute_force_search_params_t& sp, const std::string& preds_json) { - uint64_t job_id = this->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds_json); + uint64_t job_id = this->search_quantize_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds_json); return this->search_wait(job_id); } - // Async variant of search_float_with_filter. Brute force is single-GPU + // Async variant of search_quantize_with_filter. Brute force is single-GPU // only, so the bitmap eval stays on the calling thread and the GPU // search goes through worker->submit so concurrent calls can be // auto-batched in the device queue. Used by the multi-index brute-force // fallback so it dispatches in parallel with the primary IVF/CAGRA shards. - uint64_t search_float_with_filter_async(const float* queries_data, uint64_t num_queries, + // The query is the BASE type B; search_quantize_internal converts it to T. + uint64_t search_quantize_with_filter_async(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const brute_force_search_params_t& sp, const std::string& preds_json) { if (!queries_data) throw std::invalid_argument("search_async: queries_data is null"); if (num_queries == 0) throw std::invalid_argument("search_async: num_queries is 0"); if (this->dimension == 0) throw std::runtime_error("search_async: index dimension is 0"); - // See search_float_async() above for the rationale. + // See search_quantize_async() above for the rationale. if (query_dimension != this->dimension) { throw std::invalid_argument( - "search_float_with_filter_async: query_dimension (" + std::to_string(query_dimension) + + "search_quantize_with_filter_async: query_dimension (" + std::to_string(query_dimension) + ") does not match index dimension (" + std::to_string(this->dimension) + ")"); } if (!this->is_loaded_ || !index_) throw std::runtime_error("search_async: index not loaded"); if (!this->worker) throw std::runtime_error("Worker not initialized"); - auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); + auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); auto mask = this->build_filter_single_mask(preds_json); auto task = [this, num_queries, query_dimension, limit, sp, queries_copy, mask](raft_handle_wrapper_t& handle) -> std::any { - return this->search_float_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", mask.get()); + return this->search_quantize_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", mask.get()); }; return this->worker->submit(task); } @@ -608,7 +616,11 @@ class gpu_brute_force_t : public gpu_index_base_t int8/uint8; the remaining + // (B=float, T=half) instantiation casts f32 -> f16. + search_result_t search_quantize_internal(raft_handle_wrapper_t& handle, const B* queries_data, uint64_t num_queries, uint32_t /*query_dimension*/, uint32_t limit, const brute_force_search_params_t& /*sp*/, const std::string& /*preds_json*/ = "", const host_mask_bundle_t* prebuilt = nullptr) { // Same snapshot pattern as search_internal — see comment there. const brute_force_index* local_index = nullptr; uint64_t local_count = 0; @@ -616,7 +628,7 @@ class gpu_brute_force_t : public gpu_index_base_t lock(this->mutex_); if (!this->is_loaded_ || !this->index_) { - throw std::runtime_error("search_float_internal: index not loaded"); + throw std::runtime_error("search_quantize_internal: index not loaded"); } local_index = this->index_.get(); local_count = this->count; @@ -627,19 +639,20 @@ class gpu_brute_force_t : public gpu_index_base_t(*res, num_queries, this->dimension); - if constexpr (std::is_same_v) { - raft::copy(*res, q_dev_t.view(), raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + if constexpr (std::is_same_v) { + // B == T: no conversion. + raft::copy(*res, q_dev_t.view(), raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + } else if constexpr (sizeof(T) == 1) { + // sizeof(T) == 1: quantize the base-typed query B -> int8/uint8. + if (!this->quantizer_.is_trained()) throw std::runtime_error("Quantizer not trained"); + auto q_dev_b = raft::make_device_matrix(*res, num_queries, this->dimension); + raft::copy(*res, q_dev_b.view(), raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + this->quantizer_.template transform(*res, q_dev_b.view(), q_dev_t.data_handle(), true); } else { - auto q_dev_f = raft::make_device_matrix(*res, num_queries, this->dimension); - raft::copy(*res, q_dev_f.view(), raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); - - if constexpr (sizeof(T) == 1) { - if (!this->quantizer_.is_trained()) throw std::runtime_error("Quantizer not trained"); - this->quantizer_.template transform(*res, q_dev_f.view(), q_dev_t.data_handle(), true); - } else { - // T is half - raft::copy(*res, q_dev_t.view(), q_dev_f.view()); - } + // B != T and sizeof(T) != 1: (B=float, T=half) — cast f32 -> f16 on-device. + auto q_dev_b = raft::make_device_matrix(*res, num_queries, this->dimension); + raft::copy(*res, q_dev_b.view(), raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + raft::copy(*res, q_dev_t.view(), q_dev_b.view()); } // Legacy path syncs so the deletes-only sync_device_bitset below can // drain on the same stream. Prebuilt path skips: bitset H2D queues @@ -729,7 +742,7 @@ class gpu_brute_force_t : public gpu_index_base_t::info(); + std::string json = gpu_index_base_t::info(); json += ", \"type\": \"Brute-Force\", \"brute_force\": {"; if (index_) json += "\"built\": true"; else json += "\"built\": false"; diff --git a/cgo/cuvs/brute_force_c.cpp b/cgo/cuvs/brute_force_c.cpp index 2ae451dd44eb1..3f39133b621b6 100644 --- a/cgo/cuvs/brute_force_c.cpp +++ b/cgo/cuvs/brute_force_c.cpp @@ -1,4 +1,4 @@ -/* +/* * Copyright 2021 Matrix Origin * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +16,20 @@ /* * Brute-Force C Wrapper Implementation - * Supported data types (via quantization_t): Quantization_F32, Quantization_F16 + * + * Two type axes via quantization_t: + * btype = base / query / quantizer-SOURCE element type (Quantization_F32 or F16) + * qtype = storage element type (Quantization_F32 or F16) + * + * Wired (btype, qtype) combinations: + * F32 base: F32, F16 storage + * F16 base: F16 storage + * Any other combination throws "unsupported (base,storage) type combination". + * + * NOTE: unlike CAGRA, cuVS brute_force only provides build/search for + * index and index — there is NO int8_t/uint8_t + * storage path. So the INT8/UINT8 qtype combos are intentionally omitted + * here (instantiating them would fail to bind cuvs::brute_force::build/search). */ #include "brute_force_c.h" @@ -28,39 +41,70 @@ #include #include #include +#include +#include -struct gpu_brute_force_any_t { +using namespace matrixone; - quantization_t qtype; +struct gpu_brute_force_any_t { + quantization_t btype; // base / query / quantizer-source element type + quantization_t qtype; // storage element type void* ptr; - gpu_brute_force_any_t(quantization_t q, void* p) : qtype(q), ptr(p) {} - ~gpu_brute_force_any_t() { - switch (qtype) { - case Quantization_F32: delete static_cast*>(ptr); break; - case Quantization_F16: delete static_cast*>(ptr); break; - default: break; + gpu_brute_force_any_t(quantization_t b, quantization_t q, void* p) + : btype(b), qtype(q), ptr(p) {} + ~gpu_brute_force_any_t(); +}; + +// Static dispatch: resolves the concrete gpu_brute_force_t for (btype,qtype) +// and invokes fn with a typed pointer. fn is a generic lambda; recover B/Q inside +// it via decltype(idx)::base_type / ::storage_type. Throws on unsupported combos. +template +static auto brute_force_dispatch(const gpu_brute_force_any_t* a, Fn&& fn) { + switch (a->btype) { + case Quantization_F32: + switch (a->qtype) { + case Quantization_F32: return fn(static_cast*>(a->ptr)); + case Quantization_F16: return fn(static_cast*>(a->ptr)); + default: break; + } + break; + case Quantization_F16: + switch (a->qtype) { + case Quantization_F16: return fn(static_cast*>(a->ptr)); + default: break; } + break; + default: break; } -}; + throw std::runtime_error("gpu_brute_force: unsupported (base,storage) type combination"); +} + +gpu_brute_force_any_t::~gpu_brute_force_any_t() { + if (!ptr) return; + try { + brute_force_dispatch(this, [](auto* idx) { + delete idx; + }); + } catch (...) { + // unsupported combo never gets a live ptr — nothing to free + } +} extern "C" { -gpu_brute_force_c gpu_brute_force_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric_c, uint32_t nthread, int device_id, quantization_t qtype, const int64_t* ids, void* errmsg) { - void* index_ptr = nullptr; +gpu_brute_force_c gpu_brute_force_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric_c, uint32_t nthread, int device_id, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - switch (qtype) { - case Quantization_F32: - index_ptr = new matrixone::gpu_brute_force_t(static_cast(dataset_data), count_vectors, dimension, metric_c, nthread, device_id, ids); - break; - case Quantization_F16: - index_ptr = new matrixone::gpu_brute_force_t(static_cast(dataset_data), count_vectors, dimension, metric_c, nthread, device_id, ids); - break; - default: - throw std::runtime_error("Unsupported quantization type for brute force (only f32 and f16 supported)"); - } - return static_cast(new gpu_brute_force_any_t(qtype, index_ptr)); + // Construct the right gpu_brute_force_t; the native build + // constructor takes storage-typed (T) data. + std::unique_ptr holder(new gpu_brute_force_any_t(btype, qtype, nullptr)); + holder->ptr = brute_force_dispatch(holder.get(), [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using T = typename std::remove_pointer_t::storage_type; + return new gpu_brute_force_t(static_cast(dataset_data), count_vectors, dimension, metric_c, nthread, device_id, ids); + }); + return static_cast(holder.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_new", e.what()); @@ -72,21 +116,16 @@ gpu_brute_force_c gpu_brute_force_new(const void* dataset_data, uint64_t count_v } } -gpu_brute_force_c gpu_brute_force_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric_c, uint32_t nthread, int device_id, quantization_t qtype, const int64_t* ids, void* errmsg) { - void* index_ptr = nullptr; +gpu_brute_force_c gpu_brute_force_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric_c, uint32_t nthread, int device_id, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - switch (qtype) { - case Quantization_F32: - index_ptr = new matrixone::gpu_brute_force_t(total_count, dimension, metric_c, nthread, device_id, ids); - break; - case Quantization_F16: - index_ptr = new matrixone::gpu_brute_force_t(total_count, dimension, metric_c, nthread, device_id, ids); - break; - default: - throw std::runtime_error("Unsupported quantization type for brute force (only f32 and f16 supported)"); - } - return static_cast(new gpu_brute_force_any_t(qtype, index_ptr)); + std::unique_ptr holder(new gpu_brute_force_any_t(btype, qtype, nullptr)); + holder->ptr = brute_force_dispatch(holder.get(), [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using T = typename std::remove_pointer_t::storage_type; + return new gpu_brute_force_t(total_count, dimension, metric_c, nthread, device_id, ids); + }); + return static_cast(holder.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_new_empty", e.what()); @@ -101,12 +140,7 @@ gpu_brute_force_c gpu_brute_force_new_empty(uint64_t total_count, uint32_t dimen void gpu_brute_force_start(gpu_brute_force_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->start(); break; - case Quantization_F16: static_cast*>(any->ptr)->start(); break; - default: break; - } + brute_force_dispatch(static_cast(index_c), [](auto* idx) { idx->start(); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_start", e.what()); @@ -119,12 +153,7 @@ void gpu_brute_force_start(gpu_brute_force_c index_c, void* errmsg) { void gpu_brute_force_build(gpu_brute_force_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->build(); break; - case Quantization_F16: static_cast*>(any->ptr)->build(); break; - default: break; - } + brute_force_dispatch(static_cast(index_c), [](auto* idx) { idx->build(); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_build", e.what()); @@ -137,12 +166,10 @@ void gpu_brute_force_build(gpu_brute_force_c index_c, void* errmsg) { void gpu_brute_force_add_chunk(gpu_brute_force_c index_c, const void* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_F16: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - default: break; - } + brute_force_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + idx->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_add_chunk", e.what()); @@ -152,45 +179,35 @@ void gpu_brute_force_add_chunk(gpu_brute_force_c index_c, const void* chunk_data } } -void gpu_brute_force_add_chunk_float(gpu_brute_force_c index_c, const float* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { +// Base-typed (B) add: the add counterpart of search_quantize. queries_data is in +// the base element type B; add_chunk_quantize converts B -> storage T (native +// store when B==T, f32->f16 cast for (float,half), learned SQ for 1-byte). Used +// by the CDC overflow build to store base vectors at the index's storage type. +void gpu_brute_force_add_chunk_quantize(gpu_brute_force_c index_c, const void* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_F16: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - default: break; - } + brute_force_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + idx->add_chunk_quantize(static_cast(chunk_data), chunk_count, -1, ids); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, - "Error in gpu_brute_force_add_chunk_float", e.what()); + "Error in gpu_brute_force_add_chunk_quantize", e.what()); } catch (...) { matrixone::set_errmsg(errmsg, - "Error in gpu_brute_force_add_chunk_float", "unknown C++ exception"); + "Error in gpu_brute_force_add_chunk_quantize", "unknown C++ exception"); } } gpu_brute_force_search_result_c gpu_brute_force_search(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - void* result_ptr = nullptr; - switch (any->qtype) { - case Quantization_F32: { - auto res = std::make_unique::search_result_t>(); - *res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, brute_force_search_params_default()); - result_ptr = res.release(); - break; - } - case Quantization_F16: { - auto res = std::make_unique::search_result_t>(); - *res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, brute_force_search_params_default()); - result_ptr = res.release(); - break; - } - default: break; - } - return static_cast(result_ptr); + auto cpp_res = std::make_unique(); + brute_force_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + *cpp_res = idx->search(static_cast(queries_data), num_queries, query_dimension, limit, brute_force_search_params_default()); + }); + return static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search", e.what()); @@ -202,49 +219,35 @@ gpu_brute_force_search_result_c gpu_brute_force_search(gpu_brute_force_c index_c } } -gpu_brute_force_search_result_c gpu_brute_force_search_float(gpu_brute_force_c index_c, const float* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg) { +gpu_brute_force_search_result_c gpu_brute_force_search_quantize(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - void* result_ptr = nullptr; - switch (any->qtype) { - case Quantization_F32: { - auto res = std::make_unique::search_result_t>(); - *res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, brute_force_search_params_default()); - result_ptr = res.release(); - break; - } - case Quantization_F16: { - auto res = std::make_unique::search_result_t>(); - *res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, brute_force_search_params_default()); - result_ptr = res.release(); - break; - } - default: break; - } - return static_cast(result_ptr); + auto cpp_res = std::make_unique(); + brute_force_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + *cpp_res = idx->search_quantize(static_cast(queries_data), num_queries, query_dimension, limit, brute_force_search_params_default()); + }); + return static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, - "Error in gpu_brute_force_search_float", e.what()); + "Error in gpu_brute_force_search_quantize", e.what()); return nullptr; } catch (...) { matrixone::set_errmsg(errmsg, - "Error in gpu_brute_force_search_float", "unknown C++ exception"); + "Error in gpu_brute_force_search_quantize", "unknown C++ exception"); return nullptr; } } -uint64_t gpu_brute_force_search_async(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, +uint64_t gpu_brute_force_search_async(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); brute_force_search_params_t search_params; - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_F16: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - default: return 0; - } + return brute_force_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using Q = typename std::remove_pointer_t::storage_type; + return idx->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_async", e.what()); return 0; @@ -254,22 +257,20 @@ uint64_t gpu_brute_force_search_async(gpu_brute_force_c index_c, const void* que } } -uint64_t gpu_brute_force_search_float_async(gpu_brute_force_c index_c, const float* queries_data, uint64_t num_queries, +uint64_t gpu_brute_force_search_quantize_async(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); brute_force_search_params_t search_params; - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_F16: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - default: return 0; - } + return brute_force_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using B = typename std::remove_pointer_t::base_type; + return idx->search_quantize_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_float_async", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_quantize_async", e.what()); return 0; } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_float_async", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_quantize_async", "unknown C++ exception"); return 0; } } @@ -277,24 +278,11 @@ uint64_t gpu_brute_force_search_float_async(gpu_brute_force_c index_c, const flo gpu_brute_force_search_result_c gpu_brute_force_search_wait(gpu_brute_force_c index_c, uint64_t job_id, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - void* result_ptr = nullptr; - switch (any->qtype) { - case Quantization_F32: { - auto* cpp_res = new matrixone::gpu_brute_force_t::search_result_t(); - *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); - result_ptr = cpp_res; - break; - } - case Quantization_F16: { - auto* cpp_res = new matrixone::gpu_brute_force_t::search_result_t(); - *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); - result_ptr = cpp_res; - break; - } - default: break; - } - return static_cast(result_ptr); + auto cpp_res = std::make_unique(); + brute_force_dispatch(static_cast(index_c), [&](auto* idx) { + *cpp_res = idx->search_wait(job_id); + }); + return static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_wait", e.what()); return nullptr; @@ -307,7 +295,7 @@ gpu_brute_force_search_result_c gpu_brute_force_search_wait(gpu_brute_force_c in void gpu_brute_force_get_results(gpu_brute_force_search_result_c result_c, uint64_t num_queries, uint32_t limit, int64_t* neighbors, float* distances) { try { if (!result_c) return; - auto* search_result = static_cast::search_result_t*>(result_c); + auto* search_result = static_cast(result_c); size_t total = num_queries * limit; if (search_result->neighbors.size() >= total) { @@ -329,7 +317,7 @@ void gpu_brute_force_get_results(gpu_brute_force_search_result_c result_c, uint6 void gpu_brute_force_free_search_result(gpu_brute_force_search_result_c result_c) { try { if (!result_c) return; - delete static_cast::search_result_t*>(result_c); + delete static_cast(result_c); } catch (...) { matrixone::log_err("gpu_brute_force_free_search_result: unknown C++ exception (swallowed)"); } @@ -338,12 +326,7 @@ void gpu_brute_force_free_search_result(gpu_brute_force_search_result_c result_c uint64_t gpu_brute_force_cap(gpu_brute_force_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->cap(); - case Quantization_F16: return static_cast*>(any->ptr)->cap(); - default: return 0; - } + return brute_force_dispatch(static_cast(index_c), [](auto* idx) -> uint64_t { return idx->cap(); }); } catch (...) { return 0; } @@ -352,12 +335,7 @@ uint64_t gpu_brute_force_cap(gpu_brute_force_c index_c) { uint64_t gpu_brute_force_len(gpu_brute_force_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->len(); - case Quantization_F16: return static_cast*>(any->ptr)->len(); - default: return 0; - } + return brute_force_dispatch(static_cast(index_c), [](auto* idx) -> uint64_t { return idx->len(); }); } catch (...) { return 0; } @@ -367,13 +345,7 @@ char* gpu_brute_force_info(gpu_brute_force_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; if (!index_c) return nullptr; try { - auto* any = static_cast(index_c); - std::string info; - switch (any->qtype) { - case Quantization_F32: info = static_cast*>(any->ptr)->info(); break; - case Quantization_F16: info = static_cast*>(any->ptr)->info(); break; - default: return nullptr; - } + std::string info = brute_force_dispatch(static_cast(index_c), [](auto* idx) -> std::string { return idx->info(); }); return strdup(info.c_str()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, @@ -389,8 +361,7 @@ char* gpu_brute_force_info(gpu_brute_force_c index_c, void* errmsg) { void gpu_brute_force_destroy(gpu_brute_force_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - delete any; + delete static_cast(index_c); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_destroy", e.what()); @@ -406,13 +377,10 @@ void gpu_brute_force_set_filter_columns(gpu_brute_force_c index_c, const char* c uint64_t total_count, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); std::string meta = col_meta_json ? col_meta_json : ""; - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_filter_columns(meta, total_count); break; - case Quantization_F16: static_cast*>(any->ptr)->set_filter_columns(meta, total_count); break; - default: break; - } + brute_force_dispatch(static_cast(index_c), [&](auto* idx) { + idx->set_filter_columns(meta, total_count); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_set_filter_columns", e.what()); } catch (...) { @@ -425,12 +393,9 @@ void gpu_brute_force_add_filter_chunk(gpu_brute_force_c index_c, uint32_t col_id uint64_t nrows, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_F16: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - default: break; - } + brute_force_dispatch(static_cast(index_c), [&](auto* idx) { + idx->add_filter_chunk(col_idx, data, null_bitmap, nrows); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_add_filter_chunk", e.what()); } catch (...) { @@ -445,26 +410,14 @@ gpu_brute_force_search_result_c gpu_brute_force_search_with_filter(gpu_brute_for void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); brute_force_search_params_t sp; std::string preds = preds_json ? preds_json : ""; - void* result_ptr = nullptr; - switch (any->qtype) { - case Quantization_F32: { - auto* cpp_res = new matrixone::gpu_brute_force_t::search_result_t(); - *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); - result_ptr = cpp_res; - break; - } - case Quantization_F16: { - auto* cpp_res = new matrixone::gpu_brute_force_t::search_result_t(); - *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); - result_ptr = cpp_res; - break; - } - default: break; - } - return static_cast(result_ptr); + auto cpp_res = std::make_unique(); + brute_force_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + *cpp_res = idx->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); + return static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_with_filter", e.what()); return nullptr; @@ -474,62 +427,73 @@ gpu_brute_force_search_result_c gpu_brute_force_search_with_filter(gpu_brute_for } } -gpu_brute_force_search_result_c gpu_brute_force_search_float_with_filter(gpu_brute_force_c index_c, - const float* queries_data, +gpu_brute_force_search_result_c gpu_brute_force_search_quantize_with_filter(gpu_brute_force_c index_c, + const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const char* preds_json, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); brute_force_search_params_t sp; std::string preds = preds_json ? preds_json : ""; - void* result_ptr = nullptr; - switch (any->qtype) { - case Quantization_F32: { - auto* cpp_res = new matrixone::gpu_brute_force_t::search_result_t(); - *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); - result_ptr = cpp_res; - break; - } - case Quantization_F16: { - auto* cpp_res = new matrixone::gpu_brute_force_t::search_result_t(); - *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); - result_ptr = cpp_res; - break; - } - default: break; - } - return static_cast(result_ptr); + auto cpp_res = std::make_unique(); + brute_force_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + *cpp_res = idx->search_quantize_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); + return static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_float_with_filter", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_quantize_with_filter", e.what()); return nullptr; } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_float_with_filter", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_quantize_with_filter", "unknown C++ exception"); return nullptr; } } -uint64_t gpu_brute_force_search_float_with_filter_async(gpu_brute_force_c index_c, - const float* queries_data, +uint64_t gpu_brute_force_search_quantize_with_filter_async(gpu_brute_force_c index_c, + const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const char* preds_json, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); brute_force_search_params_t sp; std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_F16: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - default: return 0; - } + return brute_force_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using B = typename std::remove_pointer_t::base_type; + return idx->search_quantize_with_filter_async(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); + } catch (const std::exception& e) { + matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_quantize_with_filter_async", e.what()); + return 0; + } catch (...) { + matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_quantize_with_filter_async", "unknown C++ exception"); + return 0; + } +} + +// Native-typed (T) async filtered search. queries_data is in the index storage +// type T; no quantization/widening. Returns a job_id collected with +// gpu_brute_force_search_wait. Lets the filtered overflow stay native. +uint64_t gpu_brute_force_search_with_filter_async(gpu_brute_force_c index_c, + const void* queries_data, + uint64_t num_queries, uint32_t query_dimension, + uint32_t limit, const char* preds_json, + void* errmsg) { + if (errmsg) *(static_cast(errmsg)) = nullptr; + try { + brute_force_search_params_t sp; + std::string preds = preds_json ? preds_json : ""; + return brute_force_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using Q = typename std::remove_pointer_t::storage_type; + return idx->search_with_filter_async(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_float_with_filter_async", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_with_filter_async", e.what()); return 0; } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_float_with_filter_async", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_with_filter_async", "unknown C++ exception"); return 0; } } @@ -537,6 +501,7 @@ uint64_t gpu_brute_force_search_float_with_filter_async(gpu_brute_force_c index_ } // extern "C" namespace matrixone { -template class gpu_brute_force_t; -template class gpu_brute_force_t; +template class gpu_brute_force_t; +template class gpu_brute_force_t; +template class gpu_brute_force_t; } diff --git a/cgo/cuvs/brute_force_c.h b/cgo/cuvs/brute_force_c.h index ea989839f4c68..4d7d56e866fb5 100644 --- a/cgo/cuvs/brute_force_c.h +++ b/cgo/cuvs/brute_force_c.h @@ -31,10 +31,10 @@ typedef void* gpu_brute_force_c; typedef void* gpu_brute_force_search_result_c; // Constructor for gpu_brute_force_t -gpu_brute_force_c gpu_brute_force_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric, uint32_t nthread, int device_id, quantization_t qtype, const int64_t* ids, void* errmsg); +gpu_brute_force_c gpu_brute_force_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric, uint32_t nthread, int device_id, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg); // Constructor for an empty index (pre-allocates) -gpu_brute_force_c gpu_brute_force_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric, uint32_t nthread, int device_id, quantization_t qtype, const int64_t* ids, void* errmsg); +gpu_brute_force_c gpu_brute_force_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric, uint32_t nthread, int device_id, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg); // Starts the worker and initializes resources void gpu_brute_force_start(gpu_brute_force_c index_c, void* errmsg); @@ -45,20 +45,21 @@ void gpu_brute_force_build(gpu_brute_force_c index_c, void* errmsg); // Add chunk of data (same type as index quantization) void gpu_brute_force_add_chunk(gpu_brute_force_c index_c, const void* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); -// Add chunk of data (from float, with on-the-fly conversion if needed) -void gpu_brute_force_add_chunk_float(gpu_brute_force_c index_c, const float* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); +// Add chunk of base-typed (B) data; converts B -> storage T (the add counterpart +// of search_quantize: native store when B==T, f32->f16 cast, or learned SQ for 1-byte). +void gpu_brute_force_add_chunk_quantize(gpu_brute_force_c index_c, const void* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); // Performs a search operation gpu_brute_force_search_result_c gpu_brute_force_search(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg); -// Performs a search operation with float32 queries -gpu_brute_force_search_result_c gpu_brute_force_search_float(gpu_brute_force_c index_c, const float* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg); +// Performs a search operation with base-typed (B) queries; quantizes B -> storage T internally. +gpu_brute_force_search_result_c gpu_brute_force_search_quantize(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg); // Asynchronous search functions -uint64_t gpu_brute_force_search_async(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, +uint64_t gpu_brute_force_search_async(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg); -uint64_t gpu_brute_force_search_float_async(gpu_brute_force_c index_c, const float* queries_data, uint64_t num_queries, +uint64_t gpu_brute_force_search_quantize_async(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg); gpu_brute_force_search_result_c gpu_brute_force_search_wait(gpu_brute_force_c index_c, uint64_t job_id, void* errmsg); @@ -88,20 +89,30 @@ gpu_brute_force_search_result_c gpu_brute_force_search_with_filter(gpu_brute_for uint32_t limit, const char* preds_json, void* errmsg); -gpu_brute_force_search_result_c gpu_brute_force_search_float_with_filter(gpu_brute_force_c index_c, - const float* queries_data, +gpu_brute_force_search_result_c gpu_brute_force_search_quantize_with_filter(gpu_brute_force_c index_c, + const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const char* preds_json, void* errmsg); -// Async variant of gpu_brute_force_search_float_with_filter. Returns a job_id +// Async variant of gpu_brute_force_search_quantize_with_filter. Returns a job_id // that is collected with the existing gpu_brute_force_search_wait. -uint64_t gpu_brute_force_search_float_with_filter_async(gpu_brute_force_c index_c, - const float* queries_data, +uint64_t gpu_brute_force_search_quantize_with_filter_async(gpu_brute_force_c index_c, + const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const char* preds_json, void* errmsg); +// Native-typed (T) async variant of gpu_brute_force_search_with_filter: the +// query stays in the index element type T (f32 or f16), no widening. Returns a +// job_id collected with gpu_brute_force_search_wait. Lets the filtered overflow +// stay native half. +uint64_t gpu_brute_force_search_with_filter_async(gpu_brute_force_c index_c, + const void* queries_data, + uint64_t num_queries, uint32_t query_dimension, + uint32_t limit, const char* preds_json, + void* errmsg); + // Returns the capacity of the index buffer uint64_t gpu_brute_force_cap(gpu_brute_force_c index_c); diff --git a/cgo/cuvs/cagra.hpp b/cgo/cuvs/cagra.hpp index 078239ca96fb9..d9fbe68c3ffc3 100644 --- a/cgo/cuvs/cagra.hpp +++ b/cgo/cuvs/cagra.hpp @@ -97,7 +97,7 @@ namespace matrixone { // 2. Call start() — initializes the worker thread pool and CUDA context. // 3. Call build() — triggers CAGRA graph construction (or file load). // If is_loaded_ is already true (private constructor path), build() is a no-op. -// 4. Call search() / search_float() to query. +// 4. Call search() / search_quantize() to query. // 5. Call extend() / extend_float() to add new vectors (SINGLE_GPU or REPLICATED). // 6. Destructor calls destroy() which calls stop() on the worker. // @@ -183,7 +183,7 @@ namespace matrixone { // and for snapshotting count/dataset in build() // - NO lock during GPU calls themselves (build, extend, search kernels) // - shared_lock IS held during post-GPU CPU-side ID translation in search_internal / -// search_float_internal (protects host_ids and shard_sizes_ against concurrent extend) +// search_quantize_internal (protects host_ids and shard_sizes_ against concurrent extend) // - extend_mutex_ (std::mutex in base) serializes concurrent extend() callers // - Per-device bitset cache uses its own std::mutex (not the shared_mutex) // @@ -201,14 +201,16 @@ struct cagra_search_result_t { /** * @brief gpu_cagra_t implements a CAGRA index that can run on a single GPU or sharded across multiple GPUs. */ -template -class gpu_cagra_t : public gpu_index_base_t { +template +class gpu_cagra_t : public gpu_index_base_t { public: + using base_type = B; + using storage_type = T; using cagra_index = cuvs::neighbors::cagra::index; using search_result_t = cagra_search_result_t; // Inherited dependent type — bring into scope so search_internal can take a // const host_mask_bundle_t* parameter without `typename Base::...` everywhere. - using host_mask_bundle_t = typename gpu_index_base_t::host_mask_bundle_t; + using host_mask_bundle_t = typename gpu_index_base_t::host_mask_bundle_t; // Internal index storage std::unique_ptr index_; @@ -336,7 +338,7 @@ class gpu_cagra_t : public gpu_index_base_t { * @brief Merges multiple CAGRA indices into a single index. * Only works for SINGLE_GPU indices. */ - static std::unique_ptr> merge(const std::vector*>& base_indices, uint32_t nthread, const std::vector& devs) { + static std::unique_ptr> merge(const std::vector*>& base_indices, uint32_t nthread, const std::vector& devs) { if (base_indices.empty()) throw std::invalid_argument("base_indices empty"); uint32_t dim = base_indices[0]->dimension; @@ -351,7 +353,7 @@ class gpu_cagra_t : public gpu_index_base_t { std::vector cagra_indices; for (auto* bi : base_indices) { - auto* idx = static_cast*>(bi); + auto* idx = static_cast*>(bi); if (!idx->is_loaded_ || !idx->index_) { throw std::runtime_error("One of the indices to merge is not loaded or is a multi-GPU index."); } @@ -375,7 +377,7 @@ class gpu_cagra_t : public gpu_index_base_t { std::unique_ptr merged_idx(merged_idx_ptr); transient_worker.stop(); - auto new_idx = std::make_unique>( + auto new_idx = std::make_unique>( std::move(merged_idx), dim, m, nthread, devs ); @@ -428,6 +430,9 @@ class gpu_cagra_t : public gpu_index_base_t { // std::cout << "[DEBUG] CAGRA build: Starting build count=" << this->count << " dim=" << this->dimension << " metric=" << (int)this->metric << std::endl; + // 1-byte storage T: train the B-source quantizer on the buffered B + // sample, transform B->T, and store as T. For float/half storage this + // is a no-op. this->train_quantizer_if_needed(); if (!this->worker) throw std::runtime_error("Worker not initialized"); @@ -741,7 +746,32 @@ class gpu_cagra_t : public gpu_index_base_t { return this->search_wait(job_id); } - // Async T-typed filtered search. Mirrors search_float_with_filter_async + // Quantize a B-source query to the 1-byte storage type T via the B-source + // quantizer, writing num_queries*dimension T values into `out`. The caller + // then runs the normal native search(const T*) path. No f32 detour. + void quantize_query(const B* queries_data, uint64_t num_queries, T* out) { + if constexpr (sizeof(T) != 1) { + throw std::runtime_error("quantize_query requires a 1-byte storage type (int8/uint8)"); + } else { + uint64_t job = this->worker->submit_main( + [this, queries_data, num_queries, out](raft_handle_wrapper_t& handle) -> std::any { + auto res = handle.get_raft_resources(); + auto q_b_host = raft::make_host_matrix_view(queries_data, num_queries, this->dimension); + auto q_b_dev = raft::make_device_matrix(*res, num_queries, this->dimension); + raft::copy(*res, q_b_dev.view(), q_b_host); + if (!this->quantizer_.is_trained()) throw std::runtime_error("quantizer not trained"); + auto q_t_dev = raft::make_device_matrix(*res, num_queries, this->dimension); + this->quantizer_.template transform(*res, q_b_dev.view(), q_t_dev.data_handle(), true); + raft::copy(*res, raft::make_host_matrix_view(out, num_queries, this->dimension), q_t_dev.view()); + handle.sync(); + return std::any(); + }); + auto r = this->worker->wait(job).get(); + if (r.error) std::rethrow_exception(r.error); + } + } + + // Async T-typed filtered search. Mirrors search_quantize_with_filter_async // but for the T-typed query path (T may be float / half / int8 / uint8). // Build masks on the caller's thread, copy queries into a shared_ptr so // they outlive the Go caller, capture both in the worker lambda. @@ -1029,31 +1059,32 @@ class gpu_cagra_t : public gpu_index_base_t { } } - transform_distance(this->metric, search_res.distances); + transform_distance(this->metric, search_res.distances, this->quantized_l2_dequant_factor()); return search_res; } - // Sync float entry — wraps search_float_async + search_wait. - search_result_t search_float(const float* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const cagra_search_params_t& sp) { - uint64_t job_id = this->search_float_async(queries_data, num_queries, query_dimension, limit, sp); + // Sync quantize entry — wraps search_quantize_async + search_wait. + search_result_t search_quantize(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const cagra_search_params_t& sp) { + uint64_t job_id = this->search_quantize_async(queries_data, num_queries, query_dimension, limit, sp); return this->search_wait(job_id); } - // Sync float filtered entry — wraps search_float_with_filter_async + search_wait. - search_result_t search_float_with_filter(const float* queries_data, uint64_t num_queries, + // Sync quantize filtered entry — wraps search_quantize_with_filter_async + search_wait. + search_result_t search_quantize_with_filter(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const cagra_search_params_t& sp, const std::string& preds_json) { - uint64_t job_id = this->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds_json); + uint64_t job_id = this->search_quantize_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds_json); return this->search_wait(job_id); } - // Async variant of search_float_with_filter. Builds the host mask bundle on + // Async variant of search_quantize_with_filter. Builds the host mask bundle on // the calling thread (off-worker), copies queries into a shared_ptr so they // outlive the Go caller, captures both in the worker lambda, and returns a // job_id that search_wait() can collect. Used by the multi-index filter - // path so per-shard searches run in parallel. - uint64_t search_float_with_filter_async(const float* queries_data, uint64_t num_queries, + // path so per-shard searches run in parallel. The query is the BASE type B + // (float or half); search_quantize_internal converts it to storage T. + uint64_t search_quantize_with_filter_async(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const cagra_search_params_t& sp, const std::string& preds_json) { @@ -1066,7 +1097,7 @@ class gpu_cagra_t : public gpu_index_base_t { } if (!this->worker) throw std::runtime_error("Worker not initialized"); - auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); + auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); if (this->dist_mode == DistributionMode_SHARDED) { // Bitmap eval runs on the caller's (Go) thread — off-worker — so @@ -1076,7 +1107,7 @@ class gpu_cagra_t : public gpu_index_base_t { auto shard_masks = this->build_filter_shard_masks(preds_json); auto shard_search_task = [this, num_queries, query_dimension, limit, sp, queries_copy, shard_masks](raft_handle_wrapper_t& gpu_handle) -> std::any { int rank = gpu_handle.get_rank(); - return this->search_float_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", shard_masks[rank].get()); + return this->search_quantize_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", shard_masks[rank].get()); }; auto job_ids = this->worker->submit_all_devices_no_wait(shard_search_task); return this->worker->submit_composite_pending(std::move(job_ids), num_queries, limit); @@ -1088,12 +1119,12 @@ class gpu_cagra_t : public gpu_index_base_t { // would force serialization through main_thread_ and lose batching. auto mask = this->build_filter_single_mask(preds_json); auto task = [this, num_queries, query_dimension, limit, sp, queries_copy, mask](raft_handle_wrapper_t& handle) -> std::any { - return this->search_float_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", mask.get()); + return this->search_quantize_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", mask.get()); }; return this->worker->submit(task); } - uint64_t search_float_async(const float* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const cagra_search_params_t& sp) { + uint64_t search_quantize_async(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const cagra_search_params_t& sp) { if (!queries_data) throw std::invalid_argument("search_async: queries_data is null"); if (num_queries == 0) throw std::invalid_argument("search_async: num_queries is 0"); if (this->dimension == 0) throw std::runtime_error("search_async: index dimension is 0"); @@ -1102,13 +1133,13 @@ class gpu_cagra_t : public gpu_index_base_t { if (!this->is_loaded_ || (!index_ && this->replicated_indices_.empty())) throw std::runtime_error("search_async: index not loaded"); } - auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); + auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); if (this->dist_mode == DistributionMode_SHARDED) { // Same shape as search_async — fan out, hand back a composite id, // let search_wait() do the merge on the caller's thread. auto shard_search_task = [this, num_queries, query_dimension, limit, sp, queries_copy](raft_handle_wrapper_t& gpu_handle) -> std::any { - return this->search_float_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp); + return this->search_quantize_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp); }; auto job_ids = this->worker->submit_all_devices_no_wait(shard_search_task); return this->worker->submit_composite_pending(std::move(job_ids), num_queries, limit); @@ -1116,16 +1147,16 @@ class gpu_cagra_t : public gpu_index_base_t { // Single-GPU / replicated: the helper decides standalone vs fused; the // shared_ptr keeps the copied queries alive until the search runs. - return this->search_batchable_float(queries_copy, queries_copy->data(), num_queries, limit, sp); + return this->search_batchable_quantize(queries_copy, queries_copy->data(), num_queries, limit, sp); } - // float32-input search. Mirrors search_batchable_typed but calls - // search_float_internal; request-level batching (if enabled) happens inside it. - uint64_t search_batchable_float(std::shared_ptr> owner, const float* queries_data, + // Base-typed (B) quantize search. Mirrors search_batchable_typed but calls + // search_quantize_internal; request-level batching (if enabled) happens inside it. + uint64_t search_batchable_quantize(std::shared_ptr> owner, const B* queries_data, uint64_t num_queries, uint32_t limit, const cagra_search_params_t& sp) { if (!this->worker) throw std::runtime_error("Worker not initialized"); auto task = [this, owner, queries_data, num_queries, limit, sp](raft_handle_wrapper_t& handle) -> std::any { - return this->search_float_internal(handle, queries_data, num_queries, this->dimension, limit, sp); + return this->search_quantize_internal(handle, queries_data, num_queries, this->dimension, limit, sp); }; return this->worker->submit(task); } @@ -1134,7 +1165,14 @@ class gpu_cagra_t : public gpu_index_base_t { // semantics here (off-worker CPU mask eval, skip queries-H2D sync_stream // when prebuilt is non-null, kernel queues naturally behind the H2Ds on // the same stream). - search_result_t search_float_internal(raft_handle_wrapper_t& handle, const float* queries_data, uint64_t num_queries, uint32_t /*query_dimension*/, + // + // Takes the query in the BASE element type B (float or half) and converts + // it to the storage type T on-device: B==T is a plain copy, sizeof(T)==1 + // quantizes B -> int8/uint8 via the learned scalar quantizer, and the + // remaining (B=float, T=half) instantiation casts f32 -> f16. This is the + // "quantize" entry — see search_internal() for the already-storage-typed T + // path that performs no conversion. + search_result_t search_quantize_internal(raft_handle_wrapper_t& handle, const B* queries_data, uint64_t num_queries, uint32_t /*query_dimension*/, uint32_t limit, const cagra_search_params_t& sp, const std::string& preds_json = "", const host_mask_bundle_t* prebuilt = nullptr) { // No top-level lock: see search_internal() above — pointer fetched // via per-handle cache / narrow inner shared_lock, GPU work runs @@ -1147,26 +1185,30 @@ class gpu_cagra_t : public gpu_index_base_t { auto q_dev_t = raft::make_device_matrix_view( q_buf_t.data(), static_cast(num_queries), static_cast(this->dimension)); - if constexpr (std::is_same_v) { - raft::copy(*res, q_dev_t, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); - } else if constexpr (std::is_same_v) { - // Host-side fp32 → fp16 cast (F16C / AVX, IEEE round-to-nearest-even - // — bit-identical to mdspan_copy_kernel<__half>) into a pinned - // staging buffer, then a single half-sized H2D copy. Skips the - // q_dev_f device allocation and the mdspan_copy_kernel dispatch. + if constexpr (std::is_same_v) { + // B == T (float->float or half->half): no conversion, copy straight + // into the storage-typed workspace. + raft::copy(*res, q_dev_t, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + } else if constexpr (sizeof(T) == 1) { + // sizeof(T) == 1: quantize the base-typed query B -> int8/uint8. + // Stage the B query on its own per-thread device workspace (distinct + // from q_buf_t — see q_dev_buf), then transform B -> T on-device. + if (!this->quantizer_.is_trained()) throw std::runtime_error("Quantizer not trained"); + auto& q_buf_b = handle.template q_dev_buf(n_q_elems); + auto q_dev_b = raft::make_device_matrix_view( + q_buf_b.data(), static_cast(num_queries), static_cast(this->dimension)); + raft::copy(*res, q_dev_b, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + this->quantizer_.template transform(*res, q_dev_b, q_buf_t.data(), true); + } else { + // B != T and sizeof(T) != 1: the only such instantiation is + // B=float, T=half (f32 base -> fp16 storage). Host-side fp32 -> fp16 + // cast (F16C / AVX, IEEE round-to-nearest-even — bit-identical to + // mdspan_copy_kernel<__half>) into a pinned staging buffer, then a + // single half-sized H2D copy. __half* host_h = handle.ensure_host_half_buf(n_q_elems); matrixone::cast_float_to_half_host(queries_data, host_h, n_q_elems); raft::copy(*res, q_dev_t, raft::make_host_matrix_view(host_h, num_queries, this->dimension)); - } else { - // sizeof(T) == 1: int8 quantizer needs the fp32 device matrix. - auto& q_buf_f = handle.q_dev_buf_float(n_q_elems); - auto q_dev_f = raft::make_device_matrix_view( - q_buf_f.data(), static_cast(num_queries), static_cast(this->dimension)); - raft::copy(*res, q_dev_f, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); - - if (!this->quantizer_.is_trained()) throw std::runtime_error("Quantizer not trained"); - this->quantizer_.template transform(*res, q_dev_f, q_buf_t.data(), true); } // Legacy path syncs so build_search_bitset's stack-local host bitmap // can drain on the same stream. Prebuilt path skips: bitset H2D queues @@ -1311,12 +1353,12 @@ class gpu_cagra_t : public gpu_index_base_t { } } - transform_distance(this->metric, search_res.distances); + transform_distance(this->metric, search_res.distances, this->quantized_l2_dequant_factor()); return search_res; } std::string info() const override { - std::string json = gpu_index_base_t::info(); + std::string json = gpu_index_base_t::info(); json += ", \"type\": \"CAGRA\", \"cagra\": {"; std::shared_lock lock(this->mutex_); if (index_) json += "\"mode\": \"Single-GPU\", \"size\": " + std::to_string(index_->size()); diff --git a/cgo/cuvs/cagra_c.cpp b/cgo/cuvs/cagra_c.cpp index 907d52ff3ab95..0017aea01b0bd 100644 --- a/cgo/cuvs/cagra_c.cpp +++ b/cgo/cuvs/cagra_c.cpp @@ -1,4 +1,4 @@ -/* +/* * Copyright 2021 Matrix Origin * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +16,15 @@ /* * CAGRA C Wrapper Implementation - * Supported data types (via quantization_t): Quantization_F32, Quantization_F16, Quantization_INT8, Quantization_UINT8 + * + * Two type axes via quantization_t: + * btype = base / query / quantizer-SOURCE element type (Quantization_F32 or F16) + * qtype = storage element type (Quantization_F32, F16, INT8, UINT8) + * + * Wired (btype, qtype) combinations: + * F32 base: F32, F16, INT8, UINT8 storage + * F16 base: F16, INT8, UINT8 storage + * Any other combination throws "unsupported (base,storage) type combination". */ #include "cagra_c.h" @@ -27,78 +35,112 @@ #include #include #include +#include using namespace matrixone; struct gpu_cagra_any_t { - quantization_t qtype; + quantization_t btype; // base / query / quantizer-source element type + quantization_t qtype; // storage element type void* ptr; - gpu_cagra_any_t(quantization_t q, void* p) : qtype(q), ptr(p) {} - ~gpu_cagra_any_t() { + gpu_cagra_any_t(quantization_t b, quantization_t q, void* p) + : btype(b), qtype(q), ptr(p) {} + ~gpu_cagra_any_t(); +}; + +// Static dispatch: resolves the concrete gpu_cagra_t for (btype,qtype) and +// invokes fn with a typed pointer. fn is a generic lambda; recover B/Q inside it +// via decltype(idx)::base_type / ::storage_type. Throws on unsupported combos. +template +static auto cagra_dispatch(const gpu_cagra_any_t* a, Fn&& fn) { + switch (a->btype) { + case Quantization_F32: + switch (a->qtype) { + case Quantization_F32: return fn(static_cast*>(a->ptr)); + case Quantization_F16: return fn(static_cast*>(a->ptr)); + case Quantization_INT8: return fn(static_cast*>(a->ptr)); + case Quantization_UINT8: return fn(static_cast*>(a->ptr)); + default: break; + } + break; + case Quantization_F16: + switch (a->qtype) { + case Quantization_F16: return fn(static_cast*>(a->ptr)); + case Quantization_INT8: return fn(static_cast*>(a->ptr)); + case Quantization_UINT8: return fn(static_cast*>(a->ptr)); + default: break; + } + break; + default: break; + } + throw std::runtime_error("gpu_cagra: unsupported (base,storage) type combination"); +} + +gpu_cagra_any_t::~gpu_cagra_any_t() { + if (!ptr) return; + try { + cagra_dispatch(this, [](auto* idx) { + idx->destroy(); + delete idx; + }); + } catch (...) { + // unsupported combo never gets a live ptr — nothing to free + } +} + +// Construct a new gpu_cagra_t for the wired (btype,qtype) combos. +// Maker is a generic lambda invoked as maker(static type tag) -> void*; it +// receives a null typed pointer purely to recover B and Q. +template +static void* cagra_construct(quantization_t btype, quantization_t qtype, Maker&& maker) { + switch (btype) { + case Quantization_F32: switch (qtype) { - case Quantization_F32: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - case Quantization_F16: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - case Quantization_INT8: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - case Quantization_UINT8: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - default: break; + case Quantization_F32: return maker(static_cast*>(nullptr)); + case Quantization_F16: return maker(static_cast*>(nullptr)); + case Quantization_INT8: return maker(static_cast*>(nullptr)); + case Quantization_UINT8: return maker(static_cast*>(nullptr)); + default: break; + } + break; + case Quantization_F16: + switch (qtype) { + case Quantization_F16: return maker(static_cast*>(nullptr)); + case Quantization_INT8: return maker(static_cast*>(nullptr)); + case Quantization_UINT8: return maker(static_cast*>(nullptr)); + default: break; } + break; + default: break; } -}; + throw std::runtime_error("gpu_cagra: unsupported (base,storage) type combination"); +} extern "C" { gpu_cagra_c gpu_cagra_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric_c, cagra_build_params_t build_params, const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_cagra_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_F16: - ptr = new gpu_cagra_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_INT8: - ptr = new gpu_cagra_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_UINT8: - ptr = new gpu_cagra_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - default: return nullptr; - } - return static_cast(new gpu_cagra_any_t(qtype, ptr)); + std::unique_ptr holder(new gpu_cagra_any_t(btype, qtype, nullptr)); + holder->ptr = cagra_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + // The dataset-providing constructor takes storage-typed (Q) data and + // copies it directly into flattened_host_dataset (no quantization here; + // quantization happens via add_chunk_quantize / add_chunk_float). + return new gpu_cagra_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); + }); + return static_cast(holder.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_new", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_new", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_new", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_new", "unknown C++ exception"); } return nullptr; } @@ -106,68 +148,44 @@ gpu_cagra_c gpu_cagra_new(const void* dataset_data, uint64_t count_vectors, uint gpu_cagra_c gpu_cagra_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric_c, cagra_build_params_t build_params, const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_cagra_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_F16: - ptr = new gpu_cagra_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_INT8: - ptr = new gpu_cagra_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_UINT8: - ptr = new gpu_cagra_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - default: return nullptr; - } - return static_cast(new gpu_cagra_any_t(qtype, ptr)); + std::unique_ptr holder(new gpu_cagra_any_t(btype, qtype, nullptr)); + holder->ptr = cagra_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + return new gpu_cagra_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); + }); + return static_cast(holder.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_new_empty", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_new_empty", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_new_empty", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_new_empty", "unknown C++ exception"); } return nullptr; } gpu_cagra_c gpu_cagra_load_file(const char* filename, uint32_t dimension, distance_type_t metric_c, cagra_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, void* errmsg) { + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_cagra_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_F16: - ptr = new gpu_cagra_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_INT8: - ptr = new gpu_cagra_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_UINT8: - ptr = new gpu_cagra_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - default: return nullptr; - } - return static_cast(new gpu_cagra_any_t(qtype, ptr)); + std::unique_ptr holder(new gpu_cagra_any_t(btype, qtype, nullptr)); + holder->ptr = cagra_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + return new gpu_cagra_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); + }); + return static_cast(holder.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_load_file", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_load_file", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_load_file", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_load_file", "unknown C++ exception"); } return nullptr; } @@ -177,205 +195,152 @@ void gpu_cagra_destroy(gpu_cagra_c index_c, void* errmsg) { try { delete static_cast(index_c); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_destroy", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_destroy", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_destroy", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_destroy", "unknown C++ exception"); } } void gpu_cagra_start(gpu_cagra_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->start(); break; - case Quantization_F16: static_cast*>(any->ptr)->start(); break; - case Quantization_INT8: static_cast*>(any->ptr)->start(); break; - case Quantization_UINT8: static_cast*>(any->ptr)->start(); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [](auto* idx) { idx->start(); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_start", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_start", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_start", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_start", "unknown C++ exception"); } } void gpu_cagra_build(gpu_cagra_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->build(); break; - case Quantization_F16: static_cast*>(any->ptr)->build(); break; - case Quantization_INT8: static_cast*>(any->ptr)->build(); break; - case Quantization_UINT8: static_cast*>(any->ptr)->build(); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [](auto* idx) { idx->build(); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_build", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_build", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_build", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_build", "unknown C++ exception"); } } void gpu_cagra_add_chunk(gpu_cagra_c index_c, const void* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_F16: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + idx->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_add_chunk", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_add_chunk", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_add_chunk", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_add_chunk", "unknown C++ exception"); } } void gpu_cagra_add_chunk_float(gpu_cagra_c index_c, const float* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_F16: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + idx->add_chunk_float(chunk_data, chunk_count, -1, ids); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_add_chunk_float", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_add_chunk_float", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_add_chunk_float", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_add_chunk_float", "unknown C++ exception"); } } -void gpu_cagra_train_quantizer(gpu_cagra_c index_c, const float* train_data, uint64_t n_samples, void* errmsg) { +void gpu_cagra_add_chunk_quantize(gpu_cagra_c index_c, const void* base_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - case Quantization_F16: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - case Quantization_INT8: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - case Quantization_UINT8: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + idx->add_chunk_quantize(static_cast(base_data), chunk_count, -1, ids); + }); + } catch (const std::exception& e) { + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_add_chunk_quantize", e.what()); + } catch (...) { + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_add_chunk_quantize", "unknown C++ exception"); + } +} + +void gpu_cagra_quantize_query(gpu_cagra_c index_c, const void* base_data, uint64_t num_queries, void* out, void* errmsg) { + if (errmsg) *(static_cast(errmsg)) = nullptr; + try { + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + idx->quantize_query(static_cast(base_data), num_queries, static_cast(out)); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_train_quantizer", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_quantize_query", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_train_quantizer", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_quantize_query", "unknown C++ exception"); + } +} + +void gpu_cagra_train_quantizer(gpu_cagra_c index_c, const void* train_data, uint64_t n_samples, void* errmsg) { + if (errmsg) *(static_cast(errmsg)) = nullptr; + try { + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + idx->train_quantizer(static_cast(train_data), n_samples); + }); + } catch (const std::exception& e) { + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_train_quantizer", e.what()); + } catch (...) { + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_train_quantizer", "unknown C++ exception"); } } void gpu_cagra_set_batch_window(gpu_cagra_c index_c, int64_t window_us, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_batch_window(window_us); break; - case Quantization_F16: static_cast*>(any->ptr)->set_batch_window(window_us); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_batch_window(window_us); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_batch_window(window_us); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { idx->set_batch_window(window_us); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_set_batch_window", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_set_batch_window", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_set_batch_window", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_set_batch_window", "unknown C++ exception"); } } void gpu_cagra_set_dynb_conservative_dispatch(gpu_cagra_c index_c, bool enable, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - case Quantization_F16: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { idx->set_dynb_conservative_dispatch(enable); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_set_dynb_conservative_dispatch", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_set_dynb_conservative_dispatch", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_set_dynb_conservative_dispatch", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_set_dynb_conservative_dispatch", "unknown C++ exception"); } } void gpu_cagra_set_quantizer(gpu_cagra_c index_c, float min, float max, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_quantizer(min, max); break; - case Quantization_F16: static_cast*>(any->ptr)->set_quantizer(min, max); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_quantizer(min, max); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_quantizer(min, max); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { idx->set_quantizer(min, max); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_set_quantizer", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_set_quantizer", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_set_quantizer", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_set_quantizer", "unknown C++ exception"); } } void gpu_cagra_get_quantizer(gpu_cagra_c index_c, float* min, float* max, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->get_quantizer(min, max); break; - case Quantization_F16: static_cast*>(any->ptr)->get_quantizer(min, max); break; - case Quantization_INT8: static_cast*>(any->ptr)->get_quantizer(min, max); break; - case Quantization_UINT8: static_cast*>(any->ptr)->get_quantizer(min, max); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { idx->get_quantizer(min, max); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_get_quantizer", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_get_quantizer", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_get_quantizer", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_get_quantizer", "unknown C++ exception"); } } void gpu_cagra_save(gpu_cagra_c index_c, const char* filename, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->save(filename); break; - case Quantization_F16: static_cast*>(any->ptr)->save(filename); break; - case Quantization_INT8: static_cast*>(any->ptr)->save(filename); break; - case Quantization_UINT8: static_cast*>(any->ptr)->save(filename); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { idx->save(filename); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_save", e.what()); } catch (...) { @@ -386,14 +351,7 @@ void gpu_cagra_save(gpu_cagra_c index_c, const char* filename, void* errmsg) { void gpu_cagra_save_dir(gpu_cagra_c index_c, const char* dir, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->save_dir(dir); break; - case Quantization_F16: static_cast*>(any->ptr)->save_dir(dir); break; - case Quantization_INT8: static_cast*>(any->ptr)->save_dir(dir); break; - case Quantization_UINT8: static_cast*>(any->ptr)->save_dir(dir); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { idx->save_dir(dir); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_save_dir", e.what()); } catch (...) { @@ -404,14 +362,7 @@ void gpu_cagra_save_dir(gpu_cagra_c index_c, const char* dir, void* errmsg) { void gpu_cagra_delete_id(gpu_cagra_c index_c, int64_t id, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->delete_id(id); break; - case Quantization_F16: static_cast*>(any->ptr)->delete_id(id); break; - case Quantization_INT8: static_cast*>(any->ptr)->delete_id(id); break; - case Quantization_UINT8: static_cast*>(any->ptr)->delete_id(id); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { idx->delete_id(id); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_delete_id", e.what()); } catch (...) { @@ -423,14 +374,7 @@ void gpu_cagra_load_dir(gpu_cagra_c index_c, const char* dir, distribution_mode_t target_mode, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - case Quantization_F16: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - case Quantization_INT8: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - case Quantization_UINT8: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { idx->load_dir(dir, target_mode); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_load_dir", e.what()); } catch (...) { @@ -444,63 +388,49 @@ gpu_cagra_search_res_t gpu_cagra_search(gpu_cagra_c index_c, const void* queries if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_cagra_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + *cpp_res = idx->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_search", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_search", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search", "unknown C++ exception"); } return result; } -gpu_cagra_search_res_t gpu_cagra_search_float(gpu_cagra_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +gpu_cagra_search_res_t gpu_cagra_search_quantize(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_cagra_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + *cpp_res = idx->search_quantize(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_float", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_quantize", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_float", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_quantize", "unknown C++ exception"); } return result; } -uint64_t gpu_cagra_search_async(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_cagra_search_async(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_F16: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_INT8: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_UINT8: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - default: return 0; - } + return cagra_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using Q = typename std::remove_pointer_t::storage_type; + return idx->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_async", e.what()); return 0; @@ -510,24 +440,20 @@ uint64_t gpu_cagra_search_async(gpu_cagra_c index_c, const void* queries_data, u } } -uint64_t gpu_cagra_search_float_async(gpu_cagra_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_cagra_search_quantize_async(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_F16: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_INT8: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_UINT8: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - default: return 0; - } + return cagra_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using B = typename std::remove_pointer_t::base_type; + return idx->search_quantize_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_float_async", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_quantize_async", e.what()); return 0; } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_float_async", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_quantize_async", "unknown C++ exception"); return 0; } } @@ -536,15 +462,10 @@ gpu_cagra_search_res_t gpu_cagra_search_wait(gpu_cagra_c index_c, uint64_t job_i if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_cagra_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + *cpp_res = idx->search_wait(job_id); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_wait", e.what()); @@ -601,14 +522,7 @@ void gpu_cagra_free_result(gpu_cagra_result_c result_c) { uint64_t gpu_cagra_cap(gpu_cagra_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->cap(); - case Quantization_F16: return static_cast*>(any->ptr)->cap(); - case Quantization_INT8: return static_cast*>(any->ptr)->cap(); - case Quantization_UINT8: return static_cast*>(any->ptr)->cap(); - default: return 0; - } + return cagra_dispatch(static_cast(index_c), [](auto* idx) -> uint64_t { return idx->cap(); }); } catch (...) { return 0; } @@ -617,14 +531,7 @@ uint64_t gpu_cagra_cap(gpu_cagra_c index_c) { uint64_t gpu_cagra_len(gpu_cagra_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->len(); - case Quantization_F16: return static_cast*>(any->ptr)->len(); - case Quantization_INT8: return static_cast*>(any->ptr)->len(); - case Quantization_UINT8: return static_cast*>(any->ptr)->len(); - default: return 0; - } + return cagra_dispatch(static_cast(index_c), [](auto* idx) -> uint64_t { return idx->len(); }); } catch (...) { return 0; } @@ -640,15 +547,9 @@ char* gpu_cagra_get_filter_col_meta_json(gpu_cagra_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; if (!index_c) return strdup(""); try { - auto* any = static_cast(index_c); - std::string json; - switch (any->qtype) { - case Quantization_F32: json = matrixone::format_filter_col_meta(static_cast*>(any->ptr)->filter_host_.columns); break; - case Quantization_F16: json = matrixone::format_filter_col_meta(static_cast*>(any->ptr)->filter_host_.columns); break; - case Quantization_INT8: json = matrixone::format_filter_col_meta(static_cast*>(any->ptr)->filter_host_.columns); break; - case Quantization_UINT8: json = matrixone::format_filter_col_meta(static_cast*>(any->ptr)->filter_host_.columns); break; - default: return strdup(""); - } + std::string json = cagra_dispatch(static_cast(index_c), [](auto* idx) -> std::string { + return matrixone::format_filter_col_meta(idx->filter_host_.columns); + }); return strdup(json.c_str()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_get_filter_col_meta_json", e.what()); @@ -663,23 +564,13 @@ char* gpu_cagra_info(gpu_cagra_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; if (!index_c) return nullptr; try { - auto* any = static_cast(index_c); - std::string info; - switch (any->qtype) { - case Quantization_F32: info = static_cast*>(any->ptr)->info(); break; - case Quantization_F16: info = static_cast*>(any->ptr)->info(); break; - case Quantization_INT8: info = static_cast*>(any->ptr)->info(); break; - case Quantization_UINT8: info = static_cast*>(any->ptr)->info(); break; - default: return nullptr; - } + std::string info = cagra_dispatch(static_cast(index_c), [](auto* idx) -> std::string { return idx->info(); }); return strdup(info.c_str()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_info", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_info", e.what()); return nullptr; } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_info", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_info", "unknown C++ exception"); return nullptr; } } @@ -688,14 +579,10 @@ void gpu_cagra_extend(gpu_cagra_c index_c, const void* additional_data, uint64_t const int64_t* new_ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->extend(static_cast(additional_data), num_vectors, new_ids); break; - case Quantization_F16: static_cast*>(any->ptr)->extend(static_cast(additional_data), num_vectors, new_ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->extend(static_cast(additional_data), num_vectors, new_ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->extend(static_cast(additional_data), num_vectors, new_ids); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + idx->extend(static_cast(additional_data), num_vectors, new_ids); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_extend", e.what()); } catch (...) { @@ -708,44 +595,22 @@ gpu_cagra_c gpu_cagra_merge(gpu_cagra_c* indices_c, int num_indices, uint32_t nt try { if (num_indices <= 0) return nullptr; auto* first = static_cast(indices_c[0]); - quantization_t qtype = first->qtype; std::vector devs(devices, devices + device_count); - void* merged_ptr = nullptr; - - switch (qtype) { - case Quantization_F32: { - std::vector*> base_indices; - for (int i = 0; i < num_indices; ++i) base_indices.push_back(static_cast*>(static_cast(indices_c[i])->ptr)); - merged_ptr = gpu_cagra_t::merge(base_indices, nthread, devs).release(); - break; - } - case Quantization_F16: { - std::vector*> base_indices; - for (int i = 0; i < num_indices; ++i) base_indices.push_back(static_cast*>(static_cast(indices_c[i])->ptr)); - merged_ptr = gpu_cagra_t::merge(base_indices, nthread, devs).release(); - break; + std::unique_ptr holder(new gpu_cagra_any_t(first->btype, first->qtype, nullptr)); + holder->ptr = cagra_construct(first->btype, first->qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + std::vector*> base_indices; + for (int i = 0; i < num_indices; ++i) { + base_indices.push_back(static_cast*>(static_cast(indices_c[i])->ptr)); } - case Quantization_INT8: { - std::vector*> base_indices; - for (int i = 0; i < num_indices; ++i) base_indices.push_back(static_cast*>(static_cast(indices_c[i])->ptr)); - merged_ptr = gpu_cagra_t::merge(base_indices, nthread, devs).release(); - break; - } - case Quantization_UINT8: { - std::vector*> base_indices; - for (int i = 0; i < num_indices; ++i) base_indices.push_back(static_cast*>(static_cast(indices_c[i])->ptr)); - merged_ptr = gpu_cagra_t::merge(base_indices, nthread, devs).release(); - break; - } - default: return nullptr; - } - return static_cast(new gpu_cagra_any_t(qtype, merged_ptr)); + return gpu_cagra_t::merge(base_indices, nthread, devs).release(); + }); + return static_cast(holder.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_merge", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_merge", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_merge", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_merge", "unknown C++ exception"); } return nullptr; } @@ -756,15 +621,10 @@ void gpu_cagra_set_filter_columns(gpu_cagra_c index_c, const char* col_meta_json uint64_t total_count, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); std::string s = col_meta_json ? col_meta_json : ""; - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - case Quantization_F16: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + idx->set_filter_columns(s, total_count); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_set_filter_columns", e.what()); } catch (...) { @@ -777,14 +637,9 @@ void gpu_cagra_add_filter_chunk(gpu_cagra_c index_c, uint32_t col_idx, uint64_t nrows, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_F16: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_INT8: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_UINT8: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + idx->add_filter_chunk(col_idx, data, null_bitmap, nrows); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_add_filter_chunk", e.what()); } catch (...) { @@ -799,16 +654,12 @@ gpu_cagra_search_res_t gpu_cagra_search_with_filter(gpu_cagra_c index_c, const v if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_cagra_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + *cpp_res = idx->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_with_filter", e.what()); @@ -818,52 +669,44 @@ gpu_cagra_search_res_t gpu_cagra_search_with_filter(gpu_cagra_c index_c, const v return result; } -gpu_cagra_search_res_t gpu_cagra_search_float_with_filter(gpu_cagra_c index_c, const float* queries_data, +gpu_cagra_search_res_t gpu_cagra_search_quantize_with_filter(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, cagra_search_params_t sp, const char* preds_json, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_cagra_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + *cpp_res = idx->search_quantize_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_float_with_filter", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_quantize_with_filter", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_float_with_filter", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_quantize_with_filter", "unknown C++ exception"); } return result; } -uint64_t gpu_cagra_search_float_with_filter_async(gpu_cagra_c index_c, const float* queries_data, +uint64_t gpu_cagra_search_quantize_with_filter_async(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, cagra_search_params_t sp, const char* preds_json, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_F16: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_INT8: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_UINT8: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - default: return 0; - } + return cagra_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using B = typename std::remove_pointer_t::base_type; + return idx->search_quantize_with_filter_async(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_float_with_filter_async", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_quantize_with_filter_async", e.what()); return 0; } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_float_with_filter_async", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_quantize_with_filter_async", "unknown C++ exception"); return 0; } } @@ -871,8 +714,11 @@ uint64_t gpu_cagra_search_float_with_filter_async(gpu_cagra_c index_c, const flo } // extern "C" namespace matrixone { -template class gpu_cagra_t; -template class gpu_cagra_t; -template class gpu_cagra_t; -template class gpu_cagra_t; +template class gpu_cagra_t; +template class gpu_cagra_t; +template class gpu_cagra_t; +template class gpu_cagra_t; +template class gpu_cagra_t; +template class gpu_cagra_t; +template class gpu_cagra_t; } // namespace matrixone diff --git a/cgo/cuvs/cagra_c.h b/cgo/cuvs/cagra_c.h index 0e6a4fb463215..0d0a7229c7521 100644 --- a/cgo/cuvs/cagra_c.h +++ b/cgo/cuvs/cagra_c.h @@ -31,18 +31,22 @@ typedef void* gpu_cagra_c; // Opaque pointer to the C++ CAGRA search result object typedef void* gpu_cagra_result_c; +// btype = base/query/quantizer-source element type (Quantization_F32 or F16). +// qtype = storage element type. Wired combos: F32 base {F32,F16,INT8,UINT8}; +// F16 base {F16,INT8,UINT8}. Other combinations set errmsg and return NULL. + // Constructor for building from dataset gpu_cagra_c gpu_cagra_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric, cagra_build_params_t build_params, const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg); // Constructor for loading from file gpu_cagra_c gpu_cagra_load_file(const char* filename, uint32_t dimension, distance_type_t metric, cagra_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, void* errmsg); + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, void* errmsg); // Destructor void gpu_cagra_destroy(gpu_cagra_c index_c, void* errmsg); @@ -57,7 +61,7 @@ void gpu_cagra_build(gpu_cagra_c index_c, void* errmsg); gpu_cagra_c gpu_cagra_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric, cagra_build_params_t build_params, const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg); // Add chunk of data (same type as index quantization) @@ -66,8 +70,16 @@ void gpu_cagra_add_chunk(gpu_cagra_c index_c, const void* chunk_data, uint64_t c // Add chunk of data (from float, with on-the-fly quantization if needed) void gpu_cagra_add_chunk_float(gpu_cagra_c index_c, const float* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); +// Add chunk of base-typed (B) data, quantizing natively to a 1-byte storage type +// (int8/uint8) via the B-source quantizer. Requires int8/uint8 storage. +void gpu_cagra_add_chunk_quantize(gpu_cagra_c index_c, const void* base_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); + +// Quantize a base-typed (B) query to the 1-byte storage type via the B-source +// quantizer, writing num_queries*dimension bytes into out. Requires int8/uint8 storage. +void gpu_cagra_quantize_query(gpu_cagra_c index_c, const void* base_data, uint64_t num_queries, void* out, void* errmsg); + // Trains the scalar quantizer (if T is 1-byte) -void gpu_cagra_train_quantizer(gpu_cagra_c index_c, const float* train_data, uint64_t n_samples, void* errmsg); +void gpu_cagra_train_quantizer(gpu_cagra_c index_c, const void* train_data, uint64_t n_samples, void* errmsg); void gpu_cagra_set_batch_window(gpu_cagra_c index_c, int64_t window_us, void* errmsg); void gpu_cagra_set_dynb_conservative_dispatch(gpu_cagra_c index_c, bool enable, void* errmsg); @@ -100,17 +112,19 @@ gpu_cagra_search_res_t gpu_cagra_search(gpu_cagra_c index_c, const void* queries uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, void* errmsg); -gpu_cagra_search_res_t gpu_cagra_search_float(gpu_cagra_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +// Quantize search: query in the BASE element type B (float or half); the index +// converts it to storage type T (copy / quantize / f32->f16 cast) internally. +gpu_cagra_search_res_t gpu_cagra_search_quantize(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, void* errmsg); // Asynchronous search functions -uint64_t gpu_cagra_search_async(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_cagra_search_async(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, void* errmsg); -uint64_t gpu_cagra_search_float_async(gpu_cagra_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_cagra_search_quantize_async(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, void* errmsg); gpu_cagra_search_res_t gpu_cagra_search_wait(gpu_cagra_c index_c, uint64_t job_id, void* errmsg); @@ -162,22 +176,23 @@ void gpu_cagra_add_filter_chunk(gpu_cagra_c index_c, uint32_t col_idx, const void* data, const uint32_t* null_bitmap, uint64_t nrows, void* errmsg); -// Filtered variants of gpu_cagra_search / gpu_cagra_search_float. preds_json is a JSON +// Filtered variants of gpu_cagra_search / gpu_cagra_search_quantize. preds_json is a JSON // predicate array; passing NULL or "" yields unfiltered behavior. gpu_cagra_search_res_t gpu_cagra_search_with_filter(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, const char* preds_json, void* errmsg); -gpu_cagra_search_res_t gpu_cagra_search_float_with_filter(gpu_cagra_c index_c, const float* queries_data, +// Query in the BASE element type B (float or half); converted to storage T internally. +gpu_cagra_search_res_t gpu_cagra_search_quantize_with_filter(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, const char* preds_json, void* errmsg); -// Async variant of gpu_cagra_search_float_with_filter. Returns a job_id that +// Async variant of gpu_cagra_search_quantize_with_filter. Returns a job_id that // is collected with the existing gpu_cagra_search_wait. Lets multi-index // callers fan out filtered searches across shards in parallel. -uint64_t gpu_cagra_search_float_with_filter_async(gpu_cagra_c index_c, const float* queries_data, +uint64_t gpu_cagra_search_quantize_with_filter_async(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, const char* preds_json, void* errmsg); diff --git a/cgo/cuvs/cuvs_worker.hpp b/cgo/cuvs/cuvs_worker.hpp index 0e7eb624edc8c..d6581dec2b964 100644 --- a/cgo/cuvs/cuvs_worker.hpp +++ b/cgo/cuvs/cuvs_worker.hpp @@ -112,7 +112,7 @@ inline rmm::mr::device_memory_resource* worker_pool_mr(int device_id) { try { cudaSetDevice(device_id); auto base = std::make_shared(); - // Initial pool = 10% of free GPU memory; max = unbounded — pool + // Initial pool = 2% of free GPU memory; max = unbounded — pool // grows by allocating more from the upstream as needed. // Kept small so a subsequent huge-index load (e.g. an IVF-PQ or // CAGRA index needing ≥¾ VRAM in a single allocation) can still @@ -123,7 +123,7 @@ inline rmm::mr::device_memory_resource* worker_pool_mr(int device_id) { auto pool = std::make_shared< rmm::mr::pool_memory_resource>( base.get(), - rmm::percent_of_free_device_memory(10)); + rmm::percent_of_free_device_memory(2)); std::lock_guard lk(keepalive_mu); keepalives.push_back(pool); // pool outlives every device_uvector keepalives.push_back(base); // base outlives the pool diff --git a/cgo/cuvs/index_base.hpp b/cgo/cuvs/index_base.hpp index 6b4c6c810bb35..7cbf0bb192054 100644 --- a/cgo/cuvs/index_base.hpp +++ b/cgo/cuvs/index_base.hpp @@ -62,12 +62,12 @@ using ::distribution_mode_t; // // OVERVIEW // -------- -// gpu_index_base_t is the CRTP-style base class shared by +// gpu_index_base_t is the CRTP-style base class shared by // all three GPU index types: // -// gpu_ivf_flat_t (IdT = int64_t) -// gpu_ivf_pq_t (IdT = int64_t) -// gpu_cagra_t (IdT = uint32_t) +// gpu_ivf_flat_t (IdT = int64_t) // base hardcoded to float +// gpu_ivf_pq_t (IdT = int64_t) // B = base/source element type, T = storage type +// gpu_cagra_t (IdT = uint32_t) // B = base/source element type, T = storage type // // It provides: // - Pre-build vector buffering (flattened_host_dataset) @@ -202,17 +202,26 @@ using ::distribution_mode_t; // // QUANTIZER (1-byte types only: int8_t, uint8_t) // ------------------------------------------------ -// scalar_quantizer_t quantizer_ maps float32 values to [min, max] range -// and packs them into int8/uint8. It must be trained before add_chunk_float() -// or extend_float() is called for 1-byte types. +// scalar_quantizer_t quantizer_ maps source-type B values into the storage +// range [min, max] and packs them into int8/uint8. // -// Training: quantizer_.train(res, train_matrix) or train_quantizer(data, n). -// - Auto-training occurs in add_chunk_float if not yet trained (uses up to 500 -// samples from the first chunk). -// - For extend_float, the quantizer MUST already be trained (throws otherwise). +// Training (on the ORIGINAL float/half source data only): +// - add_chunk_float() / add_chunk_quantize() buffer their raw B chunks in +// pending_float_chunks_; flush_pending_float_chunks_internal() — invoked at +// build time via train_quantizer_if_needed() — trains the quantizer on ALL +// buffered rows at once, then quantizes them into storage. (No "first chunk" +// or 500-sample heuristic; the full buffered set is used.) +// - The quantizer is NEVER trained from flattened_host_dataset: for a 1-byte T +// that buffer holds only storage bytes, so training on it would learn the +// COMPRESSED range, not the original float range. +// - A pre-quantized index (rows added via add_chunk(T*), with no original +// floats and no set_quantizer()) therefore leaves the quantizer UNTRAINED. +// Base-typed (B) search/extend on it requires an explicit range via +// set_quantizer() first — quantize_query() (search) and +// upload_float_matrix_as_T() (extend) throw "quantizer not trained" otherwise. // -// Extended vectors must lie within the trained [min, max] range; vectors outside -// this range will be clamped and produce degraded search quality. +// Extended/searched vectors must lie within the trained [min, max] range; values +// outside it are clamped and produce degraded search quality. // // // SERIALIZATION (save_dir / load_dir) @@ -330,27 +339,37 @@ inline void fill_all_sentinel(NeighborT* neighbors, float* distances, std::fill_n(distances, count, std::numeric_limits::max()); } -// InnerProduct sign flip on the search result's distances. cuvs returns -// inner-product distances negated (so smaller is "closer") — we flip back so -// downstream callers see the true inner product. ±FLT_MAX sentinels are -// preserved (they mark padded / filtered-out slots from scatter_with_padding -// or fill_all_sentinel above). No-op for any other metric. +// Post-process a search result's distances in place: +// - InnerProduct: flip the sign. cuvs returns inner-product distances negated +// (so smaller is "closer"); we flip back so callers see the true IP. +// - quantized L2 (dequant_factor != 1): rescale the quantized-domain distance +// back to the base (f32) scale, so a 1-byte (int8/uint8) main index merges +// on the same scale as the base-typed CDC overflow brute force. The factor +// comes from quantized_l2_dequant_factor() (1/scalar^2 for squared L2). IP +// and L2-dequant are mutually exclusive — IP/cosine + int8/uint8 is rejected +// at plan time (an affine quantizer is not a pure rescale for IP/cosine). +// ±FLT_MAX sentinels (padded / filtered-out slots from scatter_with_padding or +// fill_all_sentinel above) are preserved. No-op for plain f32/f16 L2. inline void transform_distance(distance_type_t metric, - float* distances, size_t count) { - if (metric != DistanceType_InnerProduct) return; + float* distances, size_t count, + double dequant_factor = 1.0) { + const bool flip = (metric == DistanceType_InnerProduct); + const bool rescale = (dequant_factor != 1.0); + if (!flip && !rescale) return; const float kSentinel = std::numeric_limits::max(); for (size_t i = 0; i < count; ++i) { - if (distances[i] != kSentinel && distances[i] != -kSentinel) { - distances[i] *= -1.0f; - } + if (distances[i] == kSentinel || distances[i] == -kSentinel) continue; + if (flip) distances[i] *= -1.0f; + else distances[i] = static_cast(static_cast(distances[i]) * dequant_factor); } } // Convenience overload for the persistent-index path where distances live in // a std::vector. Same semantics as the (float*, size_t) form. inline void transform_distance(distance_type_t metric, - std::vector& distances) { - transform_distance(metric, distances.data(), distances.size()); + std::vector& distances, + double dequant_factor = 1.0) { + transform_distance(metric, distances.data(), distances.size(), dequant_factor); } /** @@ -359,13 +378,16 @@ inline void transform_distance(distance_type_t metric, * See the Developer Guide block above for full details on lifecycle, locking, * distribution modes, ID mapping, and the soft-delete bitset system. * - * @tparam T Element type: float, half (__half), int8_t, uint8_t + * @tparam B Base/query/quantizer-SOURCE element type: float or half + * @tparam T Storage element type: float, half (__half), int8_t, uint8_t * @tparam BuildParams Index-specific build parameter struct * @tparam IdT Neighbor ID type: int64_t (IVF) or uint32_t (CAGRA) */ -template +template class gpu_index_base_t { public: + using base_type = B; + using storage_type = T; // ---- Index configuration (immutable after build) ---- uint32_t dimension = 0; ///< Vector dimensionality distance_type_t metric; ///< Distance metric (L2, IP, cosine, ...) @@ -495,6 +517,42 @@ class gpu_index_base_t { return it->second; } + // Factor that rescales a quantized-domain L2 distance back to the base (f32) + // scale, for transform_distance(). For 1-byte storage (int8/uint8) the index + // computes L2 over the quantized vectors, where each element is + // q(x)=scalar*x+offset with scalar=255/(max-min); the per-element offset is a + // constant translation that cancels in a difference, so + // ||q(a)-q(b)||^2 = scalar^2*||a-b||^2 (and scalar*||a-b|| for the sqrt + // metrics). Returning 1/scalar^2 (resp. 1/scalar) undoes that, so a quantized + // main-index distance lands on the SAME scale as the base-typed CDC overflow + // brute force — otherwise mergeMultiResults compares scalar^2-scaled main + // distances against base-scale overflow distances and the overflow rows + // wrongly dominate the top-k. Also makes the reported l2_distance correct. + // + // Returns 1.0 (no-op) for plain f32/f16 storage, an untrained quantizer, a + // degenerate range, or a non-L2 metric (IP/cosine are not a pure rescale + // under an affine quantizer and are rejected at plan time). + double quantized_l2_dequant_factor() const { + if constexpr (sizeof(T) == 1) { + if (!this->quantizer_.is_trained()) return 1.0; + const double range = static_cast(this->quantizer_.max()) - + static_cast(this->quantizer_.min()); + if (!(range > 0.0)) return 1.0; + const double s = 255.0 / range; // scalar + switch (this->metric) { + case DistanceType_L2Expanded: + case DistanceType_L2Unexpanded: + return 1.0 / (s * s); // distances are squared L2 + case DistanceType_L2SqrtExpanded: + case DistanceType_L2SqrtUnexpanded: + return 1.0 / s; + default: + return 1.0; // IP / cosine: scale alone can't reconcile them + } + } + return 1.0; + } + // Sync a shard-local slice of the deleted bitset to device (SHARDED mode). // shard_offset must be a multiple of 32 (enforced at build time). // Bit j of the resulting device bitset = global bit (shard_offset + j). @@ -1060,15 +1118,15 @@ class gpu_index_base_t { auto res = handle.get_raft_resources(); - // --- GPU work: train quantizer on ALL pending float data — NO LOCK --- - std::vector all_floats; + // --- GPU work: train quantizer on ALL pending B-source data — NO LOCK --- + std::vector all_floats; all_floats.reserve(total * dimension); for (auto& c : chunks) { all_floats.insert(all_floats.end(), c.data.begin(), c.data.end()); } - auto train_host_view = raft::make_host_matrix_view( + auto train_host_view = raft::make_host_matrix_view( all_floats.data(), static_cast(total), static_cast(dimension)); - auto train_device = raft::make_device_matrix(*res, total, dimension); + auto train_device = raft::make_device_matrix(*res, total, dimension); raft::copy(*res, train_device.view(), train_host_view); // Train without holding the lock: GPU kernels run while lock is not held, // so concurrent readers are not blocked for the duration of training. @@ -1079,28 +1137,11 @@ class gpu_index_base_t { // acquisition by a reader is guaranteed to see is_trained() == true. { std::unique_lock _pub_lock(mutex_); } - // --- GPU work + locked store: process each buffered chunk --- + // --- Quantize each buffered chunk on the CPU and store. The quantizer + // is trained (above), so the B->T transform is a pure host affine map — + // no per-chunk GPU round-trip (this is what made a 1M-row f16-base / + // add_chunk_quantize build crawl). See transform_host(). --- for (auto& c : chunks) { - // Upload and quantize — NO LOCK - auto chunk_host_view = raft::make_host_matrix_view( - c.data.data(), static_cast(c.count), static_cast(dimension)); - auto chunk_device = raft::make_device_matrix(*res, c.count, dimension); - raft::copy(*res, chunk_device.view(), chunk_host_view); - - auto chunk_device_target = raft::make_device_matrix(*res, c.count, dimension); - - { - std::shared_lock lock(mutex_); - quantizer_.template transform(*res, chunk_device.view(), chunk_device_target.data_handle(), true); - } - - std::vector chunk_host_target(c.count * dimension); - raft::copy(*res, - raft::make_host_matrix_view(chunk_host_target.data(), static_cast(c.count), static_cast(dimension)), - chunk_device_target.view()); - handle.sync(); - - // Store into shared state — unique_lock std::unique_lock lock(mutex_); uint64_t target_offset; if (c.offset == -1) { @@ -1118,8 +1159,10 @@ class gpu_index_base_t { if (flattened_host_dataset.size() < required_elements) { flattened_host_dataset.resize(required_elements); } - std::copy(chunk_host_target.begin(), chunk_host_target.end(), - flattened_host_dataset.begin() + target_offset * dimension); + quantizer_.template transform_host( + c.data.data(), + flattened_host_dataset.data() + target_offset * dimension, + static_cast(c.count) * dimension); if (this->dist_mode == DistributionMode_SHARDED) { int num_shards = static_cast(this->devices_.size()); @@ -1152,10 +1195,16 @@ class gpu_index_base_t { if (is_loaded_) throw std::runtime_error("Cannot add chunk to built index"); } - auto res = handle.get_raft_resources(); - // If quantization is needed (T is 1-byte) if constexpr (sizeof(T) == 1) { + // The deferred-quantize buffer and the quantizer both work on + // the SOURCE type B. Convert the incoming f32 chunk to B once + // (identical bytes when B==float; per-element float->half cast + // when B==half). + std::vector chunk_b(chunk_count * dimension); + for (size_t i = 0; i < chunk_count * dimension; ++i) { + chunk_b[i] = static_cast(chunk_data[i]); + } bool trained; { std::shared_lock lock(mutex_); @@ -1165,7 +1214,7 @@ class gpu_index_base_t { if (!trained) { // Buffer this chunk for deferred training. pending_float_chunk_t c; - c.data.assign(chunk_data, chunk_data + chunk_count * dimension); + c.data = chunk_b; c.count = chunk_count; c.offset = offset; if (ids) c.ids.assign(ids, ids + chunk_count); @@ -1195,22 +1244,14 @@ class gpu_index_base_t { // c was NOT pushed to pending, so fall through to process chunk_data directly. } - // Quantizer already trained: quantize this chunk immediately. - auto queries_host_view = raft::make_host_matrix_view(chunk_data, chunk_count, dimension); - auto queries_device = raft::make_device_matrix(*res, chunk_count, dimension); - raft::copy(*res, queries_device.view(), queries_host_view); - - auto chunk_device_target = raft::make_device_matrix(*res, chunk_count, dimension); - - { - std::shared_lock lock(mutex_); - quantizer_.template transform(*res, queries_device.view(), chunk_device_target.data_handle(), true); - } - - std::vector chunk_host_target(chunk_count * dimension); - raft::copy(*res, raft::make_host_matrix_view(chunk_host_target.data(), chunk_count, dimension), chunk_device_target.view()); - handle.sync(); - + // Quantizer already trained: quantize on the CPU and write + // directly into flattened_host_dataset. Scalar quantization + // is a pure affine map from the trained [min,max], so no GPU + // round-trip (malloc + H2D copy + kernel + D2H copy + sync) + // is needed per chunk — this is the same host-only fast path + // as float/half storage. transform_host() produces bytes + // identical to the device transform(), so a CPU-quantized + // base stays consistent with a GPU-quantized query at search. std::unique_lock lock(mutex_); uint64_t target_offset; if (offset == -1) { @@ -1228,7 +1269,10 @@ class gpu_index_base_t { if (flattened_host_dataset.size() < required_elements) { flattened_host_dataset.resize(required_elements); } - std::copy(chunk_host_target.begin(), chunk_host_target.end(), flattened_host_dataset.begin() + (target_offset * dimension)); + quantizer_.template transform_host( + chunk_b.data(), + flattened_host_dataset.data() + (target_offset * dimension), + static_cast(chunk_count) * dimension); if (this->dist_mode == DistributionMode_SHARDED) { int num_shards = static_cast(this->devices_.size()); @@ -1288,12 +1332,12 @@ class gpu_index_base_t { if (res.error) std::rethrow_exception(res.error); } - void train_quantizer(const float* train_data, uint64_t n_samples) { + void train_quantizer(const B* train_data, uint64_t n_samples) { uint64_t job_id = worker->submit_main( [this, train_data, n_samples](raft_handle_wrapper_t& handle) -> std::any { auto res = handle.get_raft_resources(); - auto train_host_view = raft::make_host_matrix_view(train_data, n_samples, dimension); - auto train_device = raft::make_device_matrix(*res, n_samples, dimension); + auto train_host_view = raft::make_host_matrix_view(train_data, n_samples, dimension); + auto train_device = raft::make_device_matrix(*res, n_samples, dimension); raft::copy(*res, train_device.view(), train_host_view); quantizer_.train(*res, train_device.view()); handle.sync(); @@ -1311,44 +1355,23 @@ class gpu_index_base_t { // 1. Flush any buffered chunks first flush_pending_float_chunks_internal(handle); - // 2. Check if still not trained (might have used add_chunk instead of float). - // WARNING: if data was added via add_chunk(T*) rather than add_chunk_float(), - // flattened_host_dataset already holds T values (e.g. int8 in [-128,127]). - // Casting them to float trains the quantizer on the compressed range, not the - // original float range. extend_float() will then clamp to the wrong range. - // If extend_float() is needed after add_chunk(T*), call train_quantizer() - // explicitly with representative original float data before calling build(). - bool needs_training; - uint64_t n_train = 0; - { - std::shared_lock lock(mutex_); - needs_training = !quantizer_.is_trained() && !flattened_host_dataset.empty(); - if (needs_training) { - n_train = std::min(static_cast(500), count); - if (n_train == 0) needs_training = false; - } - } - - if (needs_training) { - std::vector train_data(n_train * dimension); - { - std::shared_lock lock(mutex_); - for (size_t i = 0; i < n_train * dimension; ++i) { - train_data[i] = static_cast(flattened_host_dataset[i]); - } - } - - auto res = handle.get_raft_resources(); - auto train_host_view = raft::make_host_matrix_view(train_data.data(), n_train, dimension); - auto train_device = raft::make_device_matrix(*res, n_train, dimension); - raft::copy(*res, train_device.view(), train_host_view); - - { - std::unique_lock lock(mutex_); - quantizer_.train(*res, train_device.view()); - } - handle.sync(); - } + // 2. Do NOT auto-train the quantizer from flattened_host_dataset. + // For a 1-byte storage type that buffer only ever holds STORAGE + // bytes — raw T from add_chunk(T*) (a pre-quantized index) or the + // post-flush quantized output — never original floats. Training on + // it would learn the COMPRESSED range (e.g. int8 [-128,127]) instead + // of the true float range, so later base-typed search/extend would + // silently quantize against the wrong min/max. + // + // Correct training happens above in flush_pending_float_chunks_internal() + // on the ORIGINAL floats buffered by add_chunk_float()/add_chunk_quantize(). + // A pre-quantized index (built solely via add_chunk(T*)) therefore + // leaves the quantizer untrained; base-typed (B) search/extend on it + // requires an explicit range via set_quantizer() first. Both base-typed + // entry points already throw "quantizer not trained" while it is + // untrained — search via quantize_query() and extend via + // upload_float_matrix_as_T() — so the op fails loudly instead of + // mis-quantizing against a wrong range. return std::any(); } ); @@ -1364,8 +1387,41 @@ class gpu_index_base_t { void get_quantizer(float* min, float* max) { std::shared_lock lock(mutex_); - *min = quantizer_.min(); - *max = quantizer_.max(); + *min = static_cast(quantizer_.min()); + *max = static_cast(quantizer_.max()); + } + + // ---- Native B-source quantization (base element B -> 1-byte T) ---- + // Base-typed (B) add: converts the SOURCE-typed chunk to storage T, the add + // counterpart of search_quantize (symmetric: same B->T conversion). Routes by + // (B,T): B==T is a native store; sizeof(T)==1 buffers the B chunk for deferred + // quantizer training (the B->T transform happens at build via + // flush_pending_float_chunks_internal — B==float and B==half both supported, + // no f32 detour for half); the remaining (B=float, T=half) case casts f32->f16 + // via add_chunk_float (std::copy into vector = __half assignment). + void add_chunk_quantize(const B* chunk_data, uint64_t chunk_count, int64_t offset = -1, const IdT* ids = nullptr) { + if constexpr (std::is_same_v) { + // B == T: no conversion — native storage add. + this->add_chunk(chunk_data, chunk_count, offset, ids); + } else if constexpr (sizeof(T) == 1) { + { + std::shared_lock lock(mutex_); + if (is_loaded_) throw std::runtime_error("Cannot add chunk to built index"); + } + pending_float_chunk_t c; + c.data.assign(chunk_data, chunk_data + chunk_count * dimension); + c.count = chunk_count; + c.offset = offset; + if (ids) c.ids.assign(ids, ids + chunk_count); + std::unique_lock lock(mutex_); + pending_total_count_ += chunk_count; + pending_float_chunks_.push_back(std::move(c)); + } else if constexpr (std::is_same_v) { + // B=float, T=half (sizeof(T)!=1): f32 -> T cast via add_chunk_float. + this->add_chunk_float(chunk_data, chunk_count, offset, ids); + } else { + throw std::runtime_error("add_chunk_quantize: unsupported (base,storage) type combination"); + } } // Returns a snapshot of host_ids by value. The previous signature @@ -1498,10 +1554,10 @@ class gpu_index_base_t { struct manifest_data_t { std::string raw; // full manifest.json content std::string comp_json; // "components" sub-object - bool has_ids = false; - bool has_quantizer = false; - bool has_bitset = false; - bool has_filter = false; + bool has_ids = false; + bool has_quantizer = false; + bool has_bitset = false; + bool has_filter = false; }; // Saves ids, quantizer, bitset, and filter data (when present) to dir. @@ -1513,23 +1569,23 @@ class gpu_index_base_t { FilterStore filter_snapshot; { std::shared_lock lock(mutex_); - has_ids = !this->host_ids.empty(); - has_quantizer = this->quantizer_.is_trained(); - has_bitset = !this->deleted_bitset_.empty(); - has_filter = !this->filter_host_.empty(); + has_ids = !this->host_ids.empty(); + has_quantizer = this->quantizer_.is_trained(); + has_bitset = !this->deleted_bitset_.empty(); + has_filter = !this->filter_host_.empty(); if (has_filter) filter_snapshot = this->filter_host_; // copy } - if (has_ids) this->save_ids(dir + "/ids.bin"); - if (has_quantizer) this->quantizer_.save_to_file(dir + "/quantizer.bin"); - if (has_bitset) this->save_bitset(dir); - if (has_filter) filter_snapshot.save(dir + "/filter_data.bin"); + if (has_ids) this->save_ids(dir + "/ids.bin"); + if (has_quantizer) this->quantizer_.save_to_file(dir + "/quantizer.bin"); + if (has_bitset) this->save_bitset(dir); + if (has_filter) filter_snapshot.save(dir + "/filter_data.bin"); std::vector entries; - if (has_ids) entries.push_back(" \"ids\": \"ids.bin\""); - if (has_quantizer) entries.push_back(" \"quantizer\": \"quantizer.bin\""); - if (has_bitset) entries.push_back(" \"bitset\": \"bitset.bin\""); - if (has_filter) entries.push_back(" \"filter_data\": \"filter_data.bin\""); + if (has_ids) entries.push_back(" \"ids\": \"ids.bin\""); + if (has_quantizer) entries.push_back(" \"quantizer\": \"quantizer.bin\""); + if (has_bitset) entries.push_back(" \"bitset\": \"bitset.bin\""); + if (has_filter) entries.push_back(" \"filter_data\": \"filter_data.bin\""); return entries; } @@ -1554,10 +1610,10 @@ class gpu_index_base_t { uint64_t cap_val, len_val, del_count, bs_ver; { std::shared_lock lock(mutex_); - has_ids = !this->host_ids.empty(); - has_quantizer = this->quantizer_.is_trained(); - has_bitset = !this->deleted_bitset_.empty(); - has_filter = !this->filter_host_.empty(); + has_ids = !this->host_ids.empty(); + has_quantizer = this->quantizer_.is_trained(); + has_bitset = !this->deleted_bitset_.empty(); + has_filter = !this->filter_host_.empty(); cap_val = this->count; len_val = this->current_offset_; del_count = this->deleted_count_; @@ -1624,11 +1680,11 @@ class gpu_index_base_t { manifest_data_t m; m.raw = raw; - m.comp_json = json_object(raw, "components"); - m.has_ids = json_bool(raw, "has_ids"); - m.has_quantizer = json_bool(raw, "has_quantizer"); - m.has_bitset = json_bool(raw, "has_bitset"); - m.has_filter = json_bool(raw, "has_filter"); + m.comp_json = json_object(raw, "components"); + m.has_ids = json_bool(raw, "has_ids"); + m.has_quantizer = json_bool(raw, "has_quantizer"); + m.has_bitset = json_bool(raw, "has_bitset"); + m.has_filter = json_bool(raw, "has_filter"); return m; } @@ -1670,7 +1726,10 @@ class gpu_index_base_t { } protected: - scalar_quantizer_t quantizer_; + // Scalar quantizer over the SOURCE element type B (float or half). Used only + // when the STORAGE type T is 1-byte (int8/uint8); for float/half storage the + // add path casts B->T directly with no quantizer. + scalar_quantizer_t quantizer_; uint64_t current_offset_ = 0; // Serializes concurrent extend() calls. Held across GPU work and count update so that // set_ids() offsets always match the GPU execution order. Does NOT block searches. @@ -1820,8 +1879,20 @@ class gpu_index_base_t { throw std::runtime_error( "upload_float_matrix_as_T: quantizer not trained"); } - this->quantizer_.template transform( - *res, float_view, storage.data(), true); + if constexpr (std::is_same_v) { + this->quantizer_.template transform( + *res, float_view, storage.data(), true); + } else { + // B == half: quantizer is half-source. Cast the f32 input to + // half on-device, then transform half -> T. + rmm::device_uvector b_storage( + static_cast(n_rows) * this->dimension, stream, matrixone::raw_device_mr()); + auto b_view = raft::make_device_matrix_view( + b_storage.data(), (int64_t)n_rows, (int64_t)this->dimension); + raft::copy(*res, b_view, float_view); + this->quantizer_.template transform( + *res, b_view, storage.data(), true); + } } else { // T is half — cast float → half raft::copy(*res, device_view, float_view); @@ -1830,13 +1901,15 @@ class gpu_index_base_t { return storage; } - // Deferred float chunk buffer for quantizer training (1-byte types only). - // See class-level comment block above for full description. + // Deferred B-source chunk buffer for quantizer training (1-byte storage T). + // See class-level comment block above for full description. Holds the raw + // SOURCE element type B (float or half); the quantizer trains on B and + // transforms B->T at flush time. struct pending_float_chunk_t { - std::vector data; ///< count * dimension floats - uint64_t count; - int64_t offset; ///< -1 = append; >= 0 = explicit position - std::vector ids; ///< empty if caller supplied no IDs + std::vector data; ///< count * dimension B elements + uint64_t count; + int64_t offset; ///< -1 = append; >= 0 = explicit position + std::vector ids; ///< empty if caller supplied no IDs }; static constexpr uint64_t kQuantizerTrainThreshold = 1000; std::vector pending_float_chunks_; diff --git a/cgo/cuvs/ivf_flat.hpp b/cgo/cuvs/ivf_flat.hpp index 414fd169dd831..e23d78893f134 100644 --- a/cgo/cuvs/ivf_flat.hpp +++ b/cgo/cuvs/ivf_flat.hpp @@ -106,10 +106,10 @@ namespace matrixone { // - Non-SHARDED: submit() (round-robin GPU assignment) // - SHARDED: submit_all_devices_no_wait() → matrixone::cpu_topk_merge_sharded() // -// search_float() is the same but accepts float32 queries and converts on the fly +// search_quantize() is the same but accepts base-typed (B) queries and converts on the fly // (via quantizer for 1-byte T, via half conversion for T=half, direct for T=float). // -// search_batchable_typed() / search_batchable_float() just submit the search to +// search_batchable_typed() / search_batchable_quantize() just submit the search to // the worker; request-level batching, when enabled (batch_window() > 0), // happens inside search_internal via cuVS dynamic_batching (see dynamic_batching.hpp). // @@ -139,15 +139,17 @@ struct ivf_flat_search_result_t { /** * @brief gpu_ivf_flat_t implements an IVF-Flat index that can run on a single GPU or sharded across multiple GPUs. */ -template -class gpu_ivf_flat_t : public gpu_index_base_t { +template +class gpu_ivf_flat_t : public gpu_index_base_t { public: + using base_type = B; + using storage_type = T; using ivf_flat_index = cuvs::neighbors::ivf_flat::index; using mg_index = cuvs::neighbors::mg_index; using search_result_t = ivf_flat_search_result_t; // Inherited dependent type — bring into scope so search_internal can take a // const host_mask_bundle_t* parameter without `typename Base::...` everywhere. - using host_mask_bundle_t = typename gpu_index_base_t::host_mask_bundle_t; + using host_mask_bundle_t = typename gpu_index_base_t::host_mask_bundle_t; std::unique_ptr index_; std::string data_filename_; @@ -611,8 +613,8 @@ class gpu_ivf_flat_t : public gpu_index_base_tsearch_wait(job_id); } - // Async T-typed filtered search. Mirrors search_float_with_filter_async - // but uses search_internal (T) instead of search_float_internal (float). + // Async T-typed filtered search. Mirrors search_quantize_with_filter_async + // but uses search_internal (T) instead of search_quantize_internal (B). uint64_t search_with_filter_async(const T* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_flat_search_params_t& sp, @@ -715,27 +717,28 @@ class gpu_ivf_flat_t : public gpu_index_base_tworker->submit(task); } - // Sync float entry — wraps search_float_async + search_wait. - search_result_t search_float(const float* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_flat_search_params_t& sp) { - uint64_t job_id = this->search_float_async(queries_data, num_queries, query_dimension, limit, sp); + // Sync quantize entry — wraps search_quantize_async + search_wait. + search_result_t search_quantize(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_flat_search_params_t& sp) { + uint64_t job_id = this->search_quantize_async(queries_data, num_queries, query_dimension, limit, sp); return this->search_wait(job_id); } - // Sync float filtered entry — wraps search_float_with_filter_async + search_wait. - search_result_t search_float_with_filter(const float* queries_data, uint64_t num_queries, + // Sync quantize filtered entry — wraps search_quantize_with_filter_async + search_wait. + search_result_t search_quantize_with_filter(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_flat_search_params_t& sp, const std::string& preds_json) { - uint64_t job_id = this->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds_json); + uint64_t job_id = this->search_quantize_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds_json); return this->search_wait(job_id); } - // Async variant of search_float_with_filter. Builds the host mask bundle on + // Async variant of search_quantize_with_filter. Builds the host mask bundle on // the calling thread (off-worker), copies queries into a shared_ptr so they // outlive the Go caller, captures both in the worker lambda, and returns a // job_id that search_wait() can collect. Used by the multi-index filter - // path so per-shard searches run in parallel. - uint64_t search_float_with_filter_async(const float* queries_data, uint64_t num_queries, + // path so per-shard searches run in parallel. The query is the BASE type B + // (float or half); search_quantize_internal converts it to storage T. + uint64_t search_quantize_with_filter_async(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_flat_search_params_t& sp, const std::string& preds_json) { @@ -748,7 +751,7 @@ class gpu_ivf_flat_t : public gpu_index_base_tworker) throw std::runtime_error("Worker not initialized"); - auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); + auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); if (this->dist_mode == DistributionMode_SHARDED) { // Bitmap eval runs on the caller's (Go) thread; per-shard searches @@ -757,7 +760,7 @@ class gpu_ivf_flat_t : public gpu_index_base_tbuild_filter_shard_masks(preds_json); auto shard_search_task = [this, num_queries, query_dimension, limit, sp, queries_copy, shard_masks](raft_handle_wrapper_t& gpu_handle) -> std::any { int rank = gpu_handle.get_rank(); - return this->search_float_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", shard_masks[rank].get()); + return this->search_quantize_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", shard_masks[rank].get()); }; auto job_ids = this->worker->submit_all_devices_no_wait(shard_search_task); return this->worker->submit_composite_pending(std::move(job_ids), num_queries, limit); @@ -769,12 +772,12 @@ class gpu_ivf_flat_t : public gpu_index_base_tbuild_filter_single_mask(preds_json); auto task = [this, num_queries, query_dimension, limit, sp, queries_copy, mask](raft_handle_wrapper_t& handle) -> std::any { - return this->search_float_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", mask.get()); + return this->search_quantize_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", mask.get()); }; return this->worker->submit(task); } - uint64_t search_float_async(const float* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_flat_search_params_t& sp) { + uint64_t search_quantize_async(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_flat_search_params_t& sp) { if (!queries_data) throw std::invalid_argument("search_async: queries_data is null"); if (num_queries == 0) throw std::invalid_argument("search_async: num_queries is 0"); if (this->dimension == 0) throw std::runtime_error("search_async: index dimension is 0"); @@ -783,13 +786,13 @@ class gpu_ivf_flat_t : public gpu_index_base_tis_loaded_ || (!index_ && this->replicated_indices_.empty())) throw std::runtime_error("search_async: index not loaded"); } - auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); + auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); if (this->dist_mode == DistributionMode_SHARDED) { // Same shape as search_async — fan out, hand back a composite id, // let search_wait() do the merge on the caller's thread. auto shard_search_task = [this, num_queries, query_dimension, limit, sp, queries_copy](raft_handle_wrapper_t& gpu_handle) -> std::any { - return this->search_float_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp); + return this->search_quantize_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp); }; auto job_ids = this->worker->submit_all_devices_no_wait(shard_search_task); return this->worker->submit_composite_pending(std::move(job_ids), num_queries, limit); @@ -797,16 +800,16 @@ class gpu_ivf_flat_t : public gpu_index_base_tsearch_batchable_float(queries_copy, queries_copy->data(), num_queries, limit, sp); + return this->search_batchable_quantize(queries_copy, queries_copy->data(), num_queries, limit, sp); } - // float32-input search. Mirrors search_batchable_typed but calls - // search_float_internal; request-level batching (if enabled) happens inside it. - uint64_t search_batchable_float(std::shared_ptr> owner, const float* queries_data, + // Base-typed (B) quantize search. Mirrors search_batchable_typed but calls + // search_quantize_internal; request-level batching (if enabled) happens inside it. + uint64_t search_batchable_quantize(std::shared_ptr> owner, const B* queries_data, uint64_t num_queries, uint32_t limit, const ivf_flat_search_params_t& sp) { if (!this->worker) throw std::runtime_error("Worker not initialized"); auto task = [this, owner, queries_data, num_queries, limit, sp](raft_handle_wrapper_t& handle) -> std::any { - return this->search_float_internal(handle, queries_data, num_queries, this->dimension, limit, sp); + return this->search_quantize_internal(handle, queries_data, num_queries, this->dimension, limit, sp); }; return this->worker->submit(task); } @@ -983,14 +986,18 @@ class gpu_ivf_flat_t : public gpu_index_base_tmetric, search_res.distances); + transform_distance(this->metric, search_res.distances, this->quantized_l2_dequant_factor()); return search_res; } // See search_internal() above for the prebuilt-bundle contract; identical // semantics here (off-worker CPU mask eval, skip queries-H2D sync_stream // when prebuilt is non-null). - search_result_t search_float_internal(raft_handle_wrapper_t& handle, const float* queries_data, uint64_t num_queries, uint32_t /*query_dimension*/, uint32_t limit, const ivf_flat_search_params_t& sp, const std::string& preds_json = "", const host_mask_bundle_t* prebuilt = nullptr) { + // Takes the query in the BASE element type B (float or half) and converts + // it to the storage type T on-device — see the cagra search_quantize_internal + // comment. B==T copies straight, sizeof(T)==1 quantizes B -> int8/uint8, and + // the (B=float, T=half) instantiation casts f32 -> f16 on the host. + search_result_t search_quantize_internal(raft_handle_wrapper_t& handle, const B* queries_data, uint64_t num_queries, uint32_t /*query_dimension*/, uint32_t limit, const ivf_flat_search_params_t& sp, const std::string& preds_json = "", const host_mask_bundle_t* prebuilt = nullptr) { // No top-level lock: see search_internal() above — pointer fetched // via per-handle cache / narrow inner shared_lock, GPU work runs // unlocked. @@ -1002,26 +1009,28 @@ class gpu_ivf_flat_t : public gpu_index_base_t( q_buf_t.data(), static_cast(num_queries), static_cast(this->dimension)); - if constexpr (std::is_same_v) { - raft::copy(*res, q_dev_t, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); - } else if constexpr (std::is_same_v) { - // Host-side fp32 → fp16 cast (F16C / AVX, IEEE round-to-nearest-even - // — bit-identical to mdspan_copy_kernel<__half>) into a pinned - // staging buffer, then a single half-sized H2D copy. Skips the - // q_dev_f device allocation and the mdspan_copy_kernel dispatch. + if constexpr (std::is_same_v) { + // B == T (float->float or half->half): no conversion. + raft::copy(*res, q_dev_t, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + } else if constexpr (sizeof(T) == 1) { + // sizeof(T) == 1: quantize the base-typed query B -> int8/uint8. + // Stage the B query on its own per-thread device workspace (distinct + // from q_buf_t — see q_dev_buf), then transform B -> T on-device. + if (!this->quantizer_.is_trained()) throw std::runtime_error("Quantizer not trained"); + auto& q_buf_b = handle.template q_dev_buf(n_q_elems); + auto q_dev_b = raft::make_device_matrix_view( + q_buf_b.data(), static_cast(num_queries), static_cast(this->dimension)); + raft::copy(*res, q_dev_b, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + this->quantizer_.template transform(*res, q_dev_b, q_buf_t.data(), true); + } else { + // B != T and sizeof(T) != 1: only (B=float, T=half). Host fp32 -> fp16 + // cast (F16C / AVX, IEEE round-to-nearest-even — bit-identical to + // mdspan_copy_kernel<__half>) into a pinned staging buffer, then a + // single half-sized H2D copy. __half* host_h = handle.ensure_host_half_buf(n_q_elems); matrixone::cast_float_to_half_host(queries_data, host_h, n_q_elems); raft::copy(*res, q_dev_t, raft::make_host_matrix_view(host_h, num_queries, this->dimension)); - } else { - // sizeof(T) == 1: int8 quantizer needs the fp32 device matrix. - auto& q_buf_f = handle.q_dev_buf_float(n_q_elems); - auto q_dev_f = raft::make_device_matrix_view( - q_buf_f.data(), static_cast(num_queries), static_cast(this->dimension)); - raft::copy(*res, q_dev_f, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); - - if (!this->quantizer_.is_trained()) throw std::runtime_error("Quantizer not trained"); - this->quantizer_.template transform(*res, q_dev_f, q_buf_t.data(), true); } // Legacy path syncs so build_search_bitset's stack-local host bitmap can // drain on the same stream. Prebuilt path skips: bitset H2D queues @@ -1175,7 +1184,7 @@ class gpu_ivf_flat_t : public gpu_index_base_tmetric, search_res.distances); + transform_distance(this->metric, search_res.distances, this->quantized_l2_dequant_factor()); return search_res; } @@ -1451,7 +1460,14 @@ class gpu_ivf_flat_t : public gpu_index_base_t(*res, n_centers, dim); if constexpr (sizeof(T) == 1) { auto centers_float_view = raft::make_device_matrix_view(centers_view.data_handle(), n_centers, dim); - this->quantizer_.template transform(*res, centers_float_view, centers_device_target.data_handle(), true); + if constexpr (std::is_same_v) { + this->quantizer_.template transform(*res, centers_float_view, centers_device_target.data_handle(), true); + } else { + // B == half: cast the float centers to B on-device, then quantize B -> T. + auto centers_b = raft::make_device_matrix(*res, n_centers, dim); + raft::copy(*res, centers_b.view(), centers_float_view); + this->quantizer_.template transform(*res, centers_b.view(), centers_device_target.data_handle(), true); + } } else { raft::copy(*res, centers_device_target.view(), centers_view); } @@ -1470,7 +1486,7 @@ class gpu_ivf_flat_t : public gpu_index_base_t::info(); + std::string json = gpu_index_base_t::info(); json += ", \"type\": \"IVF-Flat\", \"ivf_flat\": {"; if (index_) json += "\"mode\": \"Single-GPU\", \"size\": " + std::to_string(index_->size()); else if (!this->replicated_indices_.empty()) json += "\"mode\": \"Local-Indices\", \"ranks\": " + std::to_string(this->replicated_indices_.size()); diff --git a/cgo/cuvs/ivf_flat_c.cpp b/cgo/cuvs/ivf_flat_c.cpp index 6f59272749330..7e18f27049ac6 100644 --- a/cgo/cuvs/ivf_flat_c.cpp +++ b/cgo/cuvs/ivf_flat_c.cpp @@ -1,4 +1,4 @@ -/* +/* * Copyright 2021 Matrix Origin * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +16,15 @@ /* * IVF-Flat C Wrapper Implementation - * Supported data types (via quantization_t): Quantization_F32, Quantization_F16, Quantization_INT8, Quantization_UINT8 + * + * Two type axes via quantization_t: + * btype = base / query / quantizer-SOURCE element type (Quantization_F32 or F16) + * qtype = storage element type (Quantization_F32, F16, INT8, UINT8) + * + * Wired (btype, qtype) combinations: + * F32 base: F32, F16, INT8, UINT8 storage + * F16 base: F16, INT8, UINT8 storage + * Any other combination throws "unsupported (base,storage) type combination". */ #include "ivf_flat_c.h" @@ -27,146 +35,155 @@ #include #include #include +#include using namespace matrixone; struct gpu_ivf_flat_any_t { - quantization_t qtype; + quantization_t btype; // base / query / quantizer-source element type + quantization_t qtype; // storage element type void* ptr; - gpu_ivf_flat_any_t(quantization_t q, void* p) : qtype(q), ptr(p) {} - ~gpu_ivf_flat_any_t() { + gpu_ivf_flat_any_t(quantization_t b, quantization_t q, void* p) + : btype(b), qtype(q), ptr(p) {} + ~gpu_ivf_flat_any_t(); +}; + +// Static dispatch: resolves the concrete gpu_ivf_flat_t for (btype,qtype) and +// invokes fn with a typed pointer. fn is a generic lambda; recover B/Q inside it +// via decltype(idx)::base_type / ::storage_type. Throws on unsupported combos. +template +static auto ivf_flat_dispatch(const gpu_ivf_flat_any_t* a, Fn&& fn) { + switch (a->btype) { + case Quantization_F32: + switch (a->qtype) { + case Quantization_F32: return fn(static_cast*>(a->ptr)); + case Quantization_F16: return fn(static_cast*>(a->ptr)); + case Quantization_INT8: return fn(static_cast*>(a->ptr)); + case Quantization_UINT8: return fn(static_cast*>(a->ptr)); + default: break; + } + break; + case Quantization_F16: + switch (a->qtype) { + case Quantization_F16: return fn(static_cast*>(a->ptr)); + case Quantization_INT8: return fn(static_cast*>(a->ptr)); + case Quantization_UINT8: return fn(static_cast*>(a->ptr)); + default: break; + } + break; + default: break; + } + throw std::runtime_error("gpu_ivf_flat: unsupported (base,storage) type combination"); +} + +gpu_ivf_flat_any_t::~gpu_ivf_flat_any_t() { + if (!ptr) return; + try { + ivf_flat_dispatch(this, [](auto* idx) { + idx->destroy(); + delete idx; + }); + } catch (...) { + // unsupported combo never gets a live ptr — nothing to free + } +} + +// Construct a new gpu_ivf_flat_t for the wired (btype,qtype) combos. +// Maker is a generic lambda invoked as maker(static type tag) -> void*; it +// receives a null typed pointer purely to recover B and Q. +template +static void* ivf_flat_construct(quantization_t btype, quantization_t qtype, Maker&& maker) { + switch (btype) { + case Quantization_F32: switch (qtype) { - case Quantization_F32: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - case Quantization_F16: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - case Quantization_INT8: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - case Quantization_UINT8: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - default: break; + case Quantization_F32: return maker(static_cast*>(nullptr)); + case Quantization_F16: return maker(static_cast*>(nullptr)); + case Quantization_INT8: return maker(static_cast*>(nullptr)); + case Quantization_UINT8: return maker(static_cast*>(nullptr)); + default: break; + } + break; + case Quantization_F16: + switch (qtype) { + case Quantization_F16: return maker(static_cast*>(nullptr)); + case Quantization_INT8: return maker(static_cast*>(nullptr)); + case Quantization_UINT8: return maker(static_cast*>(nullptr)); + default: break; } + break; + default: break; } -}; + throw std::runtime_error("gpu_ivf_flat: unsupported (base,storage) type combination"); +} extern "C" { -gpu_ivf_flat_c gpu_ivf_flat_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, +gpu_ivf_flat_c gpu_ivf_flat_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric_c, ivf_flat_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_ivf_flat_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_F16: - ptr = new gpu_ivf_flat_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_INT8: - ptr = new gpu_ivf_flat_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_UINT8: - ptr = new gpu_ivf_flat_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - default: return nullptr; - } - return static_cast(new gpu_ivf_flat_any_t(qtype, ptr)); + std::unique_ptr holder(new gpu_ivf_flat_any_t(btype, qtype, nullptr)); + holder->ptr = ivf_flat_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + // The dataset-providing constructor takes storage-typed (Q) data. + return new gpu_ivf_flat_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); + }); + return static_cast(holder.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_new", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_new", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_new", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_new", "unknown C++ exception"); } return nullptr; } -gpu_ivf_flat_c gpu_ivf_flat_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric_c, +gpu_ivf_flat_c gpu_ivf_flat_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric_c, ivf_flat_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_ivf_flat_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_F16: - ptr = new gpu_ivf_flat_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_INT8: - ptr = new gpu_ivf_flat_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_UINT8: - ptr = new gpu_ivf_flat_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - default: return nullptr; - } return static_cast(new gpu_ivf_flat_any_t(qtype, ptr)); - } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_new_empty", e.what()); - } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_new_empty", "unknown C++ exception"); + std::unique_ptr holder(new gpu_ivf_flat_any_t(btype, qtype, nullptr)); + holder->ptr = ivf_flat_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + return new gpu_ivf_flat_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); + }); + return static_cast(holder.release()); + } catch (const std::exception& e) { + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_new_empty", e.what()); + } catch (...) { + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_new_empty", "unknown C++ exception"); } return nullptr; } gpu_ivf_flat_c gpu_ivf_flat_load_file(const char* filename, uint32_t dimension, distance_type_t metric_c, ivf_flat_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, void* errmsg) { + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_ivf_flat_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_F16: - ptr = new gpu_ivf_flat_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_INT8: - ptr = new gpu_ivf_flat_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_UINT8: - ptr = new gpu_ivf_flat_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - default: return nullptr; - } - return static_cast(new gpu_ivf_flat_any_t(qtype, ptr)); + std::unique_ptr holder(new gpu_ivf_flat_any_t(btype, qtype, nullptr)); + holder->ptr = ivf_flat_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + return new gpu_ivf_flat_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); + }); + return static_cast(holder.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_load_file", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_load_file", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_load_file", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_load_file", "unknown C++ exception"); } return nullptr; } @@ -176,51 +193,31 @@ void gpu_ivf_flat_destroy(gpu_ivf_flat_c index_c, void* errmsg) { try { delete static_cast(index_c); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_destroy", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_destroy", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_destroy", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_destroy", "unknown C++ exception"); } } void gpu_ivf_flat_start(gpu_ivf_flat_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->start(); break; - case Quantization_F16: static_cast*>(any->ptr)->start(); break; - case Quantization_INT8: static_cast*>(any->ptr)->start(); break; - case Quantization_UINT8: static_cast*>(any->ptr)->start(); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [](auto* idx) { idx->start(); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_start", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_start", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_start", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_start", "unknown C++ exception"); } } void gpu_ivf_flat_build(gpu_ivf_flat_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->build(); break; - case Quantization_F16: static_cast*>(any->ptr)->build(); break; - case Quantization_INT8: static_cast*>(any->ptr)->build(); break; - case Quantization_UINT8: static_cast*>(any->ptr)->build(); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [](auto* idx) { idx->build(); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_build", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_build", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_build", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_build", "unknown C++ exception"); } } @@ -228,14 +225,10 @@ void gpu_ivf_flat_extend(gpu_ivf_flat_c index_c, const void* new_data, uint64_t const int64_t* new_ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->extend(static_cast(new_data), n_rows, new_ids); break; - case Quantization_F16: static_cast*>(any->ptr)->extend(static_cast(new_data), n_rows, new_ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->extend(static_cast(new_data), n_rows, new_ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->extend(static_cast(new_data), n_rows, new_ids); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + idx->extend(static_cast(new_data), n_rows, new_ids); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_extend", e.what()); } catch (...) { @@ -247,14 +240,9 @@ void gpu_ivf_flat_extend_float(gpu_ivf_flat_c index_c, const float* new_data, ui const int64_t* new_ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->extend_float(new_data, n_rows, new_ids); break; - case Quantization_F16: static_cast*>(any->ptr)->extend_float(new_data, n_rows, new_ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->extend_float(new_data, n_rows, new_ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->extend_float(new_data, n_rows, new_ids); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + idx->extend_float(new_data, n_rows, new_ids); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_extend_float", e.what()); } catch (...) { @@ -265,174 +253,127 @@ void gpu_ivf_flat_extend_float(gpu_ivf_flat_c index_c, const float* new_data, ui void gpu_ivf_flat_add_chunk(gpu_ivf_flat_c index_c, const void* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_F16: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + idx->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_add_chunk", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_add_chunk", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_add_chunk", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_add_chunk", "unknown C++ exception"); } } void gpu_ivf_flat_add_chunk_float(gpu_ivf_flat_c index_c, const float* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_F16: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + idx->add_chunk_float(chunk_data, chunk_count, -1, ids); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_add_chunk_float", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_add_chunk_float", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_add_chunk_float", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_add_chunk_float", "unknown C++ exception"); + } +} + +void gpu_ivf_flat_add_chunk_quantize(gpu_ivf_flat_c index_c, const void* base_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { + if (errmsg) *(static_cast(errmsg)) = nullptr; + try { + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + idx->add_chunk_quantize(static_cast(base_data), chunk_count, -1, ids); + }); + } catch (const std::exception& e) { + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_add_chunk_quantize", e.what()); + } catch (...) { + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_add_chunk_quantize", "unknown C++ exception"); } } void gpu_ivf_flat_train_quantizer(gpu_ivf_flat_c index_c, const float* train_data, uint64_t n_samples, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - case Quantization_F16: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - case Quantization_INT8: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - case Quantization_UINT8: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + // train_quantizer takes base-typed (B) data. The C ABI hands us + // float32 (every existing caller has an F32 base); for an F16 base + // convert the host buffer to half first so all instantiations are + // both compilable and correct. + if constexpr (std::is_same_v) { + idx->train_quantizer(train_data, n_samples); + } else { + std::vector conv(static_cast(n_samples) * idx->dimension); + matrixone::cast_float_to_half_host(train_data, conv.data(), conv.size()); + idx->train_quantizer(conv.data(), n_samples); + } + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_train_quantizer", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_train_quantizer", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_train_quantizer", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_train_quantizer", "unknown C++ exception"); } } void gpu_ivf_flat_set_batch_window(gpu_ivf_flat_c index_c, int64_t window_us, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_batch_window(window_us); break; - case Quantization_F16: static_cast*>(any->ptr)->set_batch_window(window_us); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_batch_window(window_us); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_batch_window(window_us); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { idx->set_batch_window(window_us); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_set_batch_window", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_set_batch_window", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_set_batch_window", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_set_batch_window", "unknown C++ exception"); } } void gpu_ivf_flat_set_dynb_conservative_dispatch(gpu_ivf_flat_c index_c, bool enable, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - case Quantization_F16: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { idx->set_dynb_conservative_dispatch(enable); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_set_dynb_conservative_dispatch", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_set_dynb_conservative_dispatch", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_set_dynb_conservative_dispatch", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_set_dynb_conservative_dispatch", "unknown C++ exception"); } } void gpu_ivf_flat_set_quantizer(gpu_ivf_flat_c index_c, float min, float max, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_quantizer(min, max); break; - case Quantization_F16: static_cast*>(any->ptr)->set_quantizer(min, max); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_quantizer(min, max); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_quantizer(min, max); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { idx->set_quantizer(min, max); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_set_quantizer", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_set_quantizer", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_set_quantizer", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_set_quantizer", "unknown C++ exception"); } } void gpu_ivf_flat_get_quantizer(gpu_ivf_flat_c index_c, float* min, float* max, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->get_quantizer(min, max); break; - case Quantization_F16: static_cast*>(any->ptr)->get_quantizer(min, max); break; - case Quantization_INT8: static_cast*>(any->ptr)->get_quantizer(min, max); break; - case Quantization_UINT8: static_cast*>(any->ptr)->get_quantizer(min, max); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { idx->get_quantizer(min, max); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_get_quantizer", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_get_quantizer", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_get_quantizer", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_get_quantizer", "unknown C++ exception"); } } void gpu_ivf_flat_save(gpu_ivf_flat_c index_c, const char* filename, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->save(filename); break; - case Quantization_F16: static_cast*>(any->ptr)->save(filename); break; - case Quantization_INT8: static_cast*>(any->ptr)->save(filename); break; - case Quantization_UINT8: static_cast*>(any->ptr)->save(filename); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { idx->save(filename); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_save", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_save", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_save", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_save", "unknown C++ exception"); } } void gpu_ivf_flat_save_dir(gpu_ivf_flat_c index_c, const char* dir, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->save_dir(dir); break; - case Quantization_F16: static_cast*>(any->ptr)->save_dir(dir); break; - case Quantization_INT8: static_cast*>(any->ptr)->save_dir(dir); break; - case Quantization_UINT8: static_cast*>(any->ptr)->save_dir(dir); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { idx->save_dir(dir); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_save_dir", e.what()); } catch (...) { @@ -443,14 +384,7 @@ void gpu_ivf_flat_save_dir(gpu_ivf_flat_c index_c, const char* dir, void* errmsg void gpu_ivf_flat_delete_id(gpu_ivf_flat_c index_c, int64_t id, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->delete_id(id); break; - case Quantization_F16: static_cast*>(any->ptr)->delete_id(id); break; - case Quantization_INT8: static_cast*>(any->ptr)->delete_id(id); break; - case Quantization_UINT8: static_cast*>(any->ptr)->delete_id(id); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { idx->delete_id(id); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_delete_id", e.what()); } catch (...) { @@ -462,14 +396,7 @@ void gpu_ivf_flat_load_dir(gpu_ivf_flat_c index_c, const char* dir, distribution_mode_t target_mode, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - case Quantization_F16: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - case Quantization_INT8: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - case Quantization_UINT8: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { idx->load_dir(dir, target_mode); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_load_dir", e.what()); } catch (...) { @@ -477,69 +404,55 @@ void gpu_ivf_flat_load_dir(gpu_ivf_flat_c index_c, const char* dir, } } -gpu_ivf_flat_search_res_t gpu_ivf_flat_search(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +gpu_ivf_flat_search_res_t gpu_ivf_flat_search(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_flat_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + *cpp_res = idx->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_search", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_search", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search", "unknown C++ exception"); } return result; } -gpu_ivf_flat_search_res_t gpu_ivf_flat_search_float(gpu_ivf_flat_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +gpu_ivf_flat_search_res_t gpu_ivf_flat_search_quantize(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_flat_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + *cpp_res = idx->search_quantize(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_float", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_quantize", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_float", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_quantize", "unknown C++ exception"); } return result; } -uint64_t gpu_ivf_flat_search_async(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_ivf_flat_search_async(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_F16: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_INT8: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_UINT8: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - default: return 0; - } + return ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using Q = typename std::remove_pointer_t::storage_type; + return idx->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_async", e.what()); return 0; @@ -549,24 +462,20 @@ uint64_t gpu_ivf_flat_search_async(gpu_ivf_flat_c index_c, const void* queries_d } } -uint64_t gpu_ivf_flat_search_float_async(gpu_ivf_flat_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_ivf_flat_search_quantize_async(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_F16: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_INT8: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_UINT8: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - default: return 0; - } + return ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using B = typename std::remove_pointer_t::base_type; + return idx->search_quantize_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_float_async", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_quantize_async", e.what()); return 0; } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_float_async", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_quantize_async", "unknown C++ exception"); return 0; } } @@ -575,15 +484,10 @@ gpu_ivf_flat_search_res_t gpu_ivf_flat_search_wait(gpu_ivf_flat_c index_c, uint6 if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_flat_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + *cpp_res = idx->search_wait(job_id); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_wait", e.what()); @@ -642,14 +546,7 @@ void gpu_ivf_flat_free_result(gpu_ivf_flat_result_c result_c) { uint64_t gpu_ivf_flat_cap(gpu_ivf_flat_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->cap(); - case Quantization_F16: return static_cast*>(any->ptr)->cap(); - case Quantization_INT8: return static_cast*>(any->ptr)->cap(); - case Quantization_UINT8: return static_cast*>(any->ptr)->cap(); - default: return 0; - } + return ivf_flat_dispatch(static_cast(index_c), [](auto* idx) -> uint64_t { return idx->cap(); }); } catch (...) { return 0; } @@ -658,14 +555,7 @@ uint64_t gpu_ivf_flat_cap(gpu_ivf_flat_c index_c) { uint64_t gpu_ivf_flat_len(gpu_ivf_flat_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->len(); - case Quantization_F16: return static_cast*>(any->ptr)->len(); - case Quantization_INT8: return static_cast*>(any->ptr)->len(); - case Quantization_UINT8: return static_cast*>(any->ptr)->len(); - default: return 0; - } + return ivf_flat_dispatch(static_cast(index_c), [](auto* idx) -> uint64_t { return idx->len(); }); } catch (...) { return 0; } @@ -675,23 +565,13 @@ char* gpu_ivf_flat_info(gpu_ivf_flat_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; if (!index_c) return nullptr; try { - auto* any = static_cast(index_c); - std::string info; - switch (any->qtype) { - case Quantization_F32: info = static_cast*>(any->ptr)->info(); break; - case Quantization_F16: info = static_cast*>(any->ptr)->info(); break; - case Quantization_INT8: info = static_cast*>(any->ptr)->info(); break; - case Quantization_UINT8: info = static_cast*>(any->ptr)->info(); break; - default: return nullptr; - } + std::string info = ivf_flat_dispatch(static_cast(index_c), [](auto* idx) -> std::string { return idx->info(); }); return strdup(info.c_str()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_info", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_info", e.what()); return nullptr; } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_info", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_info", "unknown C++ exception"); return nullptr; } } @@ -699,50 +579,22 @@ char* gpu_ivf_flat_info(gpu_ivf_flat_c index_c, void* errmsg) { void gpu_ivf_flat_get_centers(gpu_ivf_flat_c index_c, void* centers, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: { - auto host_centers = static_cast*>(any->ptr)->get_centers(); - if (!host_centers.empty()) std::copy(host_centers.begin(), host_centers.end(), static_cast(centers)); - break; - } - case Quantization_F16: { - auto host_centers = static_cast*>(any->ptr)->get_centers(); - if (!host_centers.empty()) std::copy(host_centers.begin(), host_centers.end(), static_cast(centers)); - break; - } - case Quantization_INT8: { - auto host_centers = static_cast*>(any->ptr)->get_centers(); - if (!host_centers.empty()) std::copy(host_centers.begin(), host_centers.end(), static_cast(centers)); - break; - } - case Quantization_UINT8: { - auto host_centers = static_cast*>(any->ptr)->get_centers(); - if (!host_centers.empty()) std::copy(host_centers.begin(), host_centers.end(), static_cast(centers)); - break; - } - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + auto host_centers = idx->get_centers(); + if (!host_centers.empty()) std::copy(host_centers.begin(), host_centers.end(), static_cast(centers)); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_get_centers", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_get_centers", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_get_centers", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_get_centers", "unknown C++ exception"); } } uint32_t gpu_ivf_flat_get_n_list(gpu_ivf_flat_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->get_n_list(); - case Quantization_F16: return static_cast*>(any->ptr)->get_n_list(); - case Quantization_INT8: return static_cast*>(any->ptr)->get_n_list(); - case Quantization_UINT8: return static_cast*>(any->ptr)->get_n_list(); - default: return 0; - } + return ivf_flat_dispatch(static_cast(index_c), [](auto* idx) -> uint32_t { return idx->get_n_list(); }); } catch (...) { return 0; } @@ -754,15 +606,10 @@ void gpu_ivf_flat_set_filter_columns(gpu_ivf_flat_c index_c, const char* col_met uint64_t total_count, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); std::string s = col_meta_json ? col_meta_json : ""; - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - case Quantization_F16: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + idx->set_filter_columns(s, total_count); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_set_filter_columns", e.what()); } catch (...) { @@ -775,14 +622,9 @@ void gpu_ivf_flat_add_filter_chunk(gpu_ivf_flat_c index_c, uint32_t col_idx, uint64_t nrows, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_F16: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_INT8: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_UINT8: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + idx->add_filter_chunk(col_idx, data, null_bitmap, nrows); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_add_filter_chunk", e.what()); } catch (...) { @@ -797,16 +639,12 @@ gpu_ivf_flat_search_res_t gpu_ivf_flat_search_with_filter(gpu_ivf_flat_c index_c if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_flat_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + *cpp_res = idx->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_with_filter", e.what()); @@ -816,52 +654,44 @@ gpu_ivf_flat_search_res_t gpu_ivf_flat_search_with_filter(gpu_ivf_flat_c index_c return result; } -gpu_ivf_flat_search_res_t gpu_ivf_flat_search_float_with_filter(gpu_ivf_flat_c index_c, const float* queries_data, +gpu_ivf_flat_search_res_t gpu_ivf_flat_search_quantize_with_filter(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t sp, const char* preds_json, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_flat_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + *cpp_res = idx->search_quantize_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_float_with_filter", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_quantize_with_filter", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_float_with_filter", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_quantize_with_filter", "unknown C++ exception"); } return result; } -uint64_t gpu_ivf_flat_search_float_with_filter_async(gpu_ivf_flat_c index_c, const float* queries_data, +uint64_t gpu_ivf_flat_search_quantize_with_filter_async(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t sp, const char* preds_json, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_F16: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_INT8: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_UINT8: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - default: return 0; - } + return ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using B = typename std::remove_pointer_t::base_type; + return idx->search_quantize_with_filter_async(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_float_with_filter_async", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_quantize_with_filter_async", e.what()); return 0; } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_float_with_filter_async", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_quantize_with_filter_async", "unknown C++ exception"); return 0; } } @@ -869,8 +699,11 @@ uint64_t gpu_ivf_flat_search_float_with_filter_async(gpu_ivf_flat_c index_c, con } // extern "C" namespace matrixone { -template class gpu_ivf_flat_t; -template class gpu_ivf_flat_t; -template class gpu_ivf_flat_t; -template class gpu_ivf_flat_t; +template class gpu_ivf_flat_t; +template class gpu_ivf_flat_t; +template class gpu_ivf_flat_t; +template class gpu_ivf_flat_t; +template class gpu_ivf_flat_t; +template class gpu_ivf_flat_t; +template class gpu_ivf_flat_t; } // namespace matrixone diff --git a/cgo/cuvs/ivf_flat_c.h b/cgo/cuvs/ivf_flat_c.h index 14c6693bdd33b..8822119641379 100644 --- a/cgo/cuvs/ivf_flat_c.h +++ b/cgo/cuvs/ivf_flat_c.h @@ -31,18 +31,22 @@ typedef void* gpu_ivf_flat_c; // Opaque pointer to the C++ IVF-Flat search result object typedef void* gpu_ivf_flat_result_c; +// btype = base/query/quantizer-source element type (Quantization_F32 or F16). +// qtype = storage element type. Wired combos: F32 base {F32,F16,INT8,UINT8}; +// F16 base {F16,INT8,UINT8}. Other combinations set errmsg and return NULL. + // Constructor for building from dataset -gpu_ivf_flat_c gpu_ivf_flat_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, +gpu_ivf_flat_c gpu_ivf_flat_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric, ivf_flat_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg); // Constructor for loading from file gpu_ivf_flat_c gpu_ivf_flat_load_file(const char* filename, uint32_t dimension, distance_type_t metric, ivf_flat_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, void* errmsg); + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, void* errmsg); // Destructor void gpu_ivf_flat_destroy(gpu_ivf_flat_c index_c, void* errmsg); @@ -54,10 +58,10 @@ void gpu_ivf_flat_start(gpu_ivf_flat_c index_c, void* errmsg); void gpu_ivf_flat_build(gpu_ivf_flat_c index_c, void* errmsg); // Constructor for an empty index (pre-allocates) -gpu_ivf_flat_c gpu_ivf_flat_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric, +gpu_ivf_flat_c gpu_ivf_flat_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric, ivf_flat_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg); // Add chunk of data (same type as index quantization) void gpu_ivf_flat_add_chunk(gpu_ivf_flat_c index_c, const void* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); @@ -74,6 +78,9 @@ void gpu_ivf_flat_extend_float(gpu_ivf_flat_c index_c, const float* new_data, ui // Add chunk of data (from float, with on-the-fly quantization if needed) void gpu_ivf_flat_add_chunk_float(gpu_ivf_flat_c index_c, const float* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); +// Add chunk of base-typed (B) data; the index converts B -> storage on device. +void gpu_ivf_flat_add_chunk_quantize(gpu_ivf_flat_c index_c, const void* base_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); + // Trains the scalar quantizer (if T is 1-byte) void gpu_ivf_flat_train_quantizer(gpu_ivf_flat_c index_c, const float* train_data, uint64_t n_samples, void* errmsg); @@ -107,17 +114,19 @@ gpu_ivf_flat_search_res_t gpu_ivf_flat_search(gpu_ivf_flat_c index_c, const void uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, void* errmsg); -gpu_ivf_flat_search_res_t gpu_ivf_flat_search_float(gpu_ivf_flat_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +// Quantize search: query in the BASE element type B (float or half); the index +// converts it to storage type T (copy / quantize / f32->f16 cast) internally. +gpu_ivf_flat_search_res_t gpu_ivf_flat_search_quantize(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, void* errmsg); // Asynchronous search functions -uint64_t gpu_ivf_flat_search_async(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_ivf_flat_search_async(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, void* errmsg); -uint64_t gpu_ivf_flat_search_float_async(gpu_ivf_flat_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_ivf_flat_search_quantize_async(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, void* errmsg); gpu_ivf_flat_search_res_t gpu_ivf_flat_search_wait(gpu_ivf_flat_c index_c, uint64_t job_id, void* errmsg); @@ -160,15 +169,16 @@ gpu_ivf_flat_search_res_t gpu_ivf_flat_search_with_filter(gpu_ivf_flat_c index_c uint32_t limit, ivf_flat_search_params_t search_params, const char* preds_json, void* errmsg); -gpu_ivf_flat_search_res_t gpu_ivf_flat_search_float_with_filter(gpu_ivf_flat_c index_c, const float* queries_data, +// Query in the BASE element type B (float or half); converted to storage T internally. +gpu_ivf_flat_search_res_t gpu_ivf_flat_search_quantize_with_filter(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, const char* preds_json, void* errmsg); -// Async variant of gpu_ivf_flat_search_float_with_filter. Returns a job_id +// Async variant of gpu_ivf_flat_search_quantize_with_filter. Returns a job_id // that is collected with the existing gpu_ivf_flat_search_wait. Lets // multi-index callers fan out filtered searches across shards in parallel. -uint64_t gpu_ivf_flat_search_float_with_filter_async(gpu_ivf_flat_c index_c, const float* queries_data, +uint64_t gpu_ivf_flat_search_quantize_with_filter_async(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, const char* preds_json, void* errmsg); diff --git a/cgo/cuvs/ivf_pq.hpp b/cgo/cuvs/ivf_pq.hpp index 2cfba5311ac99..cee7908a4251e 100644 --- a/cgo/cuvs/ivf_pq.hpp +++ b/cgo/cuvs/ivf_pq.hpp @@ -71,8 +71,9 @@ namespace matrixone { // // OVERVIEW // -------- -// gpu_ivf_pq_t implements an IVF-PQ (Inverted File with Product Quantization) -// approximate nearest-neighbor index backed by cuVS. +// gpu_ivf_pq_t implements an IVF-PQ (Inverted File with Product Quantization) +// approximate nearest-neighbor index backed by cuVS (B = base/source element +// type, T = storage element type; T is 1-byte for scalar-quantized indexes). // // cuVS type: cuvs::neighbors::ivf_pq::index // Note: the cuVS IVF-PQ index type is NOT templated on T — it always stores @@ -114,7 +115,7 @@ namespace matrixone { // replicated_indices_[rank] holds a full copy per rank (cast to ivf_pq_index*). // The replicated dataset pointers (replicated_datasets_) are used during build // and erased after the first extend on each device. -// search_internal / search_float_internal use per-thread cached index ptr +// search_internal / search_quantize_internal use per-thread cached index ptr // (handle.get_index_ptr()) to avoid repeated map lookups. // // SHARDED: @@ -143,7 +144,7 @@ namespace matrixone { // For REPLICATED: uses per-thread cached index ptr to avoid mutex on hot path. // For SHARDED: called once per shard with the shard's local index. // -// search_float_internal(handle, float* queries, ...) +// search_quantize_internal(handle, B* queries, ...) // Converts float → T on device (quantize for 1-byte T, half-cast for T=half, // direct copy for T=float), then searches the same way as search_internal. // @@ -152,7 +153,7 @@ namespace matrixone { // - SHARDED: sync_shard_bitset() → bitset_filter over shard-local bit slice // Bit j of the shard bitset = global bit (rank * rows_per_shard + j) // -// search_batchable_typed() / search_batchable_float() just submit the search to +// search_batchable_typed() / search_batchable_quantize() just submit the search to // the worker; request-level batching, when enabled (batch_window() > 0), // happens inside search_internal via cuVS dynamic_batching (see dynamic_batching.hpp). // @@ -177,14 +178,16 @@ struct ivf_pq_search_result_t { /** * @brief gpu_ivf_pq_t implements an IVF-PQ index that can run on a single GPU or sharded/replicated across multiple GPUs. */ -template -class gpu_ivf_pq_t : public gpu_index_base_t { +template +class gpu_ivf_pq_t : public gpu_index_base_t { public: + using base_type = B; + using storage_type = T; using ivf_pq_index = cuvs::neighbors::ivf_pq::index; using search_result_t = ivf_pq_search_result_t; // Inherited dependent type — bring into scope so search_internal can take a // const host_mask_bundle_t* parameter without `typename Base::...` everywhere. - using host_mask_bundle_t = typename gpu_index_base_t::host_mask_bundle_t; + using host_mask_bundle_t = typename gpu_index_base_t::host_mask_bundle_t; // Internal index storage std::unique_ptr index_; @@ -368,6 +371,9 @@ class gpu_ivf_pq_t : public gpu_index_base_t return; } } + // 1-byte storage T: train the B-source quantizer on the buffered B + // sample, transform B->T, and store as T. For float/half storage this + // is a no-op. this->train_quantizer_if_needed(); if (!this->worker) throw std::runtime_error("Worker not initialized"); @@ -805,8 +811,34 @@ class gpu_ivf_pq_t : public gpu_index_base_t return this->search_wait(job_id); } - // Async T-typed filtered search. Mirrors search_float_with_filter_async - // but uses search_internal (T) instead of search_float_internal (float). + // Quantize a B-source query to the 1-byte storage type T via the B-source + // quantizer, writing num_queries*dimension T values into `out`. The caller + // then runs the normal native search(const T*) path — so sharding, overflow + // and result merge are reused unchanged. No f32 detour. + void quantize_query(const B* queries_data, uint64_t num_queries, T* out) { + if constexpr (sizeof(T) != 1) { + throw std::runtime_error("quantize_query requires a 1-byte storage type (int8/uint8)"); + } else { + uint64_t job = this->worker->submit_main( + [this, queries_data, num_queries, out](raft_handle_wrapper_t& handle) -> std::any { + auto res = handle.get_raft_resources(); + auto q_b_host = raft::make_host_matrix_view(queries_data, num_queries, this->dimension); + auto q_b_dev = raft::make_device_matrix(*res, num_queries, this->dimension); + raft::copy(*res, q_b_dev.view(), q_b_host); + if (!this->quantizer_.is_trained()) throw std::runtime_error("quantizer not trained"); + auto q_t_dev = raft::make_device_matrix(*res, num_queries, this->dimension); + this->quantizer_.template transform(*res, q_b_dev.view(), q_t_dev.data_handle(), true); + raft::copy(*res, raft::make_host_matrix_view(out, num_queries, this->dimension), q_t_dev.view()); + handle.sync(); + return std::any(); + }); + auto r = this->worker->wait(job).get(); + if (r.error) std::rethrow_exception(r.error); + } + } + + // Async T-typed filtered search. Mirrors search_quantize_with_filter_async + // but uses search_internal (T) instead of search_quantize_internal (B). uint64_t search_with_filter_async(const T* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_pq_search_params_t& sp, @@ -1221,44 +1253,46 @@ class gpu_ivf_pq_t : public gpu_index_base_t } } - transform_distance(this->metric, search_res.distances); + transform_distance(this->metric, search_res.distances, this->quantized_l2_dequant_factor()); return search_res; } - // Sync float entry — wraps search_float_async + search_wait. - search_result_t search_float(const float* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_pq_search_params_t& sp) { - uint64_t job_id = this->search_float_async(queries_data, num_queries, query_dimension, limit, sp); + // Sync quantize entry — wraps search_quantize_async + search_wait. + search_result_t search_quantize(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_pq_search_params_t& sp) { + uint64_t job_id = this->search_quantize_async(queries_data, num_queries, query_dimension, limit, sp); return this->search_wait(job_id); } - // Sync float filtered entry — wraps search_float_with_filter_async + search_wait. - search_result_t search_float_with_filter(const float* queries_data, uint64_t num_queries, + // Sync quantize filtered entry — wraps search_quantize_with_filter_async + search_wait. + search_result_t search_quantize_with_filter(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_pq_search_params_t& sp, const std::string& preds_json) { - uint64_t job_id = this->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds_json); + uint64_t job_id = this->search_quantize_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds_json); return this->search_wait(job_id); } - // Async variant of search_float_with_filter. Builds the host mask bundle on + // Async variant of search_quantize_with_filter. Builds the host mask bundle on // the calling thread (same off-worker pattern as the sync filter), copies // queries into a shared_ptr so they outlive the Go caller, captures both in // the worker lambda, and returns a job_id that search_wait() can collect. // Used by the multi-index filter path so per-shard searches run in parallel. - uint64_t search_float_with_filter_async(const float* queries_data, uint64_t num_queries, + // The query is the BASE type B (float or half); search_quantize_internal + // converts it to storage T. + uint64_t search_quantize_with_filter_async(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_pq_search_params_t& sp, const std::string& preds_json) { if (!queries_data) throw std::invalid_argument("search_async: queries_data is null"); if (num_queries == 0) throw std::invalid_argument("search_async: num_queries is 0"); if (this->dimension == 0) throw std::runtime_error("search_async: index dimension is 0"); - // Reject mismatched caller dim. search_float_internal sizes its H2D + // Reject mismatched caller dim. search_quantize_internal sizes its H2D // extent by this->dimension (query_dimension param is unused inside), // so passing a different value here would either OOB-read or // under-copy host queries. See the T-typed sibling at line ~762. if (query_dimension != this->dimension) { throw std::invalid_argument( - "search_float_with_filter_async: query_dimension (" + std::to_string(query_dimension) + + "search_quantize_with_filter_async: query_dimension (" + std::to_string(query_dimension) + ") does not match index dimension (" + std::to_string(this->dimension) + ")"); } { @@ -1267,7 +1301,7 @@ class gpu_ivf_pq_t : public gpu_index_base_t } if (!this->worker) throw std::runtime_error("Worker not initialized"); - auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); + auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); if (this->dist_mode == DistributionMode_SHARDED) { // Bitmap eval runs on the caller's (Go) thread; per-shard searches @@ -1276,7 +1310,7 @@ class gpu_ivf_pq_t : public gpu_index_base_t auto shard_masks = this->build_filter_shard_masks(preds_json); auto shard_search_task = [this, num_queries, query_dimension, limit, sp, queries_copy, shard_masks](raft_handle_wrapper_t& gpu_handle) -> std::any { int rank = gpu_handle.get_rank(); - return this->search_float_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", shard_masks[rank].get()); + return this->search_quantize_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", shard_masks[rank].get()); }; auto job_ids = this->worker->submit_all_devices_no_wait(shard_search_task); return this->worker->submit_composite_pending(std::move(job_ids), num_queries, limit); @@ -1288,19 +1322,19 @@ class gpu_ivf_pq_t : public gpu_index_base_t // would force serialization through main_thread_ and lose batching. auto mask = this->build_filter_single_mask(preds_json); auto task = [this, num_queries, query_dimension, limit, sp, queries_copy, mask](raft_handle_wrapper_t& handle) -> std::any { - return this->search_float_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", mask.get()); + return this->search_quantize_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", mask.get()); }; return this->worker->submit(task); } - uint64_t search_float_async(const float* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_pq_search_params_t& sp) { + uint64_t search_quantize_async(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_pq_search_params_t& sp) { if (!queries_data) throw std::invalid_argument("search_async: queries_data is null"); if (num_queries == 0) throw std::invalid_argument("search_async: num_queries is 0"); if (this->dimension == 0) throw std::runtime_error("search_async: index dimension is 0"); - // Reject mismatched caller dim — see search_float_with_filter_async. + // Reject mismatched caller dim — see search_quantize_with_filter_async. if (query_dimension != this->dimension) { throw std::invalid_argument( - "search_float_async: query_dimension (" + std::to_string(query_dimension) + + "search_quantize_async: query_dimension (" + std::to_string(query_dimension) + ") does not match index dimension (" + std::to_string(this->dimension) + ")"); } { @@ -1308,13 +1342,13 @@ class gpu_ivf_pq_t : public gpu_index_base_t if (!this->is_loaded_ || (!index_ && this->replicated_indices_.empty())) throw std::runtime_error("search_async: index not loaded"); } - auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); + auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); if (this->dist_mode == DistributionMode_SHARDED) { // Same shape as search_async — fan out, hand back a composite id, // let search_wait() do the merge on the caller's thread. auto shard_search_task = [this, num_queries, query_dimension, limit, sp, queries_copy](raft_handle_wrapper_t& gpu_handle) -> std::any { - return this->search_float_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp); + return this->search_quantize_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp); }; auto job_ids = this->worker->submit_all_devices_no_wait(shard_search_task); return this->worker->submit_composite_pending(std::move(job_ids), num_queries, limit); @@ -1322,22 +1356,26 @@ class gpu_ivf_pq_t : public gpu_index_base_t // Single-GPU / replicated: the helper decides standalone vs fused; the // shared_ptr keeps the copied queries alive until the search runs. - return this->search_batchable_float(queries_copy, queries_copy->data(), num_queries, limit, sp); + return this->search_batchable_quantize(queries_copy, queries_copy->data(), num_queries, limit, sp); } - // float32-input search. Mirrors search_batchable_typed but calls - // search_float_internal; request-level batching (if enabled) happens inside it. - uint64_t search_batchable_float(std::shared_ptr> owner, const float* queries_data, + // Base-typed (B) quantize search. Mirrors search_batchable_typed but calls + // search_quantize_internal; request-level batching (if enabled) happens inside it. + uint64_t search_batchable_quantize(std::shared_ptr> owner, const B* queries_data, uint64_t num_queries, uint32_t limit, const ivf_pq_search_params_t& sp) { if (!this->worker) throw std::runtime_error("Worker not initialized"); auto task = [this, owner, queries_data, num_queries, limit, sp](raft_handle_wrapper_t& handle) -> std::any { - return this->search_float_internal(handle, queries_data, num_queries, this->dimension, limit, sp); + return this->search_quantize_internal(handle, queries_data, num_queries, this->dimension, limit, sp); }; return this->worker->submit(task); } // See `search_internal` for the contract on `prebuilt`. - search_result_t search_float_internal(raft_handle_wrapper_t& handle, const float* queries_data, uint64_t num_queries, uint32_t /*query_dimension*/, + // Takes the query in the BASE element type B (float or half) and converts + // it to the storage type T on-device — see the cagra search_quantize_internal + // comment. B==T copies straight, sizeof(T)==1 quantizes B -> int8/uint8, and + // the (B=float, T=half) instantiation casts f32 -> f16 on the host. + search_result_t search_quantize_internal(raft_handle_wrapper_t& handle, const B* queries_data, uint64_t num_queries, uint32_t /*query_dimension*/, uint32_t limit, const ivf_pq_search_params_t& sp, const std::string& preds_json = "", const host_mask_bundle_t* prebuilt = nullptr) { auto res = handle.get_raft_resources(); // Step C: reuse the per-thread T-typed query workspace buffer. @@ -1346,29 +1384,28 @@ class gpu_ivf_pq_t : public gpu_index_base_t auto q_dev_t = raft::make_device_matrix_view( q_buf_t.data(), static_cast(num_queries), static_cast(this->dimension)); - if constexpr (std::is_same_v) { - raft::copy(*res, q_dev_t, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); - } else if constexpr (std::is_same_v) { - // Cast fp32 → fp16 on the host (F16C / AVX, IEEE round-to-nearest-even - // — bit-identical to mdspan_copy_kernel<__half>) into a pinned - // staging buffer, then a single H2D copy moves half the bytes. - // This eliminates one device alloc (q_dev_f), one full H2D fp32 - // upload, and the per-search mdspan_copy_kernel<__half> dispatch. + if constexpr (std::is_same_v) { + // B == T (float->float or half->half): no conversion. + raft::copy(*res, q_dev_t, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + } else if constexpr (sizeof(T) == 1) { + // sizeof(T) == 1: quantize the base-typed query B -> int8/uint8. + // Stage the B query on its own per-thread device workspace (distinct + // from q_buf_t — see q_dev_buf), then transform B -> T on-device. + if (!this->quantizer_.is_trained()) throw std::runtime_error("Quantizer not trained"); + auto& q_buf_b = handle.template q_dev_buf(n_q_elems); + auto q_dev_b = raft::make_device_matrix_view( + q_buf_b.data(), static_cast(num_queries), static_cast(this->dimension)); + raft::copy(*res, q_dev_b, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + this->quantizer_.template transform(*res, q_dev_b, q_buf_t.data(), true); + } else { + // B != T and sizeof(T) != 1: only (B=float, T=half). Cast fp32 → fp16 + // on the host (F16C / AVX, IEEE round-to-nearest-even — bit-identical + // to mdspan_copy_kernel<__half>) into a pinned staging buffer, then a + // single H2D copy moves half the bytes. __half* host_h = handle.ensure_host_half_buf(n_q_elems); matrixone::cast_float_to_half_host(queries_data, host_h, n_q_elems); raft::copy(*res, q_dev_t, raft::make_host_matrix_view(host_h, num_queries, this->dimension)); - } else { - // sizeof(T) == 1: int8 quantizer path keeps an fp32 device copy - // because quantizer_.transform reads it on-device. Reuse the - // per-thread float workspace too. - auto& q_buf_f = handle.q_dev_buf_float(n_q_elems); - auto q_dev_f = raft::make_device_matrix_view( - q_buf_f.data(), static_cast(num_queries), static_cast(this->dimension)); - raft::copy(*res, q_dev_f, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); - - if (!this->quantizer_.is_trained()) throw std::runtime_error("Quantizer not trained"); - this->quantizer_.template transform(*res, q_dev_f, q_buf_t.data(), true); } // Legacy path syncs to drain queries DMA before the stack-local host // bitmap inside build_search_bitset goes through its own sync. Prebuilt @@ -1554,7 +1591,7 @@ class gpu_ivf_pq_t : public gpu_index_base_t } } - transform_distance(this->metric, search_res.distances); + transform_distance(this->metric, search_res.distances, this->quantized_l2_dequant_factor()); return search_res; } @@ -1591,7 +1628,14 @@ class gpu_ivf_pq_t : public gpu_index_base_t if constexpr (sizeof(T) == 1) { if (!this->quantizer_.is_trained()) throw std::runtime_error("Quantizer not trained"); auto centers_float_view = raft::make_device_matrix_view(centers_view.data_handle(), n_centers, dim_ext); - this->quantizer_.template transform(*res, centers_float_view, centers_device_target.data_handle(), true); + if constexpr (std::is_same_v) { + this->quantizer_.template transform(*res, centers_float_view, centers_device_target.data_handle(), true); + } else { + // B == half: cast cuVS's float centers to half, then transform. + auto centers_b = raft::make_device_matrix(*res, n_centers, dim_ext); + raft::copy(*res, centers_b.view(), centers_float_view); + this->quantizer_.template transform(*res, centers_b.view(), centers_device_target.data_handle(), true); + } } else { raft::copy(*res, centers_device_target.view(), centers_view); } @@ -1615,7 +1659,7 @@ class gpu_ivf_pq_t : public gpu_index_base_t } std::string info() const override { - std::string json = gpu_index_base_t::info(); + std::string json = gpu_index_base_t::info(); json += ", \"type\": \"IVF-PQ\", \"ivf_pq\": {"; if (index_) json += "\"mode\": \"Single-GPU\", \"size\": " + std::to_string(index_->size()); else if (!this->replicated_indices_.empty()) json += "\"mode\": \"Local-Indices\", \"ranks\": " + std::to_string(this->replicated_indices_.size()); diff --git a/cgo/cuvs/ivf_pq_c.cpp b/cgo/cuvs/ivf_pq_c.cpp index 642f2acecc87d..490c61339c42d 100644 --- a/cgo/cuvs/ivf_pq_c.cpp +++ b/cgo/cuvs/ivf_pq_c.cpp @@ -1,4 +1,4 @@ -/* +/* * Copyright 2021 Matrix Origin * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +16,15 @@ /* * IVF-PQ C Wrapper Implementation - * Supported data types (via quantization_t): Quantization_F32, Quantization_F16, Quantization_INT8, Quantization_UINT8 + * + * Two type axes via quantization_t: + * btype = base / query / quantizer-SOURCE element type (Quantization_F32 or F16) + * qtype = storage element type (Quantization_F32, F16, INT8, UINT8) + * + * Wired (btype, qtype) combinations: + * F32 base: F32, F16, INT8, UINT8 storage + * F16 base: F16, INT8, UINT8 storage + * Any other combination throws "unsupported (base,storage) type combination". */ #include "ivf_pq_c.h" @@ -27,181 +35,179 @@ #include #include #include +#include using namespace matrixone; struct gpu_ivf_pq_any_t { - quantization_t qtype; + quantization_t btype; // base / query / quantizer-source element type + quantization_t qtype; // storage element type void* ptr; - gpu_ivf_pq_any_t(quantization_t q, void* p) : qtype(q), ptr(p) {} - ~gpu_ivf_pq_any_t() { + gpu_ivf_pq_any_t(quantization_t b, quantization_t q, void* p) + : btype(b), qtype(q), ptr(p) {} + ~gpu_ivf_pq_any_t(); +}; + +// Static dispatch: resolves the concrete gpu_ivf_pq_t for (btype,qtype) and +// invokes fn with a typed pointer. fn is a generic lambda; recover B/Q inside it +// via decltype(idx)::base_type / ::storage_type. Throws on unsupported combos. +template +static auto ivf_pq_dispatch(const gpu_ivf_pq_any_t* a, Fn&& fn) { + switch (a->btype) { + case Quantization_F32: + switch (a->qtype) { + case Quantization_F32: return fn(static_cast*>(a->ptr)); + case Quantization_F16: return fn(static_cast*>(a->ptr)); + case Quantization_INT8: return fn(static_cast*>(a->ptr)); + case Quantization_UINT8: return fn(static_cast*>(a->ptr)); + default: break; + } + break; + case Quantization_F16: + switch (a->qtype) { + case Quantization_F16: return fn(static_cast*>(a->ptr)); + case Quantization_INT8: return fn(static_cast*>(a->ptr)); + case Quantization_UINT8: return fn(static_cast*>(a->ptr)); + default: break; + } + break; + default: break; + } + throw std::runtime_error("gpu_ivf_pq: unsupported (base,storage) type combination"); +} + +gpu_ivf_pq_any_t::~gpu_ivf_pq_any_t() { + if (!ptr) return; + try { + ivf_pq_dispatch(this, [](auto* idx) { + idx->destroy(); + delete idx; + }); + } catch (...) { + // unsupported combo never gets a live ptr — nothing to free + } +} + +// Construct a new gpu_ivf_pq_t for the wired (btype,qtype) combos. +// Maker is a generic lambda invoked as maker(static type tag) -> void*; it +// receives a null typed pointer purely to recover B and Q. +template +static void* ivf_pq_construct(quantization_t btype, quantization_t qtype, Maker&& maker) { + switch (btype) { + case Quantization_F32: switch (qtype) { - case Quantization_F32: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - case Quantization_F16: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - case Quantization_INT8: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - case Quantization_UINT8: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - default: break; + case Quantization_F32: return maker(static_cast*>(nullptr)); + case Quantization_F16: return maker(static_cast*>(nullptr)); + case Quantization_INT8: return maker(static_cast*>(nullptr)); + case Quantization_UINT8: return maker(static_cast*>(nullptr)); + default: break; + } + break; + case Quantization_F16: + switch (qtype) { + case Quantization_F16: return maker(static_cast*>(nullptr)); + case Quantization_INT8: return maker(static_cast*>(nullptr)); + case Quantization_UINT8: return maker(static_cast*>(nullptr)); + default: break; } + break; + default: break; } -}; + throw std::runtime_error("gpu_ivf_pq: unsupported (base,storage) type combination"); +} extern "C" { -gpu_ivf_pq_c gpu_ivf_pq_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, +gpu_ivf_pq_c gpu_ivf_pq_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric_c, ivf_pq_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_ivf_pq_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_F16: - ptr = new gpu_ivf_pq_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_INT8: - ptr = new gpu_ivf_pq_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_UINT8: - ptr = new gpu_ivf_pq_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - default: return nullptr; - } - return static_cast(new gpu_ivf_pq_any_t(qtype, ptr)); + std::unique_ptr holder(new gpu_ivf_pq_any_t(btype, qtype, nullptr)); + holder->ptr = ivf_pq_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + // The dataset-providing constructor takes storage-typed (Q) data and + // copies it directly into flattened_host_dataset (no quantization here; + // quantization happens via add_chunk_quantize / add_chunk_float). + return new gpu_ivf_pq_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); + }); + return static_cast(holder.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_new", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_new", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_new", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_new", "unknown C++ exception"); } return nullptr; } -gpu_ivf_pq_c gpu_ivf_pq_new_from_data_file(const char* data_filename, distance_type_t metric_c, +gpu_ivf_pq_c gpu_ivf_pq_new_from_data_file(const char* data_filename, distance_type_t metric_c, ivf_pq_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, void* errmsg) { + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_ivf_pq_t(std::string(data_filename), metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_F16: - ptr = new gpu_ivf_pq_t(std::string(data_filename), metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_INT8: - ptr = new gpu_ivf_pq_t(std::string(data_filename), metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_UINT8: - ptr = new gpu_ivf_pq_t(std::string(data_filename), metric_c, build_params, devs, nthread, dist_mode); - break; - default: return nullptr; - } - return static_cast(new gpu_ivf_pq_any_t(qtype, ptr)); + std::unique_ptr holder(new gpu_ivf_pq_any_t(btype, qtype, nullptr)); + holder->ptr = ivf_pq_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + return new gpu_ivf_pq_t(std::string(data_filename), metric_c, build_params, devs, nthread, dist_mode); + }); + return static_cast(holder.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_new_from_data_file", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_new_from_data_file", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_new_from_data_file", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_new_from_data_file", "unknown C++ exception"); } return nullptr; } -gpu_ivf_pq_c gpu_ivf_pq_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric_c, +gpu_ivf_pq_c gpu_ivf_pq_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric_c, ivf_pq_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_ivf_pq_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_F16: - ptr = new gpu_ivf_pq_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_INT8: - ptr = new gpu_ivf_pq_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_UINT8: - ptr = new gpu_ivf_pq_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - default: return nullptr; - } - return static_cast(new gpu_ivf_pq_any_t(qtype, ptr)); + std::unique_ptr holder(new gpu_ivf_pq_any_t(btype, qtype, nullptr)); + holder->ptr = ivf_pq_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + return new gpu_ivf_pq_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); + }); + return static_cast(holder.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_new_empty", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_new_empty", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_new_empty", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_new_empty", "unknown C++ exception"); } return nullptr; } gpu_ivf_pq_c gpu_ivf_pq_load_file(const char* filename, uint32_t dimension, distance_type_t metric_c, ivf_pq_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, void* errmsg) { + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_ivf_pq_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_F16: - ptr = new gpu_ivf_pq_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_INT8: - ptr = new gpu_ivf_pq_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_UINT8: - ptr = new gpu_ivf_pq_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - default: return nullptr; - } - return static_cast(new gpu_ivf_pq_any_t(qtype, ptr)); + std::unique_ptr holder(new gpu_ivf_pq_any_t(btype, qtype, nullptr)); + holder->ptr = ivf_pq_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + return new gpu_ivf_pq_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); + }); + return static_cast(holder.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_load_file", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_load_file", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_load_file", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_load_file", "unknown C++ exception"); } return nullptr; } @@ -211,51 +217,31 @@ void gpu_ivf_pq_destroy(gpu_ivf_pq_c index_c, void* errmsg) { try { delete static_cast(index_c); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_destroy", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_destroy", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_destroy", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_destroy", "unknown C++ exception"); } } void gpu_ivf_pq_start(gpu_ivf_pq_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->start(); break; - case Quantization_F16: static_cast*>(any->ptr)->start(); break; - case Quantization_INT8: static_cast*>(any->ptr)->start(); break; - case Quantization_UINT8: static_cast*>(any->ptr)->start(); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [](auto* idx) { idx->start(); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_start", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_start", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_start", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_start", "unknown C++ exception"); } } void gpu_ivf_pq_build(gpu_ivf_pq_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->build(); break; - case Quantization_F16: static_cast*>(any->ptr)->build(); break; - case Quantization_INT8: static_cast*>(any->ptr)->build(); break; - case Quantization_UINT8: static_cast*>(any->ptr)->build(); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [](auto* idx) { idx->build(); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_build", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_build", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_build", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_build", "unknown C++ exception"); } } @@ -263,14 +249,10 @@ void gpu_ivf_pq_extend(gpu_ivf_pq_c index_c, const void* new_data, uint64_t n_ro const int64_t* new_ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->extend(static_cast(new_data), n_rows, new_ids); break; - case Quantization_F16: static_cast*>(any->ptr)->extend(static_cast(new_data), n_rows, new_ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->extend(static_cast(new_data), n_rows, new_ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->extend(static_cast(new_data), n_rows, new_ids); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + idx->extend(static_cast(new_data), n_rows, new_ids); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_extend", e.what()); } catch (...) { @@ -282,14 +264,9 @@ void gpu_ivf_pq_extend_float(gpu_ivf_pq_c index_c, const float* new_data, uint64 const int64_t* new_ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->extend_float(new_data, n_rows, new_ids); break; - case Quantization_F16: static_cast*>(any->ptr)->extend_float(new_data, n_rows, new_ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->extend_float(new_data, n_rows, new_ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->extend_float(new_data, n_rows, new_ids); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + idx->extend_float(new_data, n_rows, new_ids); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_extend_float", e.what()); } catch (...) { @@ -300,174 +277,132 @@ void gpu_ivf_pq_extend_float(gpu_ivf_pq_c index_c, const float* new_data, uint64 void gpu_ivf_pq_add_chunk(gpu_ivf_pq_c index_c, const void* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_F16: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + idx->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_add_chunk", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_add_chunk", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_add_chunk", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_add_chunk", "unknown C++ exception"); } } void gpu_ivf_pq_add_chunk_float(gpu_ivf_pq_c index_c, const float* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_F16: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + idx->add_chunk_float(chunk_data, chunk_count, -1, ids); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_add_chunk_float", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_add_chunk_float", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_add_chunk_float", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_add_chunk_float", "unknown C++ exception"); } } -void gpu_ivf_pq_train_quantizer(gpu_ivf_pq_c index_c, const float* train_data, uint64_t n_samples, void* errmsg) { +void gpu_ivf_pq_add_chunk_quantize(gpu_ivf_pq_c index_c, const void* base_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - case Quantization_F16: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - case Quantization_INT8: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - case Quantization_UINT8: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + idx->add_chunk_quantize(static_cast(base_data), chunk_count, -1, ids); + }); + } catch (const std::exception& e) { + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_add_chunk_quantize", e.what()); + } catch (...) { + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_add_chunk_quantize", "unknown C++ exception"); + } +} + +void gpu_ivf_pq_quantize_query(gpu_ivf_pq_c index_c, const void* base_data, uint64_t num_queries, void* out, void* errmsg) { + if (errmsg) *(static_cast(errmsg)) = nullptr; + try { + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + idx->quantize_query(static_cast(base_data), num_queries, static_cast(out)); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_train_quantizer", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_quantize_query", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_train_quantizer", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_quantize_query", "unknown C++ exception"); + } +} + +void gpu_ivf_pq_train_quantizer(gpu_ivf_pq_c index_c, const void* train_data, uint64_t n_samples, void* errmsg) { + if (errmsg) *(static_cast(errmsg)) = nullptr; + try { + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + idx->train_quantizer(static_cast(train_data), n_samples); + }); + } catch (const std::exception& e) { + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_train_quantizer", e.what()); + } catch (...) { + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_train_quantizer", "unknown C++ exception"); } } void gpu_ivf_pq_set_batch_window(gpu_ivf_pq_c index_c, int64_t window_us, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_batch_window(window_us); break; - case Quantization_F16: static_cast*>(any->ptr)->set_batch_window(window_us); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_batch_window(window_us); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_batch_window(window_us); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { idx->set_batch_window(window_us); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_set_batch_window", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_set_batch_window", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_set_batch_window", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_set_batch_window", "unknown C++ exception"); } } void gpu_ivf_pq_set_dynb_conservative_dispatch(gpu_ivf_pq_c index_c, bool enable, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - case Quantization_F16: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { idx->set_dynb_conservative_dispatch(enable); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_set_dynb_conservative_dispatch", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_set_dynb_conservative_dispatch", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_set_dynb_conservative_dispatch", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_set_dynb_conservative_dispatch", "unknown C++ exception"); } } void gpu_ivf_pq_set_quantizer(gpu_ivf_pq_c index_c, float min, float max, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_quantizer(min, max); break; - case Quantization_F16: static_cast*>(any->ptr)->set_quantizer(min, max); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_quantizer(min, max); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_quantizer(min, max); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { idx->set_quantizer(min, max); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_set_quantizer", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_set_quantizer", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_set_quantizer", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_set_quantizer", "unknown C++ exception"); } } void gpu_ivf_pq_get_quantizer(gpu_ivf_pq_c index_c, float* min, float* max, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->get_quantizer(min, max); break; - case Quantization_F16: static_cast*>(any->ptr)->get_quantizer(min, max); break; - case Quantization_INT8: static_cast*>(any->ptr)->get_quantizer(min, max); break; - case Quantization_UINT8: static_cast*>(any->ptr)->get_quantizer(min, max); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { idx->get_quantizer(min, max); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_get_quantizer", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_get_quantizer", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_get_quantizer", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_get_quantizer", "unknown C++ exception"); } } void gpu_ivf_pq_save(gpu_ivf_pq_c index_c, const char* filename, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->save(filename); break; - case Quantization_F16: static_cast*>(any->ptr)->save(filename); break; - case Quantization_INT8: static_cast*>(any->ptr)->save(filename); break; - case Quantization_UINT8: static_cast*>(any->ptr)->save(filename); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { idx->save(filename); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_save", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_save", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_save", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_save", "unknown C++ exception"); } } void gpu_ivf_pq_save_dir(gpu_ivf_pq_c index_c, const char* dir, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->save_dir(dir); break; - case Quantization_F16: static_cast*>(any->ptr)->save_dir(dir); break; - case Quantization_INT8: static_cast*>(any->ptr)->save_dir(dir); break; - case Quantization_UINT8: static_cast*>(any->ptr)->save_dir(dir); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { idx->save_dir(dir); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_save_dir", e.what()); } catch (...) { @@ -478,14 +413,7 @@ void gpu_ivf_pq_save_dir(gpu_ivf_pq_c index_c, const char* dir, void* errmsg) { void gpu_ivf_pq_delete_id(gpu_ivf_pq_c index_c, int64_t id, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->delete_id(id); break; - case Quantization_F16: static_cast*>(any->ptr)->delete_id(id); break; - case Quantization_INT8: static_cast*>(any->ptr)->delete_id(id); break; - case Quantization_UINT8: static_cast*>(any->ptr)->delete_id(id); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { idx->delete_id(id); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_delete_id", e.what()); } catch (...) { @@ -497,14 +425,7 @@ void gpu_ivf_pq_load_dir(gpu_ivf_pq_c index_c, const char* dir, distribution_mode_t target_mode, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - case Quantization_F16: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - case Quantization_INT8: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - case Quantization_UINT8: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { idx->load_dir(dir, target_mode); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_load_dir", e.what()); } catch (...) { @@ -512,69 +433,55 @@ void gpu_ivf_pq_load_dir(gpu_ivf_pq_c index_c, const char* dir, } } -gpu_ivf_pq_search_res_t gpu_ivf_pq_search(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +gpu_ivf_pq_search_res_t gpu_ivf_pq_search(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_pq_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + *cpp_res = idx->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_search", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_search", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search", "unknown C++ exception"); } return result; } -gpu_ivf_pq_search_res_t gpu_ivf_pq_search_float(gpu_ivf_pq_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +gpu_ivf_pq_search_res_t gpu_ivf_pq_search_quantize(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_pq_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + *cpp_res = idx->search_quantize(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_float", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_quantize", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_float", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_quantize", "unknown C++ exception"); } return result; } -uint64_t gpu_ivf_pq_search_async(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_ivf_pq_search_async(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_F16: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_INT8: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_UINT8: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - default: return 0; - } + return ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using Q = typename std::remove_pointer_t::storage_type; + return idx->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_async", e.what()); return 0; @@ -584,24 +491,20 @@ uint64_t gpu_ivf_pq_search_async(gpu_ivf_pq_c index_c, const void* queries_data, } } -uint64_t gpu_ivf_pq_search_float_async(gpu_ivf_pq_c index_c, const float* queries_data, uint64_t num_queries, +uint64_t gpu_ivf_pq_search_quantize_async(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_F16: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_INT8: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_UINT8: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - default: return 0; - } + return ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using B = typename std::remove_pointer_t::base_type; + return idx->search_quantize_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_float_async", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_quantize_async", e.what()); return 0; } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_float_async", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_quantize_async", "unknown C++ exception"); return 0; } } @@ -610,15 +513,10 @@ gpu_ivf_pq_search_res_t gpu_ivf_pq_search_wait(gpu_ivf_pq_c index_c, uint64_t jo if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_pq_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + *cpp_res = idx->search_wait(job_id); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_wait", e.what()); @@ -677,14 +575,7 @@ void gpu_ivf_pq_free_result(gpu_ivf_pq_result_c result_c) { uint64_t gpu_ivf_pq_cap(gpu_ivf_pq_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->cap(); - case Quantization_F16: return static_cast*>(any->ptr)->cap(); - case Quantization_INT8: return static_cast*>(any->ptr)->cap(); - case Quantization_UINT8: return static_cast*>(any->ptr)->cap(); - default: return 0; - } + return ivf_pq_dispatch(static_cast(index_c), [](auto* idx) -> uint64_t { return idx->cap(); }); } catch (...) { return 0; } @@ -693,14 +584,7 @@ uint64_t gpu_ivf_pq_cap(gpu_ivf_pq_c index_c) { uint64_t gpu_ivf_pq_len(gpu_ivf_pq_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->len(); - case Quantization_F16: return static_cast*>(any->ptr)->len(); - case Quantization_INT8: return static_cast*>(any->ptr)->len(); - case Quantization_UINT8: return static_cast*>(any->ptr)->len(); - default: return 0; - } + return ivf_pq_dispatch(static_cast(index_c), [](auto* idx) -> uint64_t { return idx->len(); }); } catch (...) { return 0; } @@ -713,15 +597,9 @@ char* gpu_ivf_pq_get_filter_col_meta_json(gpu_ivf_pq_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; if (!index_c) return strdup(""); try { - auto* any = static_cast(index_c); - std::string json; - switch (any->qtype) { - case Quantization_F32: json = matrixone::format_filter_col_meta(static_cast*>(any->ptr)->filter_host_.columns); break; - case Quantization_F16: json = matrixone::format_filter_col_meta(static_cast*>(any->ptr)->filter_host_.columns); break; - case Quantization_INT8: json = matrixone::format_filter_col_meta(static_cast*>(any->ptr)->filter_host_.columns); break; - case Quantization_UINT8: json = matrixone::format_filter_col_meta(static_cast*>(any->ptr)->filter_host_.columns); break; - default: return strdup(""); - } + std::string json = ivf_pq_dispatch(static_cast(index_c), [](auto* idx) -> std::string { + return matrixone::format_filter_col_meta(idx->filter_host_.columns); + }); return strdup(json.c_str()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_get_filter_col_meta_json", e.what()); @@ -736,23 +614,13 @@ char* gpu_ivf_pq_info(gpu_ivf_pq_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; if (!index_c) return nullptr; try { - auto* any = static_cast(index_c); - std::string info; - switch (any->qtype) { - case Quantization_F32: info = static_cast*>(any->ptr)->info(); break; - case Quantization_F16: info = static_cast*>(any->ptr)->info(); break; - case Quantization_INT8: info = static_cast*>(any->ptr)->info(); break; - case Quantization_UINT8: info = static_cast*>(any->ptr)->info(); break; - default: return nullptr; - } + std::string info = ivf_pq_dispatch(static_cast(index_c), [](auto* idx) -> std::string { return idx->info(); }); return strdup(info.c_str()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_info", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_info", e.what()); return nullptr; } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_info", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_info", "unknown C++ exception"); return nullptr; } } @@ -768,34 +636,20 @@ void gpu_ivf_pq_get_centers(gpu_ivf_pq_c index_c, void* centers, uint64_t count, std::copy(src.begin(), src.begin() + n, static_cast(dst)); }; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: copy_clamped(static_cast*>(any->ptr)->get_centers(), centers); break; - case Quantization_F16: copy_clamped(static_cast*>(any->ptr)->get_centers(), centers); break; - case Quantization_INT8: copy_clamped(static_cast*>(any->ptr)->get_centers(), centers); break; - case Quantization_UINT8: copy_clamped(static_cast*>(any->ptr)->get_centers(), centers); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + copy_clamped(idx->get_centers(), centers); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_get_centers", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_get_centers", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_get_centers", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_get_centers", "unknown C++ exception"); } } uint32_t gpu_ivf_pq_get_n_list(gpu_ivf_pq_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->get_n_list(); - case Quantization_F16: return static_cast*>(any->ptr)->get_n_list(); - case Quantization_INT8: return static_cast*>(any->ptr)->get_n_list(); - case Quantization_UINT8: return static_cast*>(any->ptr)->get_n_list(); - default: return 0; - } + return ivf_pq_dispatch(static_cast(index_c), [](auto* idx) -> uint32_t { return idx->get_n_list(); }); } catch (...) { return 0; } @@ -804,14 +658,7 @@ uint32_t gpu_ivf_pq_get_n_list(gpu_ivf_pq_c index_c) { uint32_t gpu_ivf_pq_get_dim(gpu_ivf_pq_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->get_dim(); - case Quantization_F16: return static_cast*>(any->ptr)->get_dim(); - case Quantization_INT8: return static_cast*>(any->ptr)->get_dim(); - case Quantization_UINT8: return static_cast*>(any->ptr)->get_dim(); - default: return 0; - } + return ivf_pq_dispatch(static_cast(index_c), [](auto* idx) -> uint32_t { return idx->get_dim(); }); } catch (...) { return 0; } @@ -820,14 +667,7 @@ uint32_t gpu_ivf_pq_get_dim(gpu_ivf_pq_c index_c) { uint32_t gpu_ivf_pq_get_rot_dim(gpu_ivf_pq_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->get_rot_dim(); - case Quantization_F16: return static_cast*>(any->ptr)->get_rot_dim(); - case Quantization_INT8: return static_cast*>(any->ptr)->get_rot_dim(); - case Quantization_UINT8: return static_cast*>(any->ptr)->get_rot_dim(); - default: return 0; - } + return ivf_pq_dispatch(static_cast(index_c), [](auto* idx) -> uint32_t { return idx->get_rot_dim(); }); } catch (...) { return 0; } @@ -836,14 +676,7 @@ uint32_t gpu_ivf_pq_get_rot_dim(gpu_ivf_pq_c index_c) { uint32_t gpu_ivf_pq_get_dim_ext(gpu_ivf_pq_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->get_dim_ext(); - case Quantization_F16: return static_cast*>(any->ptr)->get_dim_ext(); - case Quantization_INT8: return static_cast*>(any->ptr)->get_dim_ext(); - case Quantization_UINT8: return static_cast*>(any->ptr)->get_dim_ext(); - default: return 0; - } + return ivf_pq_dispatch(static_cast(index_c), [](auto* idx) -> uint32_t { return idx->get_dim_ext(); }); } catch (...) { return 0; } @@ -853,30 +686,11 @@ void gpu_ivf_pq_get_dataset(gpu_ivf_pq_c index_c, void* out_data) { // This is for debugging, we just copy the host dataset if it exists try { if (!index_c) return; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: { - auto& ds = static_cast*>(any->ptr)->flattened_host_dataset; - if (!ds.empty()) std::copy(ds.begin(), ds.end(), static_cast(out_data)); - break; - } - case Quantization_F16: { - auto& ds = static_cast*>(any->ptr)->flattened_host_dataset; - if (!ds.empty()) std::copy(ds.begin(), ds.end(), static_cast(out_data)); - break; - } - case Quantization_INT8: { - auto& ds = static_cast*>(any->ptr)->flattened_host_dataset; - if (!ds.empty()) std::copy(ds.begin(), ds.end(), static_cast(out_data)); - break; - } - case Quantization_UINT8: { - auto& ds = static_cast*>(any->ptr)->flattened_host_dataset; - if (!ds.empty()) std::copy(ds.begin(), ds.end(), static_cast(out_data)); - break; - } - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + auto& ds = idx->flattened_host_dataset; + if (!ds.empty()) std::copy(ds.begin(), ds.end(), static_cast(out_data)); + }); } catch (...) { matrixone::log_err("gpu_ivf_pq_get_dataset: unknown C++ exception (swallowed)"); } @@ -888,15 +702,10 @@ void gpu_ivf_pq_set_filter_columns(gpu_ivf_pq_c index_c, const char* col_meta_js uint64_t total_count, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); std::string s = col_meta_json ? col_meta_json : ""; - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - case Quantization_F16: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + idx->set_filter_columns(s, total_count); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_set_filter_columns", e.what()); } catch (...) { @@ -909,14 +718,9 @@ void gpu_ivf_pq_add_filter_chunk(gpu_ivf_pq_c index_c, uint32_t col_idx, uint64_t nrows, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_F16: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_INT8: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_UINT8: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + idx->add_filter_chunk(col_idx, data, null_bitmap, nrows); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_add_filter_chunk", e.what()); } catch (...) { @@ -931,16 +735,12 @@ gpu_ivf_pq_search_res_t gpu_ivf_pq_search_with_filter(gpu_ivf_pq_c index_c, cons if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_pq_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + *cpp_res = idx->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_with_filter", e.what()); @@ -950,52 +750,44 @@ gpu_ivf_pq_search_res_t gpu_ivf_pq_search_with_filter(gpu_ivf_pq_c index_c, cons return result; } -gpu_ivf_pq_search_res_t gpu_ivf_pq_search_float_with_filter(gpu_ivf_pq_c index_c, const float* queries_data, +gpu_ivf_pq_search_res_t gpu_ivf_pq_search_quantize_with_filter(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t sp, const char* preds_json, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_pq_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + *cpp_res = idx->search_quantize_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_float_with_filter", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_quantize_with_filter", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_float_with_filter", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_quantize_with_filter", "unknown C++ exception"); } return result; } -uint64_t gpu_ivf_pq_search_float_with_filter_async(gpu_ivf_pq_c index_c, const float* queries_data, +uint64_t gpu_ivf_pq_search_quantize_with_filter_async(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t sp, const char* preds_json, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_F16: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_INT8: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_UINT8: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - default: return 0; - } + return ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using B = typename std::remove_pointer_t::base_type; + return idx->search_quantize_with_filter_async(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_float_with_filter_async", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_quantize_with_filter_async", e.what()); return 0; } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_float_with_filter_async", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_quantize_with_filter_async", "unknown C++ exception"); return 0; } } @@ -1003,8 +795,11 @@ uint64_t gpu_ivf_pq_search_float_with_filter_async(gpu_ivf_pq_c index_c, const f } // extern "C" namespace matrixone { -template class gpu_ivf_pq_t; -template class gpu_ivf_pq_t; -template class gpu_ivf_pq_t; -template class gpu_ivf_pq_t; +template class gpu_ivf_pq_t; +template class gpu_ivf_pq_t; +template class gpu_ivf_pq_t; +template class gpu_ivf_pq_t; +template class gpu_ivf_pq_t; +template class gpu_ivf_pq_t; +template class gpu_ivf_pq_t; } // namespace matrixone diff --git a/cgo/cuvs/ivf_pq_c.h b/cgo/cuvs/ivf_pq_c.h index 4d68b6e23e27d..eeb30a6da0e86 100644 --- a/cgo/cuvs/ivf_pq_c.h +++ b/cgo/cuvs/ivf_pq_c.h @@ -31,30 +31,34 @@ typedef void* gpu_ivf_pq_c; // Opaque pointer to the C++ IVF-PQ search result object typedef void* gpu_ivf_pq_result_c; +// btype = base/query/quantizer-source element type (Quantization_F32 or F16). +// qtype = storage element type. Wired combos: F32 base {F32,F16,INT8,UINT8}; +// F16 base {F16,INT8,UINT8}. Other combinations set errmsg and return NULL. + // Constructor for building from dataset -gpu_ivf_pq_c gpu_ivf_pq_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, +gpu_ivf_pq_c gpu_ivf_pq_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric, ivf_pq_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg); // Constructor for building from MODF datafile -gpu_ivf_pq_c gpu_ivf_pq_new_from_data_file(const char* data_filename, distance_type_t metric, +gpu_ivf_pq_c gpu_ivf_pq_new_from_data_file(const char* data_filename, distance_type_t metric, ivf_pq_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, void* errmsg); + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, void* errmsg); // Constructor for loading from file gpu_ivf_pq_c gpu_ivf_pq_load_file(const char* filename, uint32_t dimension, distance_type_t metric, ivf_pq_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, void* errmsg); + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, void* errmsg); // Constructor for an empty index (pre-allocates) -gpu_ivf_pq_c gpu_ivf_pq_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric, +gpu_ivf_pq_c gpu_ivf_pq_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric, ivf_pq_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg); // Add chunk of data (same type as index quantization) @@ -72,8 +76,19 @@ void gpu_ivf_pq_extend_float(gpu_ivf_pq_c index_c, const float* new_data, uint64 // Add chunk of data (from float, with on-the-fly quantization if needed) void gpu_ivf_pq_add_chunk_float(gpu_ivf_pq_c index_c, const float* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); -// Trains the scalar quantizer (if T is 1-byte) -void gpu_ivf_pq_train_quantizer(gpu_ivf_pq_c index_c, const float* train_data, uint64_t n_samples, void* errmsg); +// Add chunk of base-typed (B) data, quantizing natively to a 1-byte storage type +// (int8/uint8) via the B-source quantizer. base_data is a host buffer of +// chunk_count*dimension B elements (passed as raw bytes; B = btype). Requires int8/uint8 storage. +void gpu_ivf_pq_add_chunk_quantize(gpu_ivf_pq_c index_c, const void* base_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); + +// Quantize a base-typed (B) query to the 1-byte storage type via the B-source +// quantizer, writing num_queries*dimension bytes into out. The caller then runs +// the normal native search with the quantized query. Requires int8/uint8 storage. +void gpu_ivf_pq_quantize_query(gpu_ivf_pq_c index_c, const void* base_data, uint64_t num_queries, void* out, void* errmsg); + +// Trains the scalar quantizer (if storage is 1-byte). train_data is a host buffer +// of n_samples*dimension B elements (B = btype), passed as raw bytes. +void gpu_ivf_pq_train_quantizer(gpu_ivf_pq_c index_c, const void* train_data, uint64_t n_samples, void* errmsg); void gpu_ivf_pq_set_batch_window(gpu_ivf_pq_c index_c, int64_t window_us, void* errmsg); void gpu_ivf_pq_set_dynb_conservative_dispatch(gpu_ivf_pq_c index_c, bool enable, void* errmsg); @@ -115,17 +130,19 @@ gpu_ivf_pq_search_res_t gpu_ivf_pq_search(gpu_ivf_pq_c index_c, const void* quer uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, void* errmsg); -gpu_ivf_pq_search_res_t gpu_ivf_pq_search_float(gpu_ivf_pq_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +// Quantize search: query in the BASE element type B (float or half); the index +// converts it to storage type T (copy / quantize / f32->f16 cast) internally. +gpu_ivf_pq_search_res_t gpu_ivf_pq_search_quantize(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, void* errmsg); // Asynchronous search functions -uint64_t gpu_ivf_pq_search_async(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_ivf_pq_search_async(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, void* errmsg); -uint64_t gpu_ivf_pq_search_float_async(gpu_ivf_pq_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_ivf_pq_search_quantize_async(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, void* errmsg); gpu_ivf_pq_search_res_t gpu_ivf_pq_search_wait(gpu_ivf_pq_c index_c, uint64_t job_id, void* errmsg); @@ -189,15 +206,16 @@ gpu_ivf_pq_search_res_t gpu_ivf_pq_search_with_filter(gpu_ivf_pq_c index_c, cons uint32_t limit, ivf_pq_search_params_t search_params, const char* preds_json, void* errmsg); -gpu_ivf_pq_search_res_t gpu_ivf_pq_search_float_with_filter(gpu_ivf_pq_c index_c, const float* queries_data, +// Query in the BASE element type B (float or half); converted to storage T internally. +gpu_ivf_pq_search_res_t gpu_ivf_pq_search_quantize_with_filter(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, const char* preds_json, void* errmsg); -// Async variant of gpu_ivf_pq_search_float_with_filter. Returns a job_id that +// Async variant of gpu_ivf_pq_search_quantize_with_filter. Returns a job_id that // is collected with the existing gpu_ivf_pq_search_wait. Lets multi-index // callers fan out filtered searches across shards in parallel. -uint64_t gpu_ivf_pq_search_float_with_filter_async(gpu_ivf_pq_c index_c, const float* queries_data, +uint64_t gpu_ivf_pq_search_quantize_with_filter_async(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, const char* preds_json, void* errmsg); diff --git a/cgo/cuvs/kmeans.hpp b/cgo/cuvs/kmeans.hpp index 2c12e0053ec59..82b26327b1fb5 100644 --- a/cgo/cuvs/kmeans.hpp +++ b/cgo/cuvs/kmeans.hpp @@ -74,8 +74,10 @@ struct kmeans_result_t { * Note: cuVS KMeans fits and predicts always use float centroids internally. */ template -class gpu_kmeans_t : public gpu_index_base_t { +class gpu_kmeans_t : public gpu_index_base_t { public: + using base_type = float; + using storage_type = T; // Internal centroids storage - ALWAYS float for cuVS KMeans std::unique_ptr> centroids_; @@ -184,6 +186,12 @@ class gpu_kmeans_t : public gpu_index_base_t std::unique_lock lock(this->mutex_); auto res = handle.get_raft_resources(); + // cuVS kmeans is not safe to run twice at once on one GPU (the same + // reason ivf_flat/cagra/ivf_pq/brute_force take this lock for build). + // Without it, concurrent async ivfflat reindexes each run their own + // GpuKMeans::fit on device 0 and race on GPU/RMM state -> SIGABRT. + std::lock_guard build_lk(matrixone::device_build_mutex(handle.get_device_id())); + cuvs::cluster::kmeans::balanced_params kmeans_params; kmeans_params.metric = static_cast(this->metric); kmeans_params.n_iters = static_cast(this->build_params.max_iter); @@ -220,6 +228,13 @@ class gpu_kmeans_t : public gpu_index_base_t std::unique_lock lock(this->mutex_); auto res = handle.get_raft_resources(); + // cuVS kmeans is not safe to run twice at once on one GPU (the + // same reason ivf_flat/cagra/ivf_pq/brute_force take this lock + // for build). Without it, concurrent async ivfflat reindexes + // each run their own GpuKMeans::fit on device 0 and race on + // GPU/RMM state -> SIGABRT. + std::lock_guard build_lk(matrixone::device_build_mutex(handle.get_device_id())); + cuvs::cluster::kmeans::balanced_params kmeans_params; kmeans_params.metric = static_cast(this->metric); kmeans_params.n_iters = static_cast(this->build_params.max_iter); @@ -345,6 +360,13 @@ class gpu_kmeans_t : public gpu_index_base_t std::unique_lock lock(this->mutex_); auto res = handle.get_raft_resources(); + // cuVS kmeans is not safe to run twice at once on one GPU (the + // same reason ivf_flat/cagra/ivf_pq/brute_force take this lock + // for build). Without it, concurrent async ivfflat reindexes + // each run their own GpuKMeans::fit on device 0 and race on + // GPU/RMM state -> SIGABRT. + std::lock_guard build_lk(matrixone::device_build_mutex(handle.get_device_id())); + cuvs::cluster::kmeans::balanced_params kmeans_params; kmeans_params.metric = static_cast(this->metric); kmeans_params.n_iters = static_cast(this->build_params.max_iter); @@ -389,12 +411,16 @@ class gpu_kmeans_t : public gpu_index_base_t kmeans_result_t fit_predict_float(const float* dataset_data, uint64_t count_vectors) { this->count = count_vectors; this->train_quantizer_if_needed(); - + uint64_t job_id = this->worker->submit_main( [&](raft_handle_wrapper_t& handle) -> std::any { std::unique_lock lock(this->mutex_); auto res = handle.get_raft_resources(); + // cuVS kmeans is not safe to run twice at once on one GPU; see + // the device_build_mutex note in fit()/build_internal(). + std::lock_guard build_lk(matrixone::device_build_mutex(handle.get_device_id())); + auto dataset_device_f = raft::make_device_matrix(*res, (int64_t)this->count, (int64_t)this->dimension); raft::copy(*res, dataset_device_f.view(), raft::make_host_matrix_view(dataset_data, this->count, this->dimension)); raft::resource::sync_stream(*res); @@ -458,7 +484,7 @@ class gpu_kmeans_t : public gpu_index_base_t } std::string info() const override { - std::string json = gpu_index_base_t::info(); + std::string json = gpu_index_base_t::info(); json += ", \"type\": \"KMeans\", \"kmeans\": {"; if (centroids_) json += "\"clusters\": " + std::to_string(centroids_->extent(0)); else json += "\"built\": false"; diff --git a/cgo/cuvs/python/cuvs.py b/cgo/cuvs/python/cuvs.py index 6a578c7beb0ec..14f2d7a40c49a 100644 --- a/cgo/cuvs/python/cuvs.py +++ b/cgo/cuvs/python/cuvs.py @@ -64,6 +64,20 @@ class Quantization(IntEnum): INT8 = 2 UINT8 = 3 +# numpy dtype for a base/storage element type. Base-typed buffers (dataset, +# query, quantizer training data) must carry these exact bytes so the C side +# reads them per btype — e.g. a vecf16 base must be passed as float16, NOT +# coerced to float32 (which would double the width and misread the halfs). +_NP_DTYPE = { + Quantization.F32: np.float32, + Quantization.F16: np.float16, + Quantization.INT8: np.int8, + Quantization.UINT8: np.uint8, +} + +def _np_dtype_for(quant): + return _NP_DTYPE[Quantization(int(quant))] + class DistributionMode(IntEnum): SINGLE_GPU = 0 SHARDED = 1 @@ -139,11 +153,11 @@ def _check_error(errmsg_ptr): _lib.gpu_adhoc_brute_force_search_float.argtypes = [ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.POINTER(ctypes.c_float), ctypes.c_void_p] # CAGRA - _lib.gpu_cagra_new.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, CagraBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] + _lib.gpu_cagra_new.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, CagraBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] _lib.gpu_cagra_new.restype = ctypes.c_void_p - _lib.gpu_cagra_new_empty.argtypes = [ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, CagraBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] + _lib.gpu_cagra_new_empty.argtypes = [ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, CagraBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] _lib.gpu_cagra_new_empty.restype = ctypes.c_void_p - _lib.gpu_cagra_load_file.argtypes = [ctypes.c_char_p, ctypes.c_uint32, ctypes.c_int, CagraBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_void_p] + _lib.gpu_cagra_load_file.argtypes = [ctypes.c_char_p, ctypes.c_uint32, ctypes.c_int, CagraBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_void_p] _lib.gpu_cagra_load_file.restype = ctypes.c_void_p _lib.gpu_cagra_destroy.argtypes = [ctypes.c_void_p, ctypes.c_void_p] _lib.gpu_cagra_start.argtypes = [ctypes.c_void_p, ctypes.c_void_p] @@ -161,12 +175,12 @@ def _check_error(errmsg_ptr): _lib.gpu_cagra_delete_id.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_void_p] _lib.gpu_cagra_search.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, CagraSearchParams, ctypes.c_void_p] _lib.gpu_cagra_search.restype = CagraSearchRes - _lib.gpu_cagra_search_float.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, CagraSearchParams, ctypes.c_void_p] - _lib.gpu_cagra_search_float.restype = CagraSearchRes + _lib.gpu_cagra_search_quantize.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, CagraSearchParams, ctypes.c_void_p] + _lib.gpu_cagra_search_quantize.restype = CagraSearchRes _lib.gpu_cagra_search_async.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, CagraSearchParams, ctypes.c_void_p] _lib.gpu_cagra_search_async.restype = ctypes.c_uint64 - _lib.gpu_cagra_search_float_async.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, CagraSearchParams, ctypes.c_void_p] - _lib.gpu_cagra_search_float_async.restype = ctypes.c_uint64 + _lib.gpu_cagra_search_quantize_async.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, CagraSearchParams, ctypes.c_void_p] + _lib.gpu_cagra_search_quantize_async.restype = ctypes.c_uint64 _lib.gpu_cagra_search_wait.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_void_p] _lib.gpu_cagra_search_wait.restype = CagraSearchRes _lib.gpu_cagra_get_neighbors.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.POINTER(ctypes.c_int64)] @@ -184,15 +198,15 @@ def _check_error(errmsg_ptr): _lib.gpu_cagra_add_filter_chunk.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint32), ctypes.c_uint64, ctypes.c_void_p] _lib.gpu_cagra_search_with_filter.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, CagraSearchParams, ctypes.c_char_p, ctypes.c_void_p] _lib.gpu_cagra_search_with_filter.restype = CagraSearchRes - _lib.gpu_cagra_search_float_with_filter.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, CagraSearchParams, ctypes.c_char_p, ctypes.c_void_p] - _lib.gpu_cagra_search_float_with_filter.restype = CagraSearchRes + _lib.gpu_cagra_search_quantize_with_filter.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, CagraSearchParams, ctypes.c_char_p, ctypes.c_void_p] + _lib.gpu_cagra_search_quantize_with_filter.restype = CagraSearchRes # IVF-Flat - _lib.gpu_ivf_flat_new.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, IvfFlatBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] + _lib.gpu_ivf_flat_new.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, IvfFlatBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] _lib.gpu_ivf_flat_new.restype = ctypes.c_void_p - _lib.gpu_ivf_flat_load_file.argtypes = [ctypes.c_char_p, ctypes.c_uint32, ctypes.c_int, IvfFlatBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_void_p] + _lib.gpu_ivf_flat_load_file.argtypes = [ctypes.c_char_p, ctypes.c_uint32, ctypes.c_int, IvfFlatBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_void_p] _lib.gpu_ivf_flat_load_file.restype = ctypes.c_void_p - _lib.gpu_ivf_flat_new_empty.argtypes = [ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, IvfFlatBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] + _lib.gpu_ivf_flat_new_empty.argtypes = [ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, IvfFlatBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] _lib.gpu_ivf_flat_new_empty.restype = ctypes.c_void_p _lib.gpu_ivf_flat_destroy.argtypes = [ctypes.c_void_p, ctypes.c_void_p] _lib.gpu_ivf_flat_start.argtypes = [ctypes.c_void_p, ctypes.c_void_p] @@ -211,12 +225,12 @@ def _check_error(errmsg_ptr): _lib.gpu_ivf_flat_delete_id.argtypes = [ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p] _lib.gpu_ivf_flat_search.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfFlatSearchParams, ctypes.c_void_p] _lib.gpu_ivf_flat_search.restype = IvfFlatSearchRes - _lib.gpu_ivf_flat_search_float.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfFlatSearchParams, ctypes.c_void_p] - _lib.gpu_ivf_flat_search_float.restype = IvfFlatSearchRes + _lib.gpu_ivf_flat_search_quantize.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfFlatSearchParams, ctypes.c_void_p] + _lib.gpu_ivf_flat_search_quantize.restype = IvfFlatSearchRes _lib.gpu_ivf_flat_search_async.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfFlatSearchParams, ctypes.c_void_p] _lib.gpu_ivf_flat_search_async.restype = ctypes.c_uint64 - _lib.gpu_ivf_flat_search_float_async.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfFlatSearchParams, ctypes.c_void_p] - _lib.gpu_ivf_flat_search_float_async.restype = ctypes.c_uint64 + _lib.gpu_ivf_flat_search_quantize_async.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfFlatSearchParams, ctypes.c_void_p] + _lib.gpu_ivf_flat_search_quantize_async.restype = ctypes.c_uint64 _lib.gpu_ivf_flat_search_wait.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_void_p] _lib.gpu_ivf_flat_search_wait.restype = IvfFlatSearchRes _lib.gpu_ivf_flat_get_neighbors.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.POINTER(ctypes.c_int64)] @@ -235,17 +249,17 @@ def _check_error(errmsg_ptr): _lib.gpu_ivf_flat_add_filter_chunk.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint32), ctypes.c_uint64, ctypes.c_void_p] _lib.gpu_ivf_flat_search_with_filter.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfFlatSearchParams, ctypes.c_char_p, ctypes.c_void_p] _lib.gpu_ivf_flat_search_with_filter.restype = IvfFlatSearchRes - _lib.gpu_ivf_flat_search_float_with_filter.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfFlatSearchParams, ctypes.c_char_p, ctypes.c_void_p] - _lib.gpu_ivf_flat_search_float_with_filter.restype = IvfFlatSearchRes + _lib.gpu_ivf_flat_search_quantize_with_filter.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfFlatSearchParams, ctypes.c_char_p, ctypes.c_void_p] + _lib.gpu_ivf_flat_search_quantize_with_filter.restype = IvfFlatSearchRes # IVF-PQ - _lib.gpu_ivf_pq_new.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, IvfPqBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] + _lib.gpu_ivf_pq_new.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, IvfPqBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] _lib.gpu_ivf_pq_new.restype = ctypes.c_void_p - _lib.gpu_ivf_pq_new_from_data_file.argtypes = [ctypes.c_char_p, ctypes.c_int, IvfPqBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_void_p] + _lib.gpu_ivf_pq_new_from_data_file.argtypes = [ctypes.c_char_p, ctypes.c_int, IvfPqBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_void_p] _lib.gpu_ivf_pq_new_from_data_file.restype = ctypes.c_void_p - _lib.gpu_ivf_pq_load_file.argtypes = [ctypes.c_char_p, ctypes.c_uint32, ctypes.c_int, IvfPqBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_void_p] + _lib.gpu_ivf_pq_load_file.argtypes = [ctypes.c_char_p, ctypes.c_uint32, ctypes.c_int, IvfPqBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_void_p] _lib.gpu_ivf_pq_load_file.restype = ctypes.c_void_p - _lib.gpu_ivf_pq_new_empty.argtypes = [ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, IvfPqBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] + _lib.gpu_ivf_pq_new_empty.argtypes = [ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, IvfPqBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] _lib.gpu_ivf_pq_new_empty.restype = ctypes.c_void_p _lib.gpu_ivf_pq_destroy.argtypes = [ctypes.c_void_p, ctypes.c_void_p] _lib.gpu_ivf_pq_start.argtypes = [ctypes.c_void_p, ctypes.c_void_p] @@ -264,12 +278,12 @@ def _check_error(errmsg_ptr): _lib.gpu_ivf_pq_delete_id.argtypes = [ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p] _lib.gpu_ivf_pq_search.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfPqSearchParams, ctypes.c_void_p] _lib.gpu_ivf_pq_search.restype = IvfPqSearchRes - _lib.gpu_ivf_pq_search_float.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfPqSearchParams, ctypes.c_void_p] - _lib.gpu_ivf_pq_search_float.restype = IvfPqSearchRes + _lib.gpu_ivf_pq_search_quantize.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfPqSearchParams, ctypes.c_void_p] + _lib.gpu_ivf_pq_search_quantize.restype = IvfPqSearchRes _lib.gpu_ivf_pq_search_async.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfPqSearchParams, ctypes.c_void_p] _lib.gpu_ivf_pq_search_async.restype = ctypes.c_uint64 - _lib.gpu_ivf_pq_search_float_async.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfPqSearchParams, ctypes.c_void_p] - _lib.gpu_ivf_pq_search_float_async.restype = ctypes.c_uint64 + _lib.gpu_ivf_pq_search_quantize_async.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfPqSearchParams, ctypes.c_void_p] + _lib.gpu_ivf_pq_search_quantize_async.restype = ctypes.c_uint64 _lib.gpu_ivf_pq_search_wait.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_void_p] _lib.gpu_ivf_pq_search_wait.restype = IvfPqSearchRes _lib.gpu_ivf_pq_get_neighbors.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.POINTER(ctypes.c_int64)] @@ -295,27 +309,28 @@ def _check_error(errmsg_ptr): _lib.gpu_ivf_pq_add_filter_chunk.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint32), ctypes.c_uint64, ctypes.c_void_p] _lib.gpu_ivf_pq_search_with_filter.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfPqSearchParams, ctypes.c_char_p, ctypes.c_void_p] _lib.gpu_ivf_pq_search_with_filter.restype = IvfPqSearchRes - _lib.gpu_ivf_pq_search_float_with_filter.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfPqSearchParams, ctypes.c_char_p, ctypes.c_void_p] - _lib.gpu_ivf_pq_search_float_with_filter.restype = IvfPqSearchRes + _lib.gpu_ivf_pq_search_quantize_with_filter.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfPqSearchParams, ctypes.c_char_p, ctypes.c_void_p] + _lib.gpu_ivf_pq_search_quantize_with_filter.restype = IvfPqSearchRes # Brute Force - _lib.gpu_brute_force_new.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] + # btype before qtype (base type, then storage type) + _lib.gpu_brute_force_new.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] _lib.gpu_brute_force_new.restype = ctypes.c_void_p - _lib.gpu_brute_force_new_empty.argtypes = [ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] + _lib.gpu_brute_force_new_empty.argtypes = [ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] _lib.gpu_brute_force_new_empty.restype = ctypes.c_void_p _lib.gpu_brute_force_destroy.argtypes = [ctypes.c_void_p, ctypes.c_void_p] _lib.gpu_brute_force_start.argtypes = [ctypes.c_void_p, ctypes.c_void_p] _lib.gpu_brute_force_build.argtypes = [ctypes.c_void_p, ctypes.c_void_p] _lib.gpu_brute_force_add_chunk.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] - _lib.gpu_brute_force_add_chunk_float.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] + _lib.gpu_brute_force_add_chunk_quantize.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] _lib.gpu_brute_force_search.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p] _lib.gpu_brute_force_search.restype = ctypes.c_void_p - _lib.gpu_brute_force_search_float.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p] - _lib.gpu_brute_force_search_float.restype = ctypes.c_void_p + _lib.gpu_brute_force_search_quantize.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p] + _lib.gpu_brute_force_search_quantize.restype = ctypes.c_void_p _lib.gpu_brute_force_search_async.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p] _lib.gpu_brute_force_search_async.restype = ctypes.c_uint64 - _lib.gpu_brute_force_search_float_async.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p] - _lib.gpu_brute_force_search_float_async.restype = ctypes.c_uint64 + _lib.gpu_brute_force_search_quantize_async.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p] + _lib.gpu_brute_force_search_quantize_async.restype = ctypes.c_uint64 _lib.gpu_brute_force_search_wait.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_void_p] _lib.gpu_brute_force_search_wait.restype = ctypes.c_void_p _lib.gpu_brute_force_get_results.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.POINTER(ctypes.c_int64), ctypes.POINTER(ctypes.c_float)] @@ -386,31 +401,31 @@ def __init__(self, handle, dimension): self.dimension = dimension @classmethod - def create(cls, dataset, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32, ids=None): + def create(cls, dataset, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32, ids=None): if build_params is None: build_params = CagraBuildParams.default() - dataset = np.ascontiguousarray(dataset, dtype=np.float32) + dataset = np.ascontiguousarray(dataset, dtype=_np_dtype_for(btype)) count, dim = dataset.shape dev_arr = (ctypes.c_int * len(devices))(*devices) id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None errmsg = ctypes.c_char_p() - h = _lib.gpu_cagra_new(dataset.ctypes.data_as(ctypes.c_void_p), count, dim, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), id_ptr, ctypes.byref(errmsg)) - _check_error(errmsg); return cls(h, dim) + h = _lib.gpu_cagra_new(dataset.ctypes.data_as(ctypes.c_void_p), count, dim, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), id_ptr, ctypes.byref(errmsg)) + _check_error(errmsg); idx = cls(h, dim); idx.btype = int(btype); return idx @classmethod - def create_empty(cls, total_count, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32, ids=None): + def create_empty(cls, total_count, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32, ids=None): if build_params is None: build_params = CagraBuildParams.default() dev_arr = (ctypes.c_int * len(devices))(*devices) id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None errmsg = ctypes.c_char_p() - h = _lib.gpu_cagra_new_empty(total_count, dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), id_ptr, ctypes.byref(errmsg)) + h = _lib.gpu_cagra_new_empty(total_count, dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), id_ptr, ctypes.byref(errmsg)) _check_error(errmsg); return cls(h, dimension) @classmethod - def load_file(cls, filename, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32): + def load_file(cls, filename, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32): if build_params is None: build_params = CagraBuildParams.default() dev_arr = (ctypes.c_int * len(devices))(*devices) errmsg = ctypes.c_char_p() - h = _lib.gpu_cagra_load_file(filename.encode('utf-8'), dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), ctypes.byref(errmsg)) + h = _lib.gpu_cagra_load_file(filename.encode('utf-8'), dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), ctypes.byref(errmsg)) _check_error(errmsg); return cls(h, dimension) def start(self): @@ -442,7 +457,7 @@ def add_chunk(self, chunk, ids=None): id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None errmsg = ctypes.c_char_p(); _lib.gpu_cagra_add_chunk_float(self.handle, chunk.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), len(chunk), id_ptr, ctypes.byref(errmsg)); _check_error(errmsg) def train_quantizer(self, train_data): - train_data = np.ascontiguousarray(train_data, dtype=np.float32) + train_data = np.ascontiguousarray(train_data, dtype=_np_dtype_for(getattr(self, 'btype', Quantization.F32))) errmsg = ctypes.c_char_p(); _lib.gpu_cagra_train_quantizer(self.handle, train_data.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), len(train_data), ctypes.byref(errmsg)); _check_error(errmsg) def set_batch_window(self, window_us): @@ -474,10 +489,10 @@ def load_dir(self, directory, target_mode=DistributionMode.SINGLE_GPU): def search(self, queries, k, search_params=None): if search_params is None: search_params = CagraSearchParams.default() - queries = np.ascontiguousarray(queries, dtype=np.float32) + queries = np.ascontiguousarray(queries, dtype=_np_dtype_for(getattr(self, 'btype', Quantization.F32))) num_q, dim = queries.shape errmsg = ctypes.c_char_p() - res = _lib.gpu_cagra_search_float(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) + res = _lib.gpu_cagra_search_quantize(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) _check_error(errmsg) neighbors = np.zeros((num_q, k), dtype=np.int64) distances = np.zeros((num_q, k), dtype=np.float32) @@ -490,7 +505,7 @@ def search_async(self, queries, k, search_params=None): queries = np.ascontiguousarray(queries, dtype=np.float32) num_q, dim = queries.shape errmsg = ctypes.c_char_p() - job_id = _lib.gpu_cagra_search_float_async(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) + job_id = _lib.gpu_cagra_search_quantize_async(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) _check_error(errmsg) return job_id @@ -522,7 +537,7 @@ def search_with_filter(self, queries, k, preds_json, search_params=None): num_q, dim = queries.shape preds = preds_json.encode('utf-8') if preds_json else None errmsg = ctypes.c_char_p() - res = _lib.gpu_cagra_search_float_with_filter(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, preds, ctypes.byref(errmsg)) + res = _lib.gpu_cagra_search_quantize_with_filter(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, preds, ctypes.byref(errmsg)) _check_error(errmsg) neighbors = np.zeros((num_q, k), dtype=np.int64) distances = np.zeros((num_q, k), dtype=np.float32) @@ -546,31 +561,31 @@ def __init__(self, handle, dimension): self.dimension = dimension @classmethod - def create(cls, dataset, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32, ids=None): + def create(cls, dataset, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32, ids=None): if build_params is None: build_params = IvfFlatBuildParams.default() dataset = np.ascontiguousarray(dataset, dtype=np.float32) count, dim = dataset.shape dev_arr = (ctypes.c_int * len(devices))(*devices) id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None errmsg = ctypes.c_char_p() - h = _lib.gpu_ivf_flat_new(dataset.ctypes.data_as(ctypes.c_void_p), count, dim, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), id_ptr, ctypes.byref(errmsg)) + h = _lib.gpu_ivf_flat_new(dataset.ctypes.data_as(ctypes.c_void_p), count, dim, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), id_ptr, ctypes.byref(errmsg)) _check_error(errmsg); return cls(h, dim) @classmethod - def create_empty(cls, total_count, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32, ids=None): + def create_empty(cls, total_count, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32, ids=None): if build_params is None: build_params = IvfFlatBuildParams.default() dev_arr = (ctypes.c_int * len(devices))(*devices) id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None errmsg = ctypes.c_char_p() - h = _lib.gpu_ivf_flat_new_empty(total_count, dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), id_ptr, ctypes.byref(errmsg)) + h = _lib.gpu_ivf_flat_new_empty(total_count, dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), id_ptr, ctypes.byref(errmsg)) _check_error(errmsg); return cls(h, dimension) @classmethod - def load_file(cls, filename, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32): + def load_file(cls, filename, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32): if build_params is None: build_params = IvfFlatBuildParams.default() dev_arr = (ctypes.c_int * len(devices))(*devices) errmsg = ctypes.c_char_p() - h = _lib.gpu_ivf_flat_load_file(filename.encode('utf-8'), dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), ctypes.byref(errmsg)) + h = _lib.gpu_ivf_flat_load_file(filename.encode('utf-8'), dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), ctypes.byref(errmsg)) _check_error(errmsg); return cls(h, dimension) def start(self): @@ -635,7 +650,7 @@ def search(self, queries, k, search_params=None): queries = np.ascontiguousarray(queries, dtype=np.float32) num_q, dim = queries.shape errmsg = ctypes.c_char_p() - res = _lib.gpu_ivf_flat_search_float(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) + res = _lib.gpu_ivf_flat_search_quantize(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) _check_error(errmsg) neighbors = np.zeros((num_q, k), dtype=np.int64) distances = np.zeros((num_q, k), dtype=np.float32) @@ -648,7 +663,7 @@ def search_async(self, queries, k, search_params=None): queries = np.ascontiguousarray(queries, dtype=np.float32) num_q, dim = queries.shape errmsg = ctypes.c_char_p() - job_id = _lib.gpu_ivf_flat_search_float_async(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) + job_id = _lib.gpu_ivf_flat_search_quantize_async(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) _check_error(errmsg) return job_id @@ -680,7 +695,7 @@ def search_with_filter(self, queries, k, preds_json, search_params=None): num_q, dim = queries.shape preds = preds_json.encode('utf-8') if preds_json else None errmsg = ctypes.c_char_p() - res = _lib.gpu_ivf_flat_search_float_with_filter(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, preds, ctypes.byref(errmsg)) + res = _lib.gpu_ivf_flat_search_quantize_with_filter(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, preds, ctypes.byref(errmsg)) _check_error(errmsg) neighbors = np.zeros((num_q, k), dtype=np.int64) distances = np.zeros((num_q, k), dtype=np.float32) @@ -716,39 +731,39 @@ def __init__(self, handle, dimension=None): self._dimension = dimension @classmethod - def create(cls, dataset, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32, ids=None): + def create(cls, dataset, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32, ids=None): if build_params is None: build_params = IvfPqBuildParams.default() - dataset = np.ascontiguousarray(dataset, dtype=np.float32) + dataset = np.ascontiguousarray(dataset, dtype=_np_dtype_for(btype)) count, dim = dataset.shape dev_arr = (ctypes.c_int * len(devices))(*devices) id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None errmsg = ctypes.c_char_p() - h = _lib.gpu_ivf_pq_new(dataset.ctypes.data_as(ctypes.c_void_p), count, dim, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), id_ptr, ctypes.byref(errmsg)) - _check_error(errmsg); return cls(h, dim) + h = _lib.gpu_ivf_pq_new(dataset.ctypes.data_as(ctypes.c_void_p), count, dim, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), id_ptr, ctypes.byref(errmsg)) + _check_error(errmsg); idx = cls(h, dim); idx.btype = int(btype); return idx @classmethod - def create_empty(cls, total_count, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32, ids=None): + def create_empty(cls, total_count, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32, ids=None): if build_params is None: build_params = IvfPqBuildParams.default() dev_arr = (ctypes.c_int * len(devices))(*devices) id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None errmsg = ctypes.c_char_p() - h = _lib.gpu_ivf_pq_new_empty(total_count, dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), id_ptr, ctypes.byref(errmsg)) + h = _lib.gpu_ivf_pq_new_empty(total_count, dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), id_ptr, ctypes.byref(errmsg)) _check_error(errmsg); return cls(h, dimension) @classmethod - def create_from_data_file(cls, filename, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32): + def create_from_data_file(cls, filename, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32): if build_params is None: build_params = IvfPqBuildParams.default() dev_arr = (ctypes.c_int * len(devices))(*devices) errmsg = ctypes.c_char_p() - h = _lib.gpu_ivf_pq_new_from_data_file(filename.encode('utf-8'), int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), ctypes.byref(errmsg)) + h = _lib.gpu_ivf_pq_new_from_data_file(filename.encode('utf-8'), int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), ctypes.byref(errmsg)) _check_error(errmsg); return cls(h) @classmethod - def load_file(cls, filename, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32): + def load_file(cls, filename, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32): if build_params is None: build_params = IvfPqBuildParams.default() dev_arr = (ctypes.c_int * len(devices))(*devices) errmsg = ctypes.c_char_p() - h = _lib.gpu_ivf_pq_load_file(filename.encode('utf-8'), dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), ctypes.byref(errmsg)) + h = _lib.gpu_ivf_pq_load_file(filename.encode('utf-8'), dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), ctypes.byref(errmsg)) _check_error(errmsg); return cls(h, dimension) def start(self): @@ -778,7 +793,7 @@ def add_chunk(self, chunk, ids=None): errmsg = ctypes.c_char_p(); _lib.gpu_ivf_pq_add_chunk_float(self.handle, chunk.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), len(chunk), id_ptr, ctypes.byref(errmsg)); _check_error(errmsg) def train_quantizer(self, train_data): - train_data = np.ascontiguousarray(train_data, dtype=np.float32) + train_data = np.ascontiguousarray(train_data, dtype=_np_dtype_for(getattr(self, 'btype', Quantization.F32))) errmsg = ctypes.c_char_p(); _lib.gpu_ivf_pq_train_quantizer(self.handle, train_data.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), len(train_data), ctypes.byref(errmsg)); _check_error(errmsg) def set_batch_window(self, window_us): @@ -810,10 +825,10 @@ def load_dir(self, directory, target_mode=DistributionMode.SINGLE_GPU): def search(self, queries, k, search_params=None): if search_params is None: search_params = IvfPqSearchParams.default() - queries = np.ascontiguousarray(queries, dtype=np.float32) + queries = np.ascontiguousarray(queries, dtype=_np_dtype_for(getattr(self, 'btype', Quantization.F32))) num_q, dim = queries.shape errmsg = ctypes.c_char_p() - res = _lib.gpu_ivf_pq_search_float(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) + res = _lib.gpu_ivf_pq_search_quantize(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) _check_error(errmsg) neighbors = np.zeros((num_q, k), dtype=np.int64) distances = np.zeros((num_q, k), dtype=np.float32) @@ -826,7 +841,7 @@ def search_async(self, queries, k, search_params=None): queries = np.ascontiguousarray(queries, dtype=np.float32) num_q, dim = queries.shape errmsg = ctypes.c_char_p() - job_id = _lib.gpu_ivf_pq_search_float_async(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) + job_id = _lib.gpu_ivf_pq_search_quantize_async(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) _check_error(errmsg) return job_id @@ -858,7 +873,7 @@ def search_with_filter(self, queries, k, preds_json, search_params=None): num_q, dim = queries.shape preds = preds_json.encode('utf-8') if preds_json else None errmsg = ctypes.c_char_p() - res = _lib.gpu_ivf_pq_search_float_with_filter(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, preds, ctypes.byref(errmsg)) + res = _lib.gpu_ivf_pq_search_quantize_with_filter(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, preds, ctypes.byref(errmsg)) _check_error(errmsg) neighbors = np.zeros((num_q, k), dtype=np.int64) distances = np.zeros((num_q, k), dtype=np.float32) @@ -915,14 +930,16 @@ def create(cls, dataset, metric=DistanceType.L2Expanded, nthread=4, device_id=0, count, dim = dataset.shape id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None errmsg = ctypes.c_char_p() - h = _lib.gpu_brute_force_new(dataset.ctypes.data_as(ctypes.c_void_p), count, dim, int(metric), nthread, device_id, int(qtype), id_ptr, ctypes.byref(errmsg)) + # btype before qtype; for the python tests base type == storage type. + h = _lib.gpu_brute_force_new(dataset.ctypes.data_as(ctypes.c_void_p), count, dim, int(metric), nthread, device_id, int(qtype), int(qtype), id_ptr, ctypes.byref(errmsg)) _check_error(errmsg); return cls(h, dim) @classmethod def create_empty(cls, total_count, dimension, metric=DistanceType.L2Expanded, nthread=4, device_id=0, qtype=Quantization.F32, ids=None): id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None errmsg = ctypes.c_char_p() - h = _lib.gpu_brute_force_new_empty(total_count, dimension, int(metric), nthread, device_id, int(qtype), id_ptr, ctypes.byref(errmsg)) + # btype before qtype; for the python tests base type == storage type. + h = _lib.gpu_brute_force_new_empty(total_count, dimension, int(metric), nthread, device_id, int(qtype), int(qtype), id_ptr, ctypes.byref(errmsg)) _check_error(errmsg); return cls(h, dimension) def start(self): @@ -932,13 +949,13 @@ def build(self): def add_chunk(self, chunk, ids=None): chunk = np.ascontiguousarray(chunk, dtype=np.float32) id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None - errmsg = ctypes.c_char_p(); _lib.gpu_brute_force_add_chunk_float(self.handle, chunk.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), len(chunk), id_ptr, ctypes.byref(errmsg)); _check_error(errmsg) + errmsg = ctypes.c_char_p(); _lib.gpu_brute_force_add_chunk_quantize(self.handle, chunk.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), len(chunk), id_ptr, ctypes.byref(errmsg)); _check_error(errmsg) def search(self, queries, k): queries = np.ascontiguousarray(queries, dtype=np.float32) num_q, dim = queries.shape errmsg = ctypes.c_char_p() - res_ptr = _lib.gpu_brute_force_search_float(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, ctypes.byref(errmsg)) + res_ptr = _lib.gpu_brute_force_search_quantize(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, ctypes.byref(errmsg)) _check_error(errmsg) neighbors = np.zeros((num_q, k), dtype=np.int64) distances = np.zeros((num_q, k), dtype=np.float32) @@ -949,7 +966,7 @@ def search_async(self, queries, k): queries = np.ascontiguousarray(queries, dtype=np.float32) num_q, dim = queries.shape errmsg = ctypes.c_char_p() - job_id = _lib.gpu_brute_force_search_float_async(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, ctypes.byref(errmsg)) + job_id = _lib.gpu_brute_force_search_quantize_async(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, ctypes.byref(errmsg)) _check_error(errmsg) return job_id diff --git a/cgo/cuvs/python/test/test_cuvs.py b/cgo/cuvs/python/test/test_cuvs.py index b5b1e051eade6..a0f786a0ea2c8 100644 --- a/cgo/cuvs/python/test/test_cuvs.py +++ b/cgo/cuvs/python/test/test_cuvs.py @@ -84,6 +84,35 @@ def test_ivf_pq(self): self.assertEqual(neighbors.shape, (5, self.k)) self.assertEqual(distances.shape, (5, self.k)) + def test_cagra_f16_quantize(self): + # vecf16 BASE quantized to int8/uint8 via the native half-source quantizer + # (btype=F16). Exercises the f16 data path: dataset + query stay half. + ds = np.random.random((self.n_rows, self.dim)).astype(np.float16) + q = ds[:5] + for qt in (cuvs.Quantization.INT8, cuvs.Quantization.UINT8): + index = cuvs.CagraIndex.create(ds, btype=cuvs.Quantization.F16, qtype=qt) + index.start() + index.train_quantizer(ds) # CAGRA from-dataset build does not auto-train the quantizer + index.build() + neighbors, distances = index.search(q, self.k) + self.assertEqual(neighbors.shape, (5, self.k)) + self.assertTrue(np.all(neighbors >= 0)) + self.assertTrue(np.all(neighbors < self.n_rows)) + + def test_ivf_pq_f16_quantize(self): + ds = np.random.random((self.n_rows, self.dim)).astype(np.float16) + q = ds[:5] + bp = cuvs.IvfPqBuildParams(n_lists=32, m=8, bits_per_code=8, add_data_on_build=True, kmeans_trainset_fraction=1.0) + for qt in (cuvs.Quantization.INT8, cuvs.Quantization.UINT8): + index = cuvs.IvfPqIndex.create(ds, build_params=bp, btype=cuvs.Quantization.F16, qtype=qt) + index.start() + index.train_quantizer(ds) + index.build() + neighbors, distances = index.search(q, self.k) + self.assertEqual(neighbors.shape, (5, self.k)) + self.assertTrue(np.all(neighbors >= 0)) + self.assertTrue(np.all(neighbors < self.n_rows)) + def test_kmeans(self): n_clusters = 5 kmeans = cuvs.KMeans(n_clusters=n_clusters, dimension=self.dim) diff --git a/cgo/cuvs/quantize.hpp b/cgo/cuvs/quantize.hpp index 0f5ced5cb5c6a..9667c3ce86116 100644 --- a/cgo/cuvs/quantize.hpp +++ b/cgo/cuvs/quantize.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -27,6 +28,8 @@ #include #include #include +#include +#include #include #include #include @@ -99,22 +102,101 @@ class scalar_quantizer_t { auto out_view = raft::make_device_matrix_view(out_ptr, n_rows, n_cols); cuvs::preprocessing::quantize::scalar::transform(res, *quantizer_, src_view, out_view); } else { - // T is uint8_t, but cuVS transform expects int8_t output + // T is uint8_t. cuVS scalar transform only emits int8 [-128,127]; + // map it to uint8 [0,255] with a MONOTONIC +128 shift, NOT a raw + // cast (raft::copy would value-cast and wrap negatives: -1->255, + // -128->128, scrambling the L2 ordering for signed/zero-centered + // data). The shift is L2-invariant — base and query both pass through + // here, so the constant cancels in (a-b) — so uint8 recall matches int8. auto chunk_device_int8 = raft::make_device_matrix(res, n_rows, n_cols); cuvs::preprocessing::quantize::scalar::transform(res, *quantizer_, src_view, chunk_device_int8.view()); - auto out_view = raft::make_device_matrix_view(out_ptr, n_rows, n_cols); - raft::copy(res, out_view, chunk_device_int8.view()); + raft::linalg::unaryOp( + out_ptr, chunk_device_int8.data_handle(), n_rows * n_cols, + [] __device__(int8_t v) { return static_cast(static_cast(v) + 128); }, + raft::resource::get_cuda_stream(res)); } } else { - // For host pointers, we must use a temporary device buffer for the transform + // For host pointers, transform into a temporary device int8 buffer first. auto tmp_dev = raft::make_device_matrix(res, n_rows, n_cols); cuvs::preprocessing::quantize::scalar::transform(res, *quantizer_, src_view, tmp_dev.view()); - auto out_view = raft::make_host_matrix_view(out_ptr, n_rows, n_cols); - raft::copy(res, out_view, tmp_dev.view()); + if constexpr (std::is_same_v) { + // Monotonic int8->uint8 (+128) on device, then copy to host — see + // the device path above for why a raw cast is wrong. + auto tmp_u8 = raft::make_device_matrix(res, n_rows, n_cols); + raft::linalg::unaryOp( + tmp_u8.data_handle(), tmp_dev.data_handle(), n_rows * n_cols, + [] __device__(int8_t v) { return static_cast(static_cast(v) + 128); }, + raft::resource::get_cuda_stream(res)); + auto out_view = raft::make_host_matrix_view(out_ptr, n_rows, n_cols); + raft::copy(res, out_view, tmp_u8.view()); + } else { + auto out_view = raft::make_host_matrix_view(out_ptr, n_rows, n_cols); + raft::copy(res, out_view, tmp_dev.view()); + } raft::resource::sync_stream(res); } } + /** + * @brief Host (CPU) equivalent of transform(): quantizes a chunk of + * SOURCE-typed (S) elements into 1-byte T entirely on the CPU. + * + * Scalar quantization is a pure per-element affine map from the trained + * [min_, max_] range, so once the quantizer is trained no GPU is needed. + * This is a bit-for-bit port of cuVS' device quantize_op + * (cuvs/preprocessing/quantize/detail/scalar.cuh): the scale/offset are + * computed in `double` (the op's default TempT), the inner clamp uses the + * source-type comparison, ties round via lroundf, and uint8 storage applies + * the same monotonic +128 shift as the device path. Producing identical + * bytes to transform() keeps a CPU-built base consistent with a + * GPU-quantized query at search time. + * + * @tparam T Target storage type (int8_t or uint8_t). + * @param src Source elements, row-major, n_elements long. + * @param out Destination (host), n_elements long. + * @param n_elements Number of scalar elements (rows * dimension). + */ + template + void transform_host(const S* src, T* out, size_t n_elements) const { + if (!quantizer_) throw std::runtime_error("Quantizer not trained"); + static_assert(sizeof(T) == 1, "Quantization target must be 1-byte"); + + // cuVS maps the float interval onto the signed range [-128, 127]; + // uint8 is the same int8 result shifted by +128 (see transform()). + constexpr int q_type_min = std::numeric_limits::min(); // -128 + constexpr int q_type_max = std::numeric_limits::max(); // 127 + + const double dmin = static_cast(quantizer_->min_); + const double dmax = static_cast(quantizer_->max_); + const double scalar = (dmax > dmin) + ? (static_cast(q_type_max - q_type_min) / (dmax - dmin)) + : 1.0; + const double offset = static_cast(q_type_min) - dmin * scalar; + + // fp_lt() compares in the source type's float domain (half is cast to + // float; float compares natively) — replicate with a float compare. + const float fmin = static_cast(quantizer_->min_); + const float fmax = static_cast(quantizer_->max_); + + for (size_t i = 0; i < n_elements; ++i) { + const float xf = static_cast(src[i]); + int8_t q; + if (!(fmin < xf)) { + q = static_cast(q_type_min); + } else if (!(xf < fmax)) { + q = static_cast(q_type_max); + } else { + q = static_cast( + std::lroundf(static_cast(scalar * static_cast(src[i]) + offset))); + } + if constexpr (std::is_same_v) { + out[i] = static_cast(static_cast(q) + 128); + } else { + out[i] = static_cast(q); + } + } + } + bool is_trained() const { return quantizer_ != nullptr; } void reset() { quantizer_.reset(); } @@ -242,10 +324,18 @@ void load_matrix_chunked_ptr(const raft::resources& res, const std::string& file if constexpr (DoQuantize) { int64_t n_train = std::min(n_rows, static_cast(500)); std::vector train_host(n_train * n_cols); - std::streamsize train_wanted = static_cast(train_host.size() * sizeof(S)); - file.read(reinterpret_cast(train_host.data()), train_wanted); - if (file.gcount() != train_wanted) { - throw std::runtime_error("Truncated training-set read from: " + filename); + // Strided sample across ALL n_rows (not the first n_train contiguous + // rows) so the scalar quantizer learns the true [min,max] range even + // when the file is sorted/clustered — otherwise higher-magnitude rows + // past the prefix saturate to the storage extreme and recall collapses. + const int64_t stride = n_rows / n_train; // >= 1 since n_train <= n_rows + const std::streamsize row_bytes = static_cast(n_cols) * sizeof(S); + for (int64_t j = 0; j < n_train; ++j) { + file.seekg(sizeof(file_header_t) + static_cast(j) * stride * row_bytes); + file.read(reinterpret_cast(train_host.data() + j * n_cols), row_bytes); + if (file.gcount() != row_bytes) { + throw std::runtime_error("Truncated training-set read from: " + filename); + } } auto train_device = raft::make_device_matrix(res, n_train, n_cols); raft::copy(train_device.data_handle(), train_host.data(), train_host.size(), raft::resource::get_cuda_stream(res)); diff --git a/cgo/cuvs/test/batching_test.cu b/cgo/cuvs/test/batching_test.cu index f3bfbcc635bb2..e90f49949765f 100644 --- a/cgo/cuvs/test/batching_test.cu +++ b/cgo/cuvs/test/batching_test.cu @@ -34,7 +34,7 @@ TEST(DynamicBatchingTest, CagraConcurrentSearch) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 8, DistributionMode_SINGLE_GPU); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 8, DistributionMode_SINGLE_GPU); index.set_batch_window(100); index.start(); @@ -69,7 +69,7 @@ TEST(DynamicBatchingTest, IvfFlatConcurrentSearch) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 10; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 8, DistributionMode_SINGLE_GPU); + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 8, DistributionMode_SINGLE_GPU); index.set_batch_window(100); index.start(); @@ -105,7 +105,7 @@ TEST(DynamicBatchingTest, IvfPqConcurrentSearch) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 10; bp.m = 8; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 8, DistributionMode_SINGLE_GPU); + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 8, DistributionMode_SINGLE_GPU); index.set_batch_window(100); index.start(); diff --git a/cgo/cuvs/test/benchmark_cuvs.cu b/cgo/cuvs/test/benchmark_cuvs.cu index cd36d81e4d0d5..d6c83b0956a7a 100644 --- a/cgo/cuvs/test/benchmark_cuvs.cu +++ b/cgo/cuvs/test/benchmark_cuvs.cu @@ -98,7 +98,14 @@ template void run_benchmark(const std::string& index_name, distribution_mode_t mode, IndexT& index, const std::vector& recall_queries, const std::vector& recall_expected_ids, const benchmark_config_t& cfg, const SearchParamsT& sp) { - + + // float-input search dispatch: cagra/ivf_flat/ivf_pq and brute force all + // expose the base-typed search_quantize (B==float here; identity quantize + // when base == storage). + auto bench_search = [&](const float* q, uint64_t nq) { + return index.search_quantize(q, nq, cfg.dimension, cfg.limit, sp); + }; + for (int64_t window_us : {(int64_t)0, (int64_t)100}) { index.set_batch_window(window_us); @@ -108,7 +115,7 @@ void run_benchmark(const std::string& index_name, distribution_mode_t mode, // Warmup for (int i = 0; i < 5; ++i) { - index.search_float(queries.data(), 1, cfg.dimension, cfg.limit, sp); + bench_search(queries.data(), 1); } std::atomic total_completed{0}; @@ -119,7 +126,7 @@ void run_benchmark(const std::string& index_name, distribution_mode_t mode, for (uint32_t t = 0; t < cfg.n_threads; ++t) { threads.emplace_back([&, t, q_per_thread]() { for (uint32_t i = 0; i < q_per_thread; ++i) { - index.search_float(queries.data() + (t * q_per_thread + i) * cfg.dimension, 1, cfg.dimension, cfg.limit, sp); + bench_search(queries.data() + (t * q_per_thread + i) * cfg.dimension, 1); total_completed++; } }); @@ -132,7 +139,7 @@ void run_benchmark(const std::string& index_name, distribution_mode_t mode, double qps = total_completed.load() / diff.count(); // Self-recall - auto res = index.search_float(recall_queries.data(), cfg.n_queries, cfg.dimension, cfg.limit, sp); + auto res = bench_search(recall_queries.data(), cfg.n_queries); double recall = calculate_recall(res.neighbors, recall_expected_ids, cfg.n_queries, cfg.limit); std::cout << std::left << std::setw(45) << full_name @@ -143,7 +150,7 @@ void run_benchmark(const std::string& index_name, distribution_mode_t mode, template void benchmark_all_indices(const std::vector& dataset, const benchmark_config_t& cfg) { - auto converted = convert_dataset(dataset, cfg.n_vectors, cfg.dimension); + [[maybe_unused]] auto converted = convert_dataset(dataset, cfg.n_vectors, cfg.dimension); // Prepare recall queries from 4 different shards std::vector recall_queries; @@ -174,14 +181,28 @@ void benchmark_all_indices(const std::vector& dataset, const benchmark_co if (mode != DistributionMode_SINGLE_GPU && active_devices.size() < 2) continue; - gpu_cagra_t index(converted.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, active_devices, cfg.n_threads, mode); - index.start(); + std::unique_ptr> idx; + if constexpr (sizeof(T) == 1) { + // 1-byte storage: feed the ORIGINAL floats through the base-typed + // quantize path so the library trains the scalar quantizer and + // quantizes the dataset with the SAME affine map used for the + // float32 queries at search time. Pre-quantizing on the host and + // using the const T* ctor leaves the quantizer untrained, which + // aborts search_quantize() with "Quantizer not trained". + idx = std::make_unique>(cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, active_devices, cfg.n_threads, mode); + idx->start(); + idx->add_chunk_quantize(dataset.data(), cfg.n_vectors); + } else { + idx = std::make_unique>(converted.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, active_devices, cfg.n_threads, mode); + idx->start(); + } + auto& index = *idx; index.build(); - + cagra_search_params_t sp = cagra_search_params_default(); sp.itopk_size = 128; sp.search_width = 1; - run_benchmark, cagra_search_params_t, T>("Cagra", mode, index, recall_queries, recall_expected_ids, cfg, sp); + run_benchmark, cagra_search_params_t, T>("Cagra", mode, index, recall_queries, recall_expected_ids, cfg, sp); index.destroy(); cudaDeviceSynchronize(); } @@ -198,13 +219,22 @@ void benchmark_all_indices(const std::vector& dataset, const benchmark_co if (mode != DistributionMode_SINGLE_GPU && active_devices.size() < 2) continue; - gpu_ivf_flat_t index(converted.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, active_devices, cfg.n_threads, mode); - index.start(); + std::unique_ptr> idx; + if constexpr (sizeof(T) == 1) { + // See CAGRA block: train the quantizer via the base-typed path. + idx = std::make_unique>(cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, active_devices, cfg.n_threads, mode); + idx->start(); + idx->add_chunk_quantize(dataset.data(), cfg.n_vectors); + } else { + idx = std::make_unique>(converted.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, active_devices, cfg.n_threads, mode); + idx->start(); + } + auto& index = *idx; index.build(); ivf_flat_search_params_t sp = ivf_flat_search_params_default(); sp.n_probes = 64; - run_benchmark, ivf_flat_search_params_t, T>("IvfFlat", mode, index, recall_queries, recall_expected_ids, cfg, sp); + run_benchmark, ivf_flat_search_params_t, T>("IvfFlat", mode, index, recall_queries, recall_expected_ids, cfg, sp); index.destroy(); } } @@ -221,13 +251,22 @@ void benchmark_all_indices(const std::vector& dataset, const benchmark_co if (mode != DistributionMode_SINGLE_GPU && active_devices.size() < 2) continue; - gpu_ivf_pq_t index(converted.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, active_devices, cfg.n_threads, mode); - index.start(); + std::unique_ptr> idx; + if constexpr (sizeof(T) == 1) { + // See CAGRA block: train the quantizer via the base-typed path. + idx = std::make_unique>(cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, active_devices, cfg.n_threads, mode); + idx->start(); + idx->add_chunk_quantize(dataset.data(), cfg.n_vectors); + } else { + idx = std::make_unique>(converted.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, active_devices, cfg.n_threads, mode); + idx->start(); + } + auto& index = *idx; index.build(); ivf_pq_search_params_t sp = ivf_pq_search_params_default(); sp.n_probes = 64; - run_benchmark, ivf_pq_search_params_t, T>("IvfPq", mode, index, recall_queries, recall_expected_ids, cfg, sp); + run_benchmark, ivf_pq_search_params_t, T>("IvfPq", mode, index, recall_queries, recall_expected_ids, cfg, sp); index.destroy(); } } @@ -237,12 +276,14 @@ void benchmark_all_indices(const std::vector& dataset, const benchmark_co distribution_mode_t mode = DistributionMode_SINGLE_GPU; std::vector active_devices = {cfg.devices[0]}; - gpu_brute_force_t index(converted.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, cfg.n_threads, active_devices[0]); + // Base type float (the benchmark queries with float32); storage T. This + // matches search_quantize(const B*=const float*) used in run_benchmark. + gpu_brute_force_t index(converted.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, cfg.n_threads, active_devices[0]); index.start(); index.build(); brute_force_search_params_t sp = brute_force_search_params_default(); - run_benchmark, brute_force_search_params_t, T>("BruteForce", mode, index, recall_queries, recall_expected_ids, cfg, sp); + run_benchmark, brute_force_search_params_t, T>("BruteForce", mode, index, recall_queries, recall_expected_ids, cfg, sp); index.destroy(); } } diff --git a/cgo/cuvs/test/benchmark_filter.cu b/cgo/cuvs/test/benchmark_filter.cu index 7739aff2ee841..735bdb51a4091 100644 --- a/cgo/cuvs/test/benchmark_filter.cu +++ b/cgo/cuvs/test/benchmark_filter.cu @@ -120,7 +120,7 @@ std::pair run_throughput(Index& index, pool.emplace_back([&, t, per_thread]() { for (uint32_t i = 0; i < per_thread; ++i) { auto t0 = std::chrono::high_resolution_clock::now(); - (void)index.search_float_with_filter( + (void)index.search_quantize_with_filter( queries.data() + (t * per_thread + i) * cfg.dimension, 1, cfg.dimension, cfg.limit, sp, preds_json); auto t1 = std::chrono::high_resolution_clock::now(); @@ -158,7 +158,7 @@ void sweep_selectivities(const std::string& tag, Index& index, const SP& sp, index.set_batch_window(window_us); for (uint32_t w = 0; w < cfg.warmup; ++w) { - (void)index.search_float_with_filter(throughput_queries.data(), 1, + (void)index.search_quantize_with_filter(throughput_queries.data(), 1, cfg.dimension, cfg.limit, sp, ""); } @@ -173,7 +173,7 @@ void sweep_selectivities(const std::string& tag, Index& index, const SP& sp, double qps = qt.first; double lat_us = qt.second; - auto res = index.search_float_with_filter( + auto res = index.search_quantize_with_filter( recall_queries.data(), cfg.n_queries, cfg.dimension, cfg.limit, sp, preds); auto r = self_recall(res.neighbors, recall_expected_ids, cats, k, cfg.n_queries, cfg.limit); @@ -241,7 +241,7 @@ int main() { { cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), cfg.n_vectors, cfg.dimension, + gpu_cagra_t index(dataset.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, devices, cfg.n_threads, DistributionMode_SINGLE_GPU); index.start(); @@ -261,7 +261,7 @@ int main() { { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 1024; - gpu_ivf_flat_t index(dataset.data(), cfg.n_vectors, cfg.dimension, + gpu_ivf_flat_t index(dataset.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, devices, cfg.n_threads, DistributionMode_SINGLE_GPU); index.start(); @@ -281,7 +281,7 @@ int main() { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 1024; bp.m = 64; - gpu_ivf_pq_t index(dataset.data(), cfg.n_vectors, cfg.dimension, + gpu_ivf_pq_t index(dataset.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, devices, cfg.n_threads, DistributionMode_SINGLE_GPU); index.start(); diff --git a/cgo/cuvs/test/brute_force_test.cu b/cgo/cuvs/test/brute_force_test.cu index 26c8f361a6118..cf90fe3ff7aa2 100644 --- a/cgo/cuvs/test/brute_force_test.cu +++ b/cgo/cuvs/test/brute_force_test.cu @@ -39,7 +39,7 @@ TEST(GpuBruteForceTest, BasicLoadAndSearch) { const uint64_t count = 2; std::vector dataset = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; - gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); index.build(); @@ -63,7 +63,7 @@ TEST(GpuBruteForceTest, BasicLoadAndSearchWithIds) { ids[i] = (int64_t)(i + 3000); } - gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0, ids.data()); + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0, ids.data()); index.start(); index.build(); @@ -94,7 +94,7 @@ TEST(GpuBruteForceTest, ParallelAddChunkWithOffset) { ids2[i] = (int64_t)(i + count_per_chunk); } - gpu_brute_force_t index(total_count, dimension, DistanceType_L2Expanded, 1, 0); + gpu_brute_force_t index(total_count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); #include @@ -113,6 +113,35 @@ TEST(GpuBruteForceTest, ParallelAddChunkWithOffset) { index.destroy(); } +// f16 overflow path, NATIVE half add: empty gpu_brute_force_t -> +// add_chunk([]half) -> build -> native half search. This is the path +// IvfpqSearch.buildOverflow should use for a vecf16 base (feed native half, not +// add_chunk_float's f32->half cast). Confirms the native half overflow works. +TEST(GpuBruteForceTest, HalfEmptyAddChunkSearch) { + const uint32_t dimension = 8; + const uint64_t count = 50; + std::vector hdata(count * dimension); + std::vector ids(count); + for (size_t i = 0; i < count; ++i) { + for (size_t j = 0; j < dimension; ++j) hdata[i * dimension + j] = __float2half((float)(i + 1)); + ids[i] = (int64_t)(i + 1); + } + + gpu_brute_force_t index(count, dimension, DistanceType_L2Expanded, 1, 0); + index.start(); + index.add_chunk(hdata.data(), count, -1, ids.data()); + index.build(); + + std::vector qh(dimension); + for (uint32_t j = 0; j < dimension; ++j) qh[j] = __float2half(25.0f); + auto result = index.search(qh.data(), 1, dimension, 3, brute_force_search_params_default()); + + ASSERT_EQ(result.neighbors.size(), (size_t)3); + ASSERT_EQ(result.neighbors[0], (int64_t)25); + + index.destroy(); +} + TEST(GpuBruteForceTest, SearchWithMultipleQueries) { const uint32_t dimension = 4; const uint64_t count = 4; @@ -123,7 +152,7 @@ TEST(GpuBruteForceTest, SearchWithMultipleQueries) { 0.0, 0.0, 0.0, 1.0 // ID 3 }; - gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); index.build(); @@ -146,7 +175,7 @@ TEST(GpuBruteForceTest, SearchWithFloat16) { std::vector f_dataset = {1.0, 1.0, 2.0, 2.0}; std::vector h_dataset = float_to_half(f_dataset); - gpu_brute_force_t index(h_dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); + gpu_brute_force_t index(h_dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); index.build(); @@ -169,7 +198,7 @@ TEST(GpuBruteForceTest, SearchWithInnerProduct) { 0.0, 1.0 }; - gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_InnerProduct, 1, 0); + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_InnerProduct, 1, 0); index.start(); index.build(); @@ -196,7 +225,7 @@ TEST(GpuBruteForceTest, EmptyDataset) { const uint32_t dimension = 128; const uint64_t count = 0; - gpu_brute_force_t index(nullptr, count, dimension, DistanceType_L2Expanded, 1, 0); + gpu_brute_force_t index(nullptr, count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); index.build(); @@ -229,7 +258,7 @@ TEST(GpuBruteForceTest, LargeLimit) { const uint64_t count = 5; std::vector dataset(count * dimension, 1.0); - gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); index.build(); @@ -263,7 +292,7 @@ TEST(GpuBruteForceTest, LargeLimitWithExplicitIds) { std::vector dataset(count * dimension, 1.0); std::vector ids = {1000, 1001, 1002, 1003, 1004}; - gpu_brute_force_t index(dataset.data(), count, dimension, + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0, ids.data()); index.start(); index.build(); @@ -297,7 +326,7 @@ TEST(GpuBruteForceTest, SoftDeleteSearch) { 7.0, 8.0, 9.0 // ID 2 }; - gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); index.build(); @@ -327,7 +356,7 @@ TEST(GpuBruteForceTest, SoftDeleteWithCustomIds) { std::vector dataset = {10, 10, 20, 20, 30, 30}; std::vector ids = {100, 200, 300}; - gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0, ids.data()); + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0, ids.data()); index.start(); index.build(); @@ -357,7 +386,7 @@ TEST(CuvsWorkerTest, BruteForceSearch) { std::vector dataset(count * dimension); for (size_t i = 0; i < dataset.size(); ++i) dataset[i] = (float)rand() / RAND_MAX; - gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); index.build(); @@ -382,7 +411,7 @@ TEST(CuvsWorkerTest, ConcurrentSearches) { } } - gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 4, 0); + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 4, 0); index.start(); index.build(); @@ -414,7 +443,7 @@ TEST(GpuBruteForceTest, KExceedsIndexSizeClampsAndPads) { } } - gpu_brute_force_t index(dataset.data(), count, dimension, + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); index.build(); @@ -457,7 +486,7 @@ TEST(GpuBruteForceTest, MultiQueryKExceedsIndexSize) { } } - gpu_brute_force_t index(dataset.data(), count, dimension, + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); index.build(); @@ -490,3 +519,10 @@ TEST(GpuBruteForceTest, MultiQueryKExceedsIndexSize) { index.destroy(); } + +// NOTE: cuVS brute force does NOT support int8_t/uint8_t. cuvs::neighbors::brute_force::search +// only provides index and index overloads (verified: compiling +// gpu_brute_force_t/.search fails with "no matching search overload"). The +// header's "Supported T: ... int8_t, uint8_t" claim does not hold for search. For direct +// narrow-base ivfpq/cagra, the int8/uint8 overflow tier therefore uses the pure-Go brute force +// (pkg/vectorindex/brute_force, native int8/uint8 kernels), NOT a cuVS C++ brute force. diff --git a/cgo/cuvs/test/cagra_test.cu b/cgo/cuvs/test/cagra_test.cu index e32c3dba0265f..d032526f72678 100644 --- a/cgo/cuvs/test/cagra_test.cu +++ b/cgo/cuvs/test/cagra_test.cu @@ -21,9 +21,100 @@ #include #include #include +#include using namespace matrixone; +// Native half (f16) build + search — validates the direct vecf16-base path +// (gpu_cagra_t native add_chunk/search, no quantizer). Linking this proves +// cuVS supports cagra over half. +TEST(GpuCagraTest, BasicLoadAndSearchHalf) { + const uint32_t dimension = 16; + const uint64_t count = 1000; + std::vector dataset(count * dimension); + std::vector ids(count); + for (size_t i = 0; i < count; ++i) { + for (size_t j = 0; j < dimension; ++j) + dataset[i * dimension + j] = __float2half((float)rand() / RAND_MAX); + ids[i] = (int64_t)(i + 1000); + } + + std::vector devices = {0}; + cagra_build_params_t bp = cagra_build_params_default(); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); + index.start(); + index.build(); + + std::vector queries(dataset.begin(), dataset.begin() + dimension); + cagra_search_params_t sp = cagra_search_params_default(); + auto result = index.search(queries.data(), 1, dimension, 5, sp); + + ASSERT_EQ(result.neighbors.size(), (size_t)5); + ASSERT_EQ(result.neighbors[0], 1000LL); + + index.destroy(); +} + +// vecf16 base -> int8/uint8 storage via the native B(half)-source quantizer for +// CAGRA (gpu_cagra_t + add_chunk_quantize). Mirrors the ivf_pq +// HalfQuantizeToInt8Build coverage, which cagra previously lacked entirely. +// Verifies: train the half-source quantizer on the buffered vecf16 sample, +// transform half->T, build a CAGRA graph over the quantized codes, and search +// it (both with a native-T query and with a half query quantized via +// quantize_query). No f32 detour. +namespace { +template +void run_cagra_half_quantize_build(const char* label) { + TEST_LOG("CAGRA half-quantize build/search: " << label); + const uint32_t dimension = 16; + const uint64_t count = 2000; + std::vector dataset(count * dimension); + std::vector ids(count); + for (size_t i = 0; i < count; ++i) { + for (size_t j = 0; j < dimension; ++j) + dataset[i * dimension + j] = __float2half((float)(rand() % 256) / 255.0f); + ids[i] = (int64_t)(i + 5000); + } + + std::vector devices = {0}; + cagra_build_params_t bp = cagra_build_params_default(); + gpu_cagra_t index(count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + index.start(); + index.add_chunk_quantize(dataset.data(), count, -1, ids.data()); + index.build(); + + cagra_search_params_t sp = cagra_search_params_default(); + + // 1) search with a native-T query (raw storage codes). + std::vector qnative(dimension, 0); + auto r1 = index.search(qnative.data(), 1, dimension, 5, sp); + ASSERT_EQ(r1.neighbors.size(), (size_t)5); + for (auto n : r1.neighbors) { + ASSERT_GE(n, (int64_t)5000); + ASSERT_LT(n, (int64_t)(5000 + count)); + } + + // 2) search with a half query quantized through the half-source quantizer + // (the production path) — the nearest neighbor of base[0] must be itself. + std::vector qhalf(dataset.begin(), dataset.begin() + dimension); + std::vector qcodes(dimension); + index.quantize_query(qhalf.data(), 1, qcodes.data()); + auto r2 = index.search(qcodes.data(), 1, dimension, 5, sp); + ASSERT_EQ(r2.neighbors.size(), (size_t)5); + ASSERT_EQ(r2.neighbors[0], (int64_t)5000); + + index.destroy(); +} +} // namespace + +TEST(GpuCagraTest, HalfQuantizeToInt8Build) { + run_cagra_half_quantize_build("f16->int8"); +} + +TEST(GpuCagraTest, HalfQuantizeToUint8Build) { + run_cagra_half_quantize_build("f16->uint8"); +} + TEST(GpuCagraTest, BasicLoadAndSearch) { const uint32_t dimension = 16; const uint64_t count = 1000; @@ -32,7 +123,7 @@ TEST(GpuCagraTest, BasicLoadAndSearch) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); @@ -58,7 +149,7 @@ TEST(GpuCagraTest, BasicLoadAndSearchWithIds) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); index.start(); index.build(); @@ -93,7 +184,7 @@ TEST(GpuCagraTest, ParallelAddChunkWithOffset) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); // Pre-allocate with total_count - gpu_cagra_t index(total_count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_cagra_t index(total_count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); // Add chunks in parallel threads @@ -128,7 +219,7 @@ TEST(GpuCagraTest, SaveAndLoadFromFile) { // 1. Build and Save { cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); index.start(); index.build(); index.save(filename); @@ -138,7 +229,7 @@ TEST(GpuCagraTest, SaveAndLoadFromFile) { // 2. Load and Search { cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(filename, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_cagra_t index(filename, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.load(filename); @@ -169,7 +260,7 @@ TEST(GpuCagraTest, ReplicatedModeSimulation) { gpu_get_device_list(devices.data(), dev_count); cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED); index.start(); index.build(); std::vector queries(dataset.begin(), dataset.begin() + dimension); @@ -198,7 +289,7 @@ TEST(GpuCagraTest, ManualShardedSearch) { gpu_get_device_list(devices.data(), dev_count); cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); index.start(); index.build(); @@ -230,7 +321,7 @@ TEST(GpuCagraTest, ManualShardedSearchWithIds) { gpu_get_device_list(devices.data(), dev_count); cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, ids.data()); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, ids.data()); index.start(); index.build(); @@ -260,7 +351,7 @@ TEST(GpuCagraTest, SoftDeleteSearch) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); @@ -303,7 +394,7 @@ TEST(GpuCagraTest, SoftDeleteWithCustomIds) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); index.start(); index.build(); @@ -342,7 +433,7 @@ TEST(GpuCagraTest, FilteredSearchIncludesOnlyAllowedCategories) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -393,7 +484,7 @@ TEST(GpuCagraTest, FilteredSearchCombinesWithDeleteBitset) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -433,7 +524,7 @@ TEST(GpuCagraTest, FilteredSearchEmptyPredsMatchesUnfiltered) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -472,7 +563,7 @@ TEST(GpuCagraTest, ExtendReplicatedWithHostIds) { } cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), n_base, dimension, + gpu_cagra_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED, base_ids.data()); index.start(); @@ -523,7 +614,7 @@ TEST(GpuCagraTest, ExtendShardedThrows) { for (size_t i = 0; i < dataset.size(); ++i) dataset[i] = (float)rand() / RAND_MAX; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), n_base, dimension, + gpu_cagra_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); index.start(); @@ -552,7 +643,7 @@ TEST(GpuCagraTest, BuildParamsTooLargeForShardThrows) { for (size_t i = 0; i < dataset.size(); ++i) dataset[i] = (float)rand() / RAND_MAX; cagra_build_params_t bp = cagra_build_params_default(); // intermediate=128, graph=64 - gpu_cagra_t index(dataset.data(), n_base, dimension, + gpu_cagra_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); index.start(); @@ -572,7 +663,7 @@ TEST(GpuCagraTest, ExtendWithoutHostIds) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), n_base, dimension, + gpu_cagra_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -615,7 +706,7 @@ TEST(GpuCagraTest, ExtendWithHostIds) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), n_base, dimension, + gpu_cagra_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, base_ids.data()); index.start(); @@ -663,7 +754,7 @@ TEST(GpuCagraTest, KExceedsIndexSizeClampsAndPads) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); @@ -708,7 +799,7 @@ TEST(GpuCagraTest, MultiQueryKExceedsIndexSize) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); diff --git a/cgo/cuvs/test/filter_test.cu b/cgo/cuvs/test/filter_test.cu index a93e416928be5..a98b5cf1323b6 100644 --- a/cgo/cuvs/test/filter_test.cu +++ b/cgo/cuvs/test/filter_test.cu @@ -613,7 +613,7 @@ namespace { // Minimal derived index used as a stand-in for the real index types. We never // call start()/build()/search() — only the filter ingest + persistence methods. -struct test_index_t : public gpu_index_base_t { +struct test_index_t : public gpu_index_base_t { test_index_t() { // Populate the fields write_manifest reads so the file is valid JSON. this->dimension = 4; @@ -630,7 +630,7 @@ std::string make_tmp_dir(const std::string& tag) { // Best-effort cleanup from prior runs. std::string rm = "rm -rf " + path; ::system(rm.c_str()); - gpu_index_base_t::ensure_dir(path); + gpu_index_base_t::ensure_dir(path); return path; } diff --git a/cgo/cuvs/test/ivf_flat_test.cu b/cgo/cuvs/test/ivf_flat_test.cu index 785001dda9507..52a55e16f779f 100644 --- a/cgo/cuvs/test/ivf_flat_test.cu +++ b/cgo/cuvs/test/ivf_flat_test.cu @@ -37,7 +37,7 @@ TEST(GpuIvfFlatTest, BasicLoadSearchAndCenters) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 2; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); @@ -71,7 +71,7 @@ TEST(GpuIvfFlatTest, BasicLoadAndSearchWithIds) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 100; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); index.start(); index.build(); @@ -106,7 +106,7 @@ TEST(GpuIvfFlatTest, ParallelAddChunkWithOffset) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 100; - gpu_ivf_flat_t index(total_count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_flat_t index(total_count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); std::thread t1([&]() { index.add_chunk(chunk1.data(), count_per_chunk, 0, ids1.data()); }); @@ -136,7 +136,7 @@ TEST(GpuIvfFlatTest, SaveAndLoadFromFile) { { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 2; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); index.save(filename); @@ -148,7 +148,7 @@ TEST(GpuIvfFlatTest, SaveAndLoadFromFile) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 2; // Construct without loading immediately - gpu_ivf_flat_t index(filename, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_flat_t index(filename, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); // Start worker first index.load(filename); // Then load explicitly @@ -180,7 +180,7 @@ TEST(GpuIvfFlatTest, ReplicatedModeSimulation) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 10; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED); + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED); index.start(); index.build(); @@ -207,7 +207,7 @@ TEST(GpuIvfFlatTest, ReplicatedLoadSearch) { { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 10; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, single_device, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, single_device, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); index.save(filename); @@ -224,7 +224,7 @@ TEST(GpuIvfFlatTest, ReplicatedLoadSearch) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 10; - gpu_ivf_flat_t index(filename, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED); + gpu_ivf_flat_t index(filename, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED); index.start(); index.load(filename); @@ -251,7 +251,7 @@ TEST(GpuIvfFlatTest, SetGetQuantizer) { bp.n_lists = 5; std::vector devices = {0}; - gpu_ivf_flat_t index(count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_flat_t index(count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); float min = -1.5f; float max = 2.5f; @@ -283,7 +283,7 @@ TEST(GpuIvfFlatTest, ManualShardedSearch) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 50; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); index.start(); index.build(); @@ -316,7 +316,7 @@ TEST(GpuIvfFlatTest, ManualShardedSearchWithIds) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 50; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, ids.data()); + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, ids.data()); index.start(); index.build(); @@ -349,7 +349,7 @@ TEST(GpuIvfFlatTest, SimulatedReplicatedBuildSearch) { std::vector sim2 = {0, 0}; // 2 logical GPUs on physical device 0 ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 4; - gpu_ivf_flat_t index(ds.data(), count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED, ids.data()); + gpu_ivf_flat_t index(ds.data(), count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED, ids.data()); index.start(); index.build(); ASSERT_TRUE(index.info().find("\"ranks\": 2") != std::string::npos); // 2 replicas coexist @@ -373,7 +373,7 @@ TEST(GpuIvfFlatTest, SimulatedShardedBuildSearch) { std::vector sim2 = {0, 0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 4; - gpu_ivf_flat_t index(ds.data(), count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_SHARDED, ids.data()); + gpu_ivf_flat_t index(ds.data(), count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_SHARDED, ids.data()); index.start(); index.build(); ASSERT_TRUE(index.info().find("\"ranks\": 2") != std::string::npos); // 2 shards coexist @@ -405,7 +405,7 @@ TEST(GpuIvfFlatTest, SimulatedShardedDeleteSearch) { std::vector sim2 = {0, 0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 4; - gpu_ivf_flat_t index(ds.data(), count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_SHARDED, ids.data()); + gpu_ivf_flat_t index(ds.data(), count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_SHARDED, ids.data()); index.start(); index.build(); ASSERT_TRUE(index.info().find("\"ranks\": 2") != std::string::npos); @@ -453,7 +453,7 @@ TEST(GpuIvfFlatTest, SimulatedReplicatedExtend) { std::vector sim2 = {0, 0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 4; - gpu_ivf_flat_t index(ds.data(), base, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED, ids.data()); + gpu_ivf_flat_t index(ds.data(), base, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED, ids.data()); index.start(); index.build(); ASSERT_TRUE(index.info().find("\"ranks\": 2") != std::string::npos); @@ -484,7 +484,7 @@ TEST(GpuIvfFlatTest, SimulatedSaveLoadAcrossModes) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 4; ivf_flat_search_params_t sp = ivf_flat_search_params_default(); sp.n_probes = 4; - auto probe = [&](gpu_ivf_flat_t& idx, const std::vector& data, const std::vector rows) { + auto probe = [&](gpu_ivf_flat_t& idx, const std::vector& data, const std::vector rows) { for (int r : rows) { std::vector q(data.begin()+r*dim, data.begin()+(r+1)*dim); auto res = idx.search(q.data(), 1, dim, 1, sp); @@ -497,19 +497,19 @@ TEST(GpuIvfFlatTest, SimulatedSaveLoadAcrossModes) { std::string dirR = "/tmp/mo_sim_ivf_flat_rep"; system(("rm -rf " + dirR).c_str()); { - gpu_ivf_flat_t idx(ds.data(), count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED, ids.data()); + gpu_ivf_flat_t idx(ds.data(), count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED, ids.data()); idx.start(); idx.build(); ASSERT_TRUE(idx.info().find("\"ranks\": 2") != std::string::npos); idx.save_dir(dirR); idx.destroy(); } { - gpu_ivf_flat_t idx(count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED); + gpu_ivf_flat_t idx(count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED); idx.start(); idx.load_dir(dirR, DistributionMode_REPLICATED); ASSERT_TRUE(idx.info().find("\"ranks\": 2") != std::string::npos); probe(idx, ds, {0, 9, 15}); idx.destroy(); } { - gpu_ivf_flat_t idx(count, dim, DistanceType_L2Expanded, bp, one, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_flat_t idx(count, dim, DistanceType_L2Expanded, bp, one, 1, DistributionMode_SINGLE_GPU); idx.start(); idx.load_dir(dirR, DistributionMode_SINGLE_GPU); probe(idx, ds, {0, 9, 15}); idx.destroy(); } @@ -518,11 +518,11 @@ TEST(GpuIvfFlatTest, SimulatedSaveLoadAcrossModes) { std::string dirS = "/tmp/mo_sim_ivf_flat_single"; system(("rm -rf " + dirS).c_str()); { - gpu_ivf_flat_t idx(ds.data(), count, dim, DistanceType_L2Expanded, bp, one, 1, DistributionMode_SINGLE_GPU, ids.data()); + gpu_ivf_flat_t idx(ds.data(), count, dim, DistanceType_L2Expanded, bp, one, 1, DistributionMode_SINGLE_GPU, ids.data()); idx.start(); idx.build(); idx.save_dir(dirS); idx.destroy(); } { - gpu_ivf_flat_t idx(count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED); + gpu_ivf_flat_t idx(count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED); idx.start(); idx.load_dir(dirS, DistributionMode_REPLICATED); ASSERT_TRUE(idx.info().find("\"ranks\": 2") != std::string::npos); probe(idx, ds, {0, 9, 15}); idx.destroy(); @@ -535,13 +535,13 @@ TEST(GpuIvfFlatTest, SimulatedSaveLoadAcrossModes) { std::string dirSh = "/tmp/mo_sim_ivf_flat_shard"; system(("rm -rf " + dirSh).c_str()); { - gpu_ivf_flat_t idx(sds.data(), scount, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_SHARDED, sids.data()); + gpu_ivf_flat_t idx(sds.data(), scount, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_SHARDED, sids.data()); idx.start(); idx.build(); ASSERT_TRUE(idx.info().find("\"ranks\": 2") != std::string::npos); idx.save_dir(dirSh); idx.destroy(); } { - gpu_ivf_flat_t idx(scount, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_SHARDED); + gpu_ivf_flat_t idx(scount, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_SHARDED); idx.start(); idx.load_dir(dirSh, DistributionMode_SHARDED); ASSERT_TRUE(idx.info().find("\"ranks\": 2") != std::string::npos); probe(idx, sds, {3, 40, 63}); idx.destroy(); @@ -563,7 +563,7 @@ TEST(GpuIvfFlatTest, ExtendWithoutHostIds) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 10; - gpu_ivf_flat_t index(dataset.data(), n_base, dimension, + gpu_ivf_flat_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -611,7 +611,7 @@ TEST(GpuIvfFlatTest, ExtendWithHostIds) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 10; - gpu_ivf_flat_t index(dataset.data(), n_base, dimension, + gpu_ivf_flat_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, base_ids.data()); index.start(); @@ -664,7 +664,7 @@ TEST(GpuIvfFlatTest, ExtendReplicatedWithHostIds) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 10; - gpu_ivf_flat_t index(dataset.data(), n_base, dimension, + gpu_ivf_flat_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED, base_ids.data()); index.start(); @@ -712,7 +712,7 @@ TEST(GpuIvfFlatTest, ExtendShardedWithHostIds) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 10; - gpu_ivf_flat_t index(dataset.data(), n_base, dimension, + gpu_ivf_flat_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, base_ids.data()); index.start(); @@ -759,7 +759,7 @@ TEST(GpuIvfFlatTest, ExtendShardedWithoutHostIds) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 10; - gpu_ivf_flat_t index(dataset.data(), n_base, dimension, + gpu_ivf_flat_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, nullptr); index.start(); @@ -802,7 +802,7 @@ TEST(GpuIvfFlatTest, ManualShardedGetCenters) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 50; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); index.start(); index.build(); @@ -833,7 +833,7 @@ TEST(GpuIvfFlatTest, FilteredSearchIncludesOnlyAllowedCategories) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 4; - gpu_ivf_flat_t index(dataset.data(), count, dimension, + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -880,7 +880,7 @@ TEST(GpuIvfFlatTest, FilteredSearchCombinesWithDeleteBitset) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 4; - gpu_ivf_flat_t index(dataset.data(), count, dimension, + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -920,7 +920,7 @@ TEST(GpuIvfFlatTest, FilteredSearchEmptyPredsMatchesUnfiltered) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 4; - gpu_ivf_flat_t index(dataset.data(), count, dimension, + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -956,7 +956,7 @@ TEST(GpuIvfFlatTest, KExceedsIndexSizeClampsAndPads) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 2; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); @@ -1004,7 +1004,7 @@ TEST(GpuIvfFlatTest, MultiQueryKExceedsIndexSize) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 2; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); diff --git a/cgo/cuvs/test/ivf_pq_test.cu b/cgo/cuvs/test/ivf_pq_test.cu index 53601988ff13e..0e7463fe1405b 100644 --- a/cgo/cuvs/test/ivf_pq_test.cu +++ b/cgo/cuvs/test/ivf_pq_test.cu @@ -21,6 +21,11 @@ #include #include #include +#include +#include +#include +#include +#include using namespace matrixone; @@ -38,7 +43,7 @@ TEST(GpuIvfPqTest, BasicLoadSearchAndCenters) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 2; bp.m = 8; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); @@ -73,7 +78,7 @@ TEST(GpuIvfPqTest, BasicLoadAndSearchWithIds) { std::vector devices = {0}; ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 100; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); index.start(); index.build(); @@ -87,6 +92,353 @@ TEST(GpuIvfPqTest, BasicLoadAndSearchWithIds) { index.destroy(); } +// Native half (f16) build + search — validates the direct vecf16-base path +// (gpu_ivf_pq_t native add_chunk/search, no quantizer). Linking this proves +// cuVS supports ivf_pq over half (unlike brute force over int8/uint8). +TEST(GpuIvfPqTest, BasicLoadAndSearchHalf) { + const uint32_t dimension = 16; + const uint64_t count = 1000; + std::vector dataset(count * dimension); + std::vector ids(count); + for (size_t i = 0; i < count; ++i) { + for (size_t j = 0; j < dimension; ++j) + dataset[i * dimension + j] = __float2half((float)rand() / RAND_MAX); + ids[i] = (int64_t)(i + 2000); + } + + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); + bp.n_lists = 100; + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); + index.start(); + index.build(); + + // Query == row 0, so the nearest neighbour must be its id (2000). + std::vector queries(dataset.begin(), dataset.begin() + dimension); + ivf_pq_search_params_t sp = ivf_pq_search_params_default(); + auto result = index.search(queries.data(), 1, dimension, 5, sp); + + ASSERT_EQ(result.neighbors.size(), (size_t)5); + ASSERT_EQ(result.neighbors[0], 2000); + + index.destroy(); +} + +// vecf16 base -> int8 storage via the native B(half)-source quantizer +// (add_chunk_quantize). Verifies the quantize-build path: train the half-source +// quantizer on the buffered vecf16 sample, transform half->int8, store as int8, +// and build a searchable int8 index. No f32 detour. +TEST(GpuIvfPqTest, HalfQuantizeToInt8Build) { + const uint32_t dimension = 16; + const uint64_t count = 2000; + std::vector dataset(count * dimension); + std::vector ids(count); + for (size_t i = 0; i < count; ++i) { + for (size_t j = 0; j < dimension; ++j) + dataset[i * dimension + j] = __float2half((float)(rand() % 256) / 255.0f); + ids[i] = (int64_t)(i + 5000); + } + + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); + bp.n_lists = 50; + gpu_ivf_pq_t index(count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + index.start(); + index.add_chunk_quantize(dataset.data(), count, -1, ids.data()); + index.build(); + + // The resulting int8 index is searchable with a native int8 query. + std::vector q(dimension, 0); + ivf_pq_search_params_t sp = ivf_pq_search_params_default(); + sp.n_probes = 50; + auto result = index.search(q.data(), 1, dimension, 5, sp); + ASSERT_EQ(result.neighbors.size(), (size_t)5); + for (auto n : result.neighbors) { + ASSERT_GE(n, (int64_t)5000); + ASSERT_LT(n, (int64_t)(5000 + count)); + } + + index.destroy(); +} + +// --------------------------------------------------------------------------- +// REPRODUCTION: f32 base -> int8 vs uint8 storage recall on SIGNED data. +// +// Isolates whether the uint8 quantization recall collapse (seen at 1M scale: +// int8 ~0.83, uint8 ~0.24) is a cuVS-layer bug or mo Go plumbing. This test +// builds BOTH indexes purely through the C++ cuVS wrapper (no mo storage/CDC), +// from the SAME signed dataset with the SAME scalar quantizer. transform +// differs from transform only by a monotonic +128 shift (asserted below) +// which is L2-invariant -- so cuVS uint8 recall MUST match int8 unless cuVS +// mishandles uint8 datasets. +namespace { +struct RecallData { + uint32_t dim; uint64_t count; uint64_t nq; + std::vector base, queries; + std::vector ids; + std::vector> gt; +}; + +RecallData make_signed_recall_data(uint32_t dim, uint64_t count, uint64_t nq, uint32_t k) { + RecallData d; d.dim = dim; d.count = count; d.nq = nq; + d.base.resize(count * dim); d.queries.resize(nq * dim); d.ids.resize(count); + srand(1234); + auto sgn = []() { return ((float)rand() / RAND_MAX) * 2.0f - 1.0f; }; // [-1,1], zero-centered + for (uint64_t i = 0; i < count; ++i) { + for (uint32_t j = 0; j < dim; ++j) d.base[i * dim + j] = sgn(); + d.ids[i] = (int64_t)i; + } + for (uint64_t q = 0; q < nq; ++q) + for (uint32_t j = 0; j < dim; ++j) d.queries[q * dim + j] = sgn(); + d.gt.resize(nq); + for (uint64_t q = 0; q < nq; ++q) { + std::vector> dist(count); + for (uint64_t i = 0; i < count; ++i) { + float s = 0; + for (uint32_t j = 0; j < dim; ++j) { float df = d.queries[q * dim + j] - d.base[i * dim + j]; s += df * df; } + dist[i] = {s, (int64_t)i}; + } + std::partial_sort(dist.begin(), dist.begin() + k, dist.end()); + d.gt[q].resize(k); + for (uint32_t r = 0; r < k; ++r) d.gt[q][r] = dist[r].second; + } + return d; +} + +template +double measure_quantize_recall(RecallData& d, uint32_t k) { + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); + bp.n_lists = 64; + gpu_ivf_pq_t index(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + index.start(); + index.add_chunk_quantize(d.base.data(), d.count, -1, d.ids.data()); + index.build(); + ivf_pq_search_params_t sp = ivf_pq_search_params_default(); + sp.n_probes = 64; + auto res = index.search_quantize(d.queries.data(), d.nq, d.dim, k, sp); + size_t hit = 0, tot = 0; + for (uint64_t q = 0; q < d.nq; ++q) { + std::set gt(d.gt[q].begin(), d.gt[q].end()); + for (uint32_t r = 0; r < k; ++r) { + int64_t n = res.neighbors[q * k + r]; + if (gt.count(n)) ++hit; + } + tot += k; + } + index.destroy(); + return (double)hit / (double)tot; +} +} // namespace + +TEST(GpuIvfPqRecall, Int8VsUint8SignedData) { + const uint32_t dim = 32, k = 10; + const uint64_t count = 4000, nq = 200; + RecallData d = make_signed_recall_data(dim, count, nq, k); + + double r_int8 = measure_quantize_recall(d, k); + double r_uint8 = measure_quantize_recall(d, k); + + // Confirm uint8 codes == int8 codes + 128 (monotonic, L2-invariant). If 0 + // mismatches, the quantization is identical up to a constant shift, so any + // recall gap is purely cuVS's uint8 dataset handling. + int mism = 0; + { + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 64; + gpu_ivf_pq_t qi(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + qi.start(); qi.add_chunk_quantize(d.base.data(), d.count, -1, d.ids.data()); qi.build(); + gpu_ivf_pq_t qu(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + qu.start(); qu.add_chunk_quantize(d.base.data(), d.count, -1, d.ids.data()); qu.build(); + std::vector ci(dim); std::vector cu(dim); + qi.quantize_query(d.queries.data(), 1, ci.data()); + qu.quantize_query(d.queries.data(), 1, cu.data()); + for (uint32_t j = 0; j < dim; ++j) if ((int)cu[j] != (int)ci[j] + 128) ++mism; + qi.destroy(); qu.destroy(); + } + + printf("[repro] +128 mismatches: %d / %u dims (0 => uint8==int8+128, L2-invariant)\n", mism, dim); + printf("[repro] f32->int8 recall@%u = %.4f\n", k, r_int8); + printf("[repro] f32->uint8 recall@%u = %.4f\n", k, r_uint8); + printf("[repro] VERDICT: int8 high + uint8 low + 0 mismatches => cuVS uint8 bug (not mo plumbing)\n"); + + ASSERT_TRUE(r_int8 > 0.5); // int8 quantize recall should be reasonable + ASSERT_TRUE(mism == 0); // uint8 codes must equal int8 codes + 128 +} + +// --------------------------------------------------------------------------- +// PURE cuVS reproduction: calls cuvs::neighbors::ivf_pq::build/search DIRECTLY +// on raw int8 vs uint8 device matrices. No gpu_ivf_pq_t wrapper, no +// scalar_quantizer_t, no add_chunk/search_float -- zero matrixone code in the +// index path. The uint8 dataset is the int8 dataset + 128 (an exact, monotonic, +// L2-identical shift), so cuVS MUST return identical recall unless it mishandles +// uint8 ivf_pq datasets. This is the definitive cuVS-vs-ours test. +namespace { +// CPU scalar quantize float -> int8 via a global [lo,hi] -> [-128,127] map. +void cpu_quantize_int8(const std::vector& src, std::vector& out, float lo, float hi) { + out.resize(src.size()); + const float scale = 255.0f / (hi - lo); + for (size_t i = 0; i < src.size(); ++i) { + int iv = (int)lroundf((src[i] - lo) * scale - 128.0f); // [lo,hi] -> [-128,127] + iv = std::max(-128, std::min(127, iv)); + out[i] = (int8_t)iv; + } +} + +template +double pure_cuvs_ivfpq_recall(const std::vector& base_q, const std::vector& query_q, + uint64_t count, uint64_t nq, uint32_t dim, uint32_t k, + const std::vector>& gt) { + raft::resources res; + auto base_dev = raft::make_device_matrix(res, count, dim); + raft::copy(res, base_dev.view(), raft::make_host_matrix_view(base_q.data(), count, dim)); + auto query_dev = raft::make_device_matrix(res, nq, dim); + raft::copy(res, query_dev.view(), raft::make_host_matrix_view(query_q.data(), nq, dim)); + raft::resource::sync_stream(res); + + cuvs::neighbors::ivf_pq::index_params ip; + ip.metric = cuvs::distance::DistanceType::L2Expanded; + ip.n_lists = 64; + ip.pq_dim = dim / 2; + ip.pq_bits = 8; + auto index = cuvs::neighbors::ivf_pq::build(res, ip, raft::make_const_mdspan(base_dev.view())); + + cuvs::neighbors::ivf_pq::search_params sp; + sp.n_probes = 64; + auto neighbors = raft::make_device_matrix(res, nq, k); + auto distances = raft::make_device_matrix(res, nq, k); + cuvs::neighbors::ivf_pq::search(res, sp, index, raft::make_const_mdspan(query_dev.view()), + neighbors.view(), distances.view()); + raft::resource::sync_stream(res); + + std::vector nh(nq * k); + raft::copy(res, raft::make_host_matrix_view(nh.data(), nq, k), neighbors.view()); + raft::resource::sync_stream(res); + + size_t hit = 0, tot = 0; + for (uint64_t q = 0; q < nq; ++q) { + std::set g(gt[q].begin(), gt[q].end()); + for (uint32_t r = 0; r < k; ++r) if (g.count(nh[q * k + r])) ++hit; + tot += k; + } + return (double)hit / (double)tot; +} +} // namespace + +TEST(PureCuvsIvfPqRecall, Int8VsUint8SignedData) { + const uint32_t dim = 32, k = 10; + const uint64_t count = 4000, nq = 200; + RecallData d = make_signed_recall_data(dim, count, nq, k); // signed float data + float-L2 GT + + float lo = 1e30f, hi = -1e30f; + for (float v : d.base) { lo = std::min(lo, v); hi = std::max(hi, v); } + + std::vector base_i8, query_i8; + cpu_quantize_int8(d.base, base_i8, lo, hi); + cpu_quantize_int8(d.queries, query_i8, lo, hi); + + // uint8 dataset = int8 dataset + 128 (exact; L2(a-b) is identical). + std::vector base_u8(base_i8.size()), query_u8(query_i8.size()); + for (size_t i = 0; i < base_i8.size(); ++i) base_u8[i] = (uint8_t)((int)base_i8[i] + 128); + for (size_t i = 0; i < query_i8.size(); ++i) query_u8[i] = (uint8_t)((int)query_i8[i] + 128); + + double r_i8 = pure_cuvs_ivfpq_recall(base_i8, query_i8, count, nq, dim, k, d.gt); + double r_u8 = pure_cuvs_ivfpq_recall(base_u8, query_u8, count, nq, dim, k, d.gt); + + printf("[pure-cuvs] f32->int8 recall@%u = %.4f\n", k, r_i8); + printf("[pure-cuvs] f32->uint8 recall@%u = %.4f (uint8 = int8+128, L2-identical)\n", k, r_u8); + printf("[pure-cuvs] VERDICT: int8 high + uint8 low => cuVS uint8 ivf_pq bug (zero matrixone code in path)\n"); + ASSERT_TRUE(r_i8 > 0.3); +} + +// --------------------------------------------------------------------------- +// REPRODUCTION: f16 (half) base -> int8 vs uint8 storage recall on SIGNED data. +// +// Same isolation as GpuIvfPqRecall.Int8VsUint8SignedData, but the source +// element type is half (gpu_ivf_pq_t + the native half-source +// quantizer) instead of float. The base/query halves are derived from the SAME +// signed float dataset and graded against the SAME float-L2 ground truth. +// transform again differs from transform only by the monotonic +// +128 shift (asserted below), which is L2-invariant -- so cuVS uint8 recall +// MUST match int8 unless cuVS mishandles uint8 ivf_pq datasets. Proves the +// uint8 collapse is independent of the source element type (f32 vs f16). +namespace { +template +double measure_half_quantize_recall(RecallData& d, uint32_t k) { + // Convert the float base/queries to half — the native B source type. + std::vector base_h(d.base.size()), query_h(d.queries.size()); + for (size_t i = 0; i < d.base.size(); ++i) base_h[i] = __float2half(d.base[i]); + for (size_t i = 0; i < d.queries.size(); ++i) query_h[i] = __float2half(d.queries[i]); + + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); + bp.n_lists = 64; + gpu_ivf_pq_t index(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + index.start(); + index.add_chunk_quantize(base_h.data(), d.count, -1, d.ids.data()); + index.build(); + + // Quantize the half queries to storage codes via the half-source quantizer, + // then run the native T search path (same as production). + std::vector qcodes(d.nq * d.dim); + index.quantize_query(query_h.data(), d.nq, qcodes.data()); + ivf_pq_search_params_t sp = ivf_pq_search_params_default(); + sp.n_probes = 64; + auto res = index.search(qcodes.data(), d.nq, d.dim, k, sp); + + size_t hit = 0, tot = 0; + for (uint64_t q = 0; q < d.nq; ++q) { + std::set gt(d.gt[q].begin(), d.gt[q].end()); + for (uint32_t r = 0; r < k; ++r) { + int64_t n = res.neighbors[q * k + r]; + if (gt.count(n)) ++hit; + } + tot += k; + } + index.destroy(); + return (double)hit / (double)tot; +} +} // namespace + +TEST(GpuIvfPqRecall, Int8VsUint8SignedDataHalf) { + const uint32_t dim = 32, k = 10; + const uint64_t count = 4000, nq = 200; + RecallData d = make_signed_recall_data(dim, count, nq, k); + + double r_int8 = measure_half_quantize_recall(d, k); + double r_uint8 = measure_half_quantize_recall(d, k); + + // Confirm uint8 query codes == int8 query codes + 128 (monotonic, + // L2-invariant), produced by the SAME half source. 0 mismatches => any + // recall gap is purely cuVS's uint8 dataset handling, not f16 conversion. + int mism = 0; + { + std::vector base_h(d.base.size()), query_h(d.queries.size()); + for (size_t i = 0; i < d.base.size(); ++i) base_h[i] = __float2half(d.base[i]); + for (size_t i = 0; i < d.queries.size(); ++i) query_h[i] = __float2half(d.queries[i]); + + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 64; + gpu_ivf_pq_t qi(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + qi.start(); qi.add_chunk_quantize(base_h.data(), d.count, -1, d.ids.data()); qi.build(); + gpu_ivf_pq_t qu(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + qu.start(); qu.add_chunk_quantize(base_h.data(), d.count, -1, d.ids.data()); qu.build(); + std::vector ci(dim); std::vector cu(dim); + qi.quantize_query(query_h.data(), 1, ci.data()); + qu.quantize_query(query_h.data(), 1, cu.data()); + for (uint32_t j = 0; j < dim; ++j) if ((int)cu[j] != (int)ci[j] + 128) ++mism; + qi.destroy(); qu.destroy(); + } + + printf("[repro-f16] +128 mismatches: %d / %u dims (0 => uint8==int8+128, L2-invariant)\n", mism, dim); + printf("[repro-f16] f16->int8 recall@%u = %.4f\n", k, r_int8); + printf("[repro-f16] f16->uint8 recall@%u = %.4f\n", k, r_uint8); + printf("[repro-f16] VERDICT: int8 high + uint8 low + 0 mismatches => cuVS uint8 bug (f16 source)\n"); + + ASSERT_TRUE(r_int8 > 0.5); // f16->int8 quantize recall should be reasonable + ASSERT_TRUE(mism == 0); // uint8 codes must equal int8 codes + 128 +} + TEST(GpuIvfPqTest, ParallelAddChunkWithOffset) { const uint32_t dimension = 16; const uint64_t count_per_chunk = 500; @@ -108,7 +460,7 @@ TEST(GpuIvfPqTest, ParallelAddChunkWithOffset) { std::vector devices = {0}; ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 100; - gpu_ivf_pq_t index(total_count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_pq_t index(total_count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); std::thread t1([&]() { index.add_chunk(chunk1.data(), count_per_chunk, 0, ids1.data()); }); @@ -144,7 +496,7 @@ TEST(GpuIvfPqTest, SaveAndLoadFromFile) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 2; bp.m = 2; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); index.save(filename); @@ -156,7 +508,7 @@ TEST(GpuIvfPqTest, SaveAndLoadFromFile) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 2; bp.m = 2; - gpu_ivf_pq_t index(filename, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_pq_t index(filename, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.load(filename); @@ -192,7 +544,7 @@ TEST(GpuIvfPqTest, ManualShardedSearch) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 50; bp.m = 8; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); index.start(); index.build(); @@ -226,7 +578,7 @@ TEST(GpuIvfPqTest, ManualShardedSearchWithIds) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 50; bp.m = 8; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, ids.data()); + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, ids.data()); index.start(); index.build(); @@ -258,7 +610,7 @@ TEST(GpuIvfPqTest, ManualShardedGetCenters) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 50; bp.m = 8; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); index.start(); index.build(); @@ -287,7 +639,7 @@ TEST(GpuIvfPqTest, ReplicatedModeSimulation) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 100; bp.m = 8; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED); + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED); index.start(); index.build(); std::vector queries(dataset.begin(), dataset.begin() + dimension); @@ -315,7 +667,7 @@ TEST(GpuIvfPqTest, ExtendWithoutHostIds) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 10; bp.m = 8; - gpu_ivf_pq_t index(dataset.data(), n_base, dimension, + gpu_ivf_pq_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -383,7 +735,7 @@ TEST(GpuIvfPqTest, ExtendWithHostIds) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 10; bp.m = 8; - gpu_ivf_pq_t index(dataset.data(), n_base, dimension, + gpu_ivf_pq_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, base_ids.data()); index.start(); @@ -439,7 +791,7 @@ TEST(GpuIvfPqTest, ExtendReplicatedWithHostIds) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 10; bp.m = 8; - gpu_ivf_pq_t index(dataset.data(), n_base, dimension, + gpu_ivf_pq_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED, base_ids.data()); index.start(); @@ -483,7 +835,7 @@ TEST(GpuIvfPqTest, ExtendShardedWithHostIds) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 10; - gpu_ivf_pq_t index(dataset.data(), n_base, dimension, + gpu_ivf_pq_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, base_ids.data()); index.start(); @@ -531,7 +883,7 @@ TEST(GpuIvfPqTest, ExtendShardedWithoutHostIds) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 10; - gpu_ivf_pq_t index(dataset.data(), n_base, dimension, + gpu_ivf_pq_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, nullptr); index.start(); @@ -580,7 +932,7 @@ TEST(GpuIvfPqTest, FilteredSearchExcludesForbiddenCategory) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 4; bp.m = 4; - gpu_ivf_pq_t index(dataset.data(), count, dimension, + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -627,7 +979,7 @@ TEST(GpuIvfPqTest, FilteredSearchCombinesWithDeleteBitset) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 4; bp.m = 4; - gpu_ivf_pq_t index(dataset.data(), count, dimension, + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -689,7 +1041,7 @@ TEST(GpuIvfPqTest, FilteredSearchEmptyPredsMatchesUnfiltered) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 4; bp.m = 4; - gpu_ivf_pq_t index(dataset.data(), count, dimension, + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -729,7 +1081,7 @@ TEST(GpuIvfPqTest, KExceedsIndexSizeClampsAndPads) { bp.n_lists = 2; bp.m = 2; bp.bits_per_code = 8; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); @@ -790,7 +1142,7 @@ TEST(GpuIvfPqTest, MultiQueryKExceedsIndexSize) { bp.n_lists = 2; bp.m = 2; bp.bits_per_code = 8; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); diff --git a/cgo/cuvs/test/uint8_quant_bug.cu b/cgo/cuvs/test/uint8_quant_bug.cu new file mode 100644 index 0000000000000..a606475d7d2cc --- /dev/null +++ b/cgo/cuvs/test/uint8_quant_bug.cu @@ -0,0 +1,214 @@ +/* + * Copyright 2021 Matrix Origin + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// --------------------------------------------------------------------------- +// STANDALONE repro for the uint8 quantization recall collapse. +// +// Isolates whether the uint8 quantization recall collapse (seen at 1M scale: +// int8 ~0.83, uint8 ~0.24) is a cuVS-layer bug or mo Go plumbing. It builds +// int8 vs uint8 indexes purely through the C++ cuVS wrapper (no mo storage/CDC) +// from the SAME signed dataset and the SAME scalar quantizer, for BOTH source +// element types: +// +// * f32 source (gpu_ivf_pq_t) via search_float (auto query quant) +// * f16 source (gpu_ivf_pq_t) via quantize_query + native search +// +// transform differs from transform only by a monotonic +128 shift +// (asserted: 0 mismatches), which is L2-invariant -- so cuVS uint8 recall MUST +// match int8 unless cuVS mishandles uint8 ivf_pq datasets. +// +// Build/run as its OWN executable (does not run the whole test suite): +// make uint8_quant_bug && ./uint8_quant_bug + +#include "cuvs_worker.hpp" +#include "ivf_pq.hpp" +#include "helper.h" +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace matrixone; + +namespace { + +struct RecallData { + uint32_t dim; uint64_t count; uint64_t nq; + std::vector base, queries; + std::vector ids; + std::vector> gt; +}; + +RecallData make_signed_recall_data(uint32_t dim, uint64_t count, uint64_t nq, uint32_t k) { + RecallData d; d.dim = dim; d.count = count; d.nq = nq; + d.base.resize(count * dim); d.queries.resize(nq * dim); d.ids.resize(count); + srand(1234); + auto sgn = []() { return ((float)rand() / RAND_MAX) * 2.0f - 1.0f; }; // [-1,1], zero-centered + for (uint64_t i = 0; i < count; ++i) { + for (uint32_t j = 0; j < dim; ++j) d.base[i * dim + j] = sgn(); + d.ids[i] = (int64_t)i; + } + for (uint64_t q = 0; q < nq; ++q) + for (uint32_t j = 0; j < dim; ++j) d.queries[q * dim + j] = sgn(); + d.gt.resize(nq); + for (uint64_t q = 0; q < nq; ++q) { + std::vector> dist(count); + for (uint64_t i = 0; i < count; ++i) { + float s = 0; + for (uint32_t j = 0; j < dim; ++j) { float df = d.queries[q * dim + j] - d.base[i * dim + j]; s += df * df; } + dist[i] = {s, (int64_t)i}; + } + std::partial_sort(dist.begin(), dist.begin() + k, dist.end()); + d.gt[q].resize(k); + for (uint32_t r = 0; r < k; ++r) d.gt[q][r] = dist[r].second; + } + return d; +} + +double recall_at_k(const ivf_pq_search_result_t& res, const RecallData& d, uint32_t k) { + size_t hit = 0, tot = 0; + for (uint64_t q = 0; q < d.nq; ++q) { + std::set gt(d.gt[q].begin(), d.gt[q].end()); + for (uint32_t r = 0; r < k; ++r) { + int64_t n = res.neighbors[q * k + r]; + if (gt.count(n)) ++hit; + } + tot += k; + } + return (double)hit / (double)tot; +} + +// --- f32 source path: search_float auto-quantizes the float queries. --------- +template +double f32_quantize_recall(RecallData& d, uint32_t k) { + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); + bp.n_lists = 64; + gpu_ivf_pq_t index(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + index.start(); + index.add_chunk_quantize(d.base.data(), d.count, -1, d.ids.data()); + index.build(); + ivf_pq_search_params_t sp = ivf_pq_search_params_default(); + sp.n_probes = 64; + auto res = index.search_quantize(d.queries.data(), d.nq, d.dim, k, sp); + double r = recall_at_k(res, d, k); + index.destroy(); + return r; +} + +// --- f16 source path: quantize_query then native T search. ------------------- +template +double f16_quantize_recall(RecallData& d, uint32_t k) { + std::vector base_h(d.base.size()), query_h(d.queries.size()); + for (size_t i = 0; i < d.base.size(); ++i) base_h[i] = __float2half(d.base[i]); + for (size_t i = 0; i < d.queries.size(); ++i) query_h[i] = __float2half(d.queries[i]); + + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); + bp.n_lists = 64; + gpu_ivf_pq_t index(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + index.start(); + index.add_chunk_quantize(base_h.data(), d.count, -1, d.ids.data()); + index.build(); + + std::vector qcodes(d.nq * d.dim); + index.quantize_query(query_h.data(), d.nq, qcodes.data()); + ivf_pq_search_params_t sp = ivf_pq_search_params_default(); + sp.n_probes = 64; + auto res = index.search(qcodes.data(), d.nq, d.dim, k, sp); + double r = recall_at_k(res, d, k); + index.destroy(); + return r; +} + +// Quantize one query through both int8 and uint8 quantizers (same source) and +// count dims where uint8 != int8 + 128. 0 => monotonic, L2-invariant shift. +template +int plus128_mismatches(RecallData& d, const std::vector& base_b, const std::vector& query_b) { + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 64; + gpu_ivf_pq_t qi(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + qi.start(); qi.add_chunk_quantize(base_b.data(), d.count, -1, d.ids.data()); qi.build(); + gpu_ivf_pq_t qu(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + qu.start(); qu.add_chunk_quantize(base_b.data(), d.count, -1, d.ids.data()); qu.build(); + std::vector ci(d.dim); std::vector cu(d.dim); + qi.quantize_query(query_b.data(), 1, ci.data()); + qu.quantize_query(query_b.data(), 1, cu.data()); + int mism = 0; + for (uint32_t j = 0; j < d.dim; ++j) if ((int)cu[j] != (int)ci[j] + 128) ++mism; + qi.destroy(); qu.destroy(); + return mism; +} + +} // namespace + +int main() { + const uint32_t dim = 32, k = 10; + const uint64_t count = 4000, nq = 200; + RecallData d = make_signed_recall_data(dim, count, nq, k); + + int failures = 0; + + // ---- f32 source ------------------------------------------------------- + { + double r_i8 = f32_quantize_recall(d, k); + double r_u8 = f32_quantize_recall(d, k); + int mism = plus128_mismatches(d, d.base, d.queries); + + printf("\n=== f32 source ===\n"); + printf("[f32] +128 mismatches: %d / %u dims (0 => uint8==int8+128, L2-invariant)\n", mism, dim); + printf("[f32] f32->int8 recall@%u = %.4f\n", k, r_i8); + printf("[f32] f32->uint8 recall@%u = %.4f\n", k, r_u8); + // Data-driven verdict: with 0 mismatches the codes are identical up to a + // constant shift, so a large recall gap can only come from cuVS's uint8 + // dataset handling. A small gap => the collapse does NOT reproduce here. + printf("[f32] VERDICT: %s (gap=%.4f, mism=%d)\n", + (mism == 0 && r_i8 - r_u8 > 0.20) ? "uint8 COLLAPSE reproduced => cuVS uint8 bug" + : "no collapse at this scale (uint8 ~= int8)", + r_i8 - r_u8, mism); + if (!(r_i8 > 0.5)) { printf("[f32] FAIL: int8 recall too low (%.4f)\n", r_i8); ++failures; } + if (mism != 0) { printf("[f32] FAIL: %d +128 mismatches\n", mism); ++failures; } + } + + // ---- f16 source ------------------------------------------------------- + { + std::vector base_h(d.base.size()), query_h(d.queries.size()); + for (size_t i = 0; i < d.base.size(); ++i) base_h[i] = __float2half(d.base[i]); + for (size_t i = 0; i < d.queries.size(); ++i) query_h[i] = __float2half(d.queries[i]); + + double r_i8 = f16_quantize_recall(d, k); + double r_u8 = f16_quantize_recall(d, k); + int mism = plus128_mismatches(d, base_h, query_h); + + printf("\n=== f16 source ===\n"); + printf("[f16] +128 mismatches: %d / %u dims (0 => uint8==int8+128, L2-invariant)\n", mism, dim); + printf("[f16] f16->int8 recall@%u = %.4f\n", k, r_i8); + printf("[f16] f16->uint8 recall@%u = %.4f\n", k, r_u8); + printf("[f16] VERDICT: %s (gap=%.4f, mism=%d)\n", + (mism == 0 && r_i8 - r_u8 > 0.20) ? "uint8 COLLAPSE reproduced => cuVS uint8 bug (f16 source)" + : "no collapse at this scale (uint8 ~= int8)", + r_i8 - r_u8, mism); + if (!(r_i8 > 0.5)) { printf("[f16] FAIL: int8 recall too low (%.4f)\n", r_i8); ++failures; } + if (mism != 0) { printf("[f16] FAIL: %d +128 mismatches\n", mism); ++failures; } + } + + printf("\n%s (%d failure(s))\n", failures == 0 ? "PASSED" : "FAILED", failures); + return failures == 0 ? 0 : 1; +} diff --git a/cgo/cuvs/test/wiki1m_uint8_bug.cu b/cgo/cuvs/test/wiki1m_uint8_bug.cu new file mode 100644 index 0000000000000..8c42e2c4722c2 --- /dev/null +++ b/cgo/cuvs/test/wiki1m_uint8_bug.cu @@ -0,0 +1,308 @@ +/* + * Copyright 2021 Matrix Origin + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// --------------------------------------------------------------------------- +// REAL-DATASET repro for the uint8 quantization recall collapse, at 1M scale. +// +// The synthetic 4k repro (uint8_quant_bug.cu) did NOT reproduce the collapse, +// so this drives the SAME pure-C++ cuVS path (no mo Go plumbing) over the real +// wiki_all_1M dataset (1M x 768) with its published ground truth — the exact +// data/scale where the collapse (int8 ~0.83, uint8 ~0.24) was first seen. +// +// For each source element type it builds int8 and uint8 ivf_pq indexes from the +// SAME float base via the SAME scalar quantizer, then grades recall@k against +// the dataset's ground-truth neighbors. transform differs from +// transform only by a monotonic +128 shift (asserted: 0 mismatches), so +// uint8 recall MUST match int8 unless cuVS mishandles uint8 ivf_pq datasets. +// +// Build/run as its own executable: +// make wiki1m_uint8_bug && ./wiki1m_uint8_bug [base.fbin] [queries.fbin] [gt.ibin] +// Env knobs: MOQ=#queries (default 1000), MOK=k (default 10), +// MO_NLISTS (default 1024), MO_NPROBES (default 64), MO_F16=1. + +#include "cuvs_worker.hpp" +#include "ivf_pq.hpp" +#include "helper.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace matrixone; + +namespace { + +const char* kBase = "../../../vector_benchmark/wiki_all_1M/base.1M.fbin"; +const char* kQueries = "../../../vector_benchmark/wiki_all_1M/queries.fbin"; +const char* kGt = "../../../vector_benchmark/wiki_all_1M/groundtruth.1M.neighbors.ibin"; + +uint64_t env_u64(const char* k, uint64_t def) { + const char* v = getenv(k); + return v ? strtoull(v, nullptr, 10) : def; +} + +// .fbin: int32 n, int32 dim, then row-major float32[n*dim]. +// max_rows<=0 loads all; otherwise only the first max_rows rows. +std::vector load_fbin(const std::string& path, uint64_t& n, uint32_t& dim, int64_t max_rows = -1) { + FILE* f = fopen(path.c_str(), "rb"); + if (!f) throw std::runtime_error("cannot open " + path); + int32_t hn = 0, hd = 0; + if (fread(&hn, 4, 1, f) != 1 || fread(&hd, 4, 1, f) != 1) { fclose(f); throw std::runtime_error("bad header " + path); } + n = (uint64_t)hn; dim = (uint32_t)hd; + if (max_rows > 0 && (uint64_t)max_rows < n) n = (uint64_t)max_rows; + std::vector data(n * dim); + size_t got = fread(data.data(), sizeof(float), n * dim, f); + fclose(f); + if (got != n * dim) throw std::runtime_error("short read " + path); + return data; +} + +// .ibin: int32 n, int32 k, then row-major int32[n*k]. Returns first `keep` cols. +std::vector> load_ibin_gt(const std::string& path, uint64_t nq, uint32_t keep) { + FILE* f = fopen(path.c_str(), "rb"); + if (!f) throw std::runtime_error("cannot open " + path); + int32_t hn = 0, hk = 0; + if (fread(&hn, 4, 1, f) != 1 || fread(&hk, 4, 1, f) != 1) { fclose(f); throw std::runtime_error("bad header " + path); } + uint32_t k = (uint32_t)hk; + if (keep > k) throw std::runtime_error("gt k too small"); + std::vector row(k); + std::vector> gt(nq); + for (uint64_t q = 0; q < nq; ++q) { + if (fread(row.data(), 4, k, f) != k) { fclose(f); throw std::runtime_error("short gt read"); } + gt[q].resize(keep); + for (uint32_t r = 0; r < keep; ++r) gt[q][r] = (int64_t)row[r]; + } + fclose(f); + return gt; +} + +double recall_at_k(const ivf_pq_search_result_t& res, + const std::vector>& gt, uint64_t nq, uint32_t k) { + size_t hit = 0, tot = 0; + for (uint64_t q = 0; q < nq; ++q) { + std::set g(gt[q].begin(), gt[q].end()); + for (uint32_t r = 0; r < k; ++r) + if (g.count(res.neighbors[q * k + r])) ++hit; + tot += k; + } + return (double)hit / (double)tot; +} + +struct Cfg { uint32_t n_lists, n_probes, m, bits; }; + +inline void apply_cfg(ivf_pq_build_params_t& bp, const Cfg& c) { + bp.n_lists = c.n_lists; + if (c.m) bp.m = c.m; + if (c.bits) bp.bits_per_code = c.bits; +} + +template +double f32_recall(const std::vector& base, uint64_t count, uint32_t dim, + const std::vector& queries, uint64_t nq, + const std::vector>& gt, uint32_t k, const Cfg& c) { + std::vector ids(count); + std::iota(ids.begin(), ids.end(), (int64_t)0); // row index == gt id space + + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); + apply_cfg(bp, c); + gpu_ivf_pq_t index(count, dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + index.start(); + index.add_chunk_quantize(base.data(), count, -1, ids.data()); + index.build(); + ivf_pq_search_params_t sp = ivf_pq_search_params_default(); + sp.n_probes = c.n_probes; + auto res = index.search_quantize(queries.data(), nq, dim, k, sp); + double r = recall_at_k(res, gt, nq, k); + index.destroy(); + return r; +} + +// Same as f32_recall but exercises the MO persist->load cycle: build, save_dir, +// destroy, reload a fresh index via load_dir, then search. This is what +// mo-service does (Pack/Unpack -> save_dir/load_dir). If uint8 recall is fine in +// f32_recall but collapses here, the bug is in cuVS serialize/deserialize of a +// uint8-built ivf_pq index (the quantizer.bin is byte-identical for int8/uint8). +template +double f32_recall_saveload(const std::vector& base, uint64_t count, uint32_t dim, + const std::vector& queries, uint64_t nq, + const std::vector>& gt, uint32_t k, const Cfg& c, + const std::string& dir) { + std::vector ids(count); + std::iota(ids.begin(), ids.end(), (int64_t)0); + + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); + apply_cfg(bp, c); + { + gpu_ivf_pq_t index(count, dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + index.start(); + index.add_chunk_quantize(base.data(), count, -1, ids.data()); + index.build(); + index.save_dir(dir); // writes index.bin + quantizer.bin + manifest + index.destroy(); + } + // Fresh index, load from disk exactly like LoadIndex/Unpack does. + gpu_ivf_pq_t reloaded(count, dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + reloaded.start(); + reloaded.load_dir(dir, DistributionMode_SINGLE_GPU); + ivf_pq_search_params_t sp = ivf_pq_search_params_default(); + sp.n_probes = c.n_probes; + auto res = reloaded.search_quantize(queries.data(), nq, dim, k, sp); + double r = recall_at_k(res, gt, nq, k); + reloaded.destroy(); + return r; +} + +template +double f16_recall(const std::vector& base_h, uint64_t count, uint32_t dim, + const std::vector& query_h, uint64_t nq, + const std::vector>& gt, uint32_t k, const Cfg& c) { + std::vector ids(count); + std::iota(ids.begin(), ids.end(), (int64_t)0); + + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); + apply_cfg(bp, c); + gpu_ivf_pq_t index(count, dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + index.start(); + index.add_chunk_quantize(base_h.data(), count, -1, ids.data()); + index.build(); + + std::vector qcodes(nq * dim); + index.quantize_query(query_h.data(), nq, qcodes.data()); + ivf_pq_search_params_t sp = ivf_pq_search_params_default(); + sp.n_probes = c.n_probes; + auto res = index.search(qcodes.data(), nq, dim, k, sp); + double r = recall_at_k(res, gt, nq, k); + index.destroy(); + return r; +} + +// Count dims where uint8 query code != int8 query code + 128, using the same +// B source quantizer trained on the same base. 0 => monotonic L2-invariant. +template +int plus128_mismatches(const std::vector& base_b, const std::vector& query_b, + uint64_t count, uint32_t dim, const Cfg& c) { + std::vector ids(count); + std::iota(ids.begin(), ids.end(), (int64_t)0); + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); apply_cfg(bp, c); + gpu_ivf_pq_t qi(count, dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + qi.start(); qi.add_chunk_quantize(base_b.data(), count, -1, ids.data()); qi.build(); + gpu_ivf_pq_t qu(count, dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + qu.start(); qu.add_chunk_quantize(base_b.data(), count, -1, ids.data()); qu.build(); + std::vector ci(dim); std::vector cu(dim); + qi.quantize_query(query_b.data(), 1, ci.data()); + qu.quantize_query(query_b.data(), 1, cu.data()); + int mism = 0; + for (uint32_t j = 0; j < dim; ++j) if ((int)cu[j] != (int)ci[j] + 128) ++mism; + qi.destroy(); qu.destroy(); + return mism; +} + +} // namespace + +int main(int argc, char** argv) { + const char* base_path = argc > 1 ? argv[1] : kBase; + const char* qry_path = argc > 2 ? argv[2] : kQueries; + const char* gt_path = argc > 3 ? argv[3] : kGt; + + const uint32_t k = (uint32_t)env_u64("MOK", 10); + const uint64_t nq = env_u64("MOQ", 1000); + Cfg cfg{ (uint32_t)env_u64("MO_NLISTS", 1024), (uint32_t)env_u64("MO_NPROBES", 64), + (uint32_t)env_u64("MO_M", 0), (uint32_t)env_u64("MO_BITS", 0) }; // m/bits 0 => cuVS default + const bool do_f16 = env_u64("MO_F16", 0) != 0; + + printf("Loading base %s ...\n", base_path); + uint64_t count = 0; uint32_t dim = 0; + std::vector base = load_fbin(base_path, count, dim, -1); // FULL 1M (gt requires it) + printf(" base: count=%lu dim=%u\n", (unsigned long)count, dim); + + uint64_t qn_file = 0; uint32_t qdim = 0; + std::vector queries = load_fbin(qry_path, qn_file, qdim, (int64_t)nq); + if (qdim != dim) { printf("dim mismatch base=%u query=%u\n", dim, qdim); return 2; } + uint64_t use_nq = qn_file; + printf(" queries: nq=%lu dim=%u\n", (unsigned long)use_nq, qdim); + + auto gt = load_ibin_gt(gt_path, use_nq, k); + printf(" gt loaded for %lu queries, k=%u\n", (unsigned long)use_nq, k); + printf(" cfg: n_lists=%u n_probes=%u m=%u bits=%u (0=cuVS default)\n\n", + cfg.n_lists, cfg.n_probes, cfg.m, cfg.bits); + + const bool do_saveload = env_u64("MO_SAVELOAD", 0) != 0; + int failures = 0; + + // ---- f32 source ------------------------------------------------------- + { + double r_i8 = f32_recall(base, count, dim, queries, use_nq, gt, k, cfg); + double r_u8 = f32_recall(base, count, dim, queries, use_nq, gt, k, cfg); + int mism = plus128_mismatches(base, queries, count, dim, cfg); + + printf("\n=== f32 source (wiki_all_1M) ===\n"); + printf("[f32] +128 mismatches: %d / %u dims (0 => uint8==int8+128, L2-invariant)\n", mism, dim); + printf("[f32] f32->int8 recall@%u = %.4f\n", k, r_i8); + printf("[f32] f32->uint8 recall@%u = %.4f\n", k, r_u8); + printf("[f32] VERDICT: %s (gap=%.4f, mism=%d)\n", + (mism == 0 && r_i8 - r_u8 > 0.20) ? "uint8 COLLAPSE reproduced => cuVS uint8 bug" + : "no collapse (uint8 ~= int8)", + r_i8 - r_u8, mism); + if (mism != 0) { printf("[f32] FAIL: %d +128 mismatches\n", mism); ++failures; } + + // The real mo path: build -> save_dir -> load_dir -> search. + if (do_saveload) { + double sl_i8 = f32_recall_saveload(base, count, dim, queries, use_nq, gt, k, cfg, "/tmp/ivfpq_sl_i8"); + double sl_u8 = f32_recall_saveload(base, count, dim, queries, use_nq, gt, k, cfg, "/tmp/ivfpq_sl_u8"); + printf("[f32+saveload] f32->int8 recall@%u = %.4f (in-proc %.4f)\n", k, sl_i8, r_i8); + printf("[f32+saveload] f32->uint8 recall@%u = %.4f (in-proc %.4f)\n", k, sl_u8, r_u8); + printf("[f32+saveload] VERDICT: %s\n", + (sl_i8 - sl_u8 > 0.20) + ? "uint8 COLLAPSE after save/load => cuVS serialize/deserialize bug for uint8 ivf_pq" + : "uint8 survives save/load (~= int8)"); + } + } + + // ---- f16 source (optional; converts the 3GB float base -> half) ------- + if (do_f16) { + std::vector base_h(base.size()), query_h(queries.size()); + for (size_t i = 0; i < base.size(); ++i) base_h[i] = __float2half(base[i]); + for (size_t i = 0; i < queries.size(); ++i) query_h[i] = __float2half(queries[i]); + + double r_i8 = f16_recall(base_h, count, dim, query_h, use_nq, gt, k, cfg); + double r_u8 = f16_recall(base_h, count, dim, query_h, use_nq, gt, k, cfg); + int mism = plus128_mismatches(base_h, query_h, count, dim, cfg); + + printf("\n=== f16 source (wiki_all_1M) ===\n"); + printf("[f16] +128 mismatches: %d / %u dims (0 => uint8==int8+128, L2-invariant)\n", mism, dim); + printf("[f16] f16->int8 recall@%u = %.4f\n", k, r_i8); + printf("[f16] f16->uint8 recall@%u = %.4f\n", k, r_u8); + printf("[f16] VERDICT: %s (gap=%.4f, mism=%d)\n", + (mism == 0 && r_i8 - r_u8 > 0.20) ? "uint8 COLLAPSE reproduced => cuVS uint8 bug (f16 source)" + : "no collapse (uint8 ~= int8)", + r_i8 - r_u8, mism); + if (mism != 0) { printf("[f16] FAIL: %d +128 mismatches\n", mism); ++failures; } + } + + printf("\n%s (%d hard failure(s); inspect recall gaps above)\n", + failures == 0 ? "DONE" : "FAILED", failures); + return failures == 0 ? 0 : 1; +} diff --git a/pkg/bm25/plugin/compile/compile.go b/pkg/bm25/plugin/compile/compile.go new file mode 100644 index 0000000000000..5ca13c9daa32c --- /dev/null +++ b/pkg/bm25/plugin/compile/compile.go @@ -0,0 +1,332 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package compile implements the bm25 plugin's compile-layer (DDL) hooks. +// +// Phase 2 (sync-only): HandleCreateIndex / HandleReindex build the binary +// (WAND) index synchronously from the source rows via a single +// +// SELECT f.* FROM src CROSS APPLY bm25_create(params, cfg{FromSource}, pk, cols…) +// +// statement — the create TVF tokenizes each row in-Go and splits at +// max_index_capacity, so there is no postings round-trip. CDC (live DML sync), +// idxcron merge-compaction, and restore land in Phase 4. +package compile + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + bm25runtime "github.com/matrixorigin/matrixone/pkg/bm25/plugin/runtime" + "github.com/matrixorigin/matrixone/pkg/bm25/wand" + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/sqlquote" + compileplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/compile" + "github.com/matrixorigin/matrixone/pkg/pb/plan" +) + +// DefaultMaxIndexCapacity caps each tag=0 sub-index's doc count when the +// WITH max_index_capacity option is omitted. +const DefaultMaxIndexCapacity = int64(1000000) + +// actionBm25Reindex is the idxcron action key for bm25's scheduled compaction; +// must match the runtime plugin's SyncDescriptor.IdxcronAction. +const actionBm25Reindex = "bm25_reindex" + +// Compile-time interface check. +var _ compileplugin.Hooks = Hooks{} + +// Hooks implements plugin/compile.Hooks for bm25. +type Hooks struct{} + +func (Hooks) HandleCreateIndex(ctx compileplugin.CompileContext, indexDefs map[string]*plan.IndexDef) error { + // Gate CREATE INDEX ... USING bm25 behind experimental_bm25_index. Frontend-only: + // re-checking here (in addition to the framework gate in pkg/sql/compile/util.go) + // catches a flag toggled off since the original CREATE; the background CDC/reindex + // context may not surface the user's session value, so skip the check there. + if ctx.IsFrontend() { + if ok, err := ctx.IsExperimentalEnabled(bm25runtime.Bm25IndexFlag); err != nil { + return err + } else if !ok { + return moerr.NewInternalErrorNoCtx("experimental_bm25_index is not enabled") + } + } + return handleCreate(ctx, indexDefs, false) +} + +// HandleReindex — ALTER … REINDEX. Default (merge=false) rebuilds the whole +// binary index from the current source rows (a full re-tokenize). merge=true +// runs incremental compaction: fold the tag=1 CdcTail into the tag=0 base + +// tiered-merge the base, without re-tokenizing. A REINDEX is always a SYNC +// rebuild (forceSync=true) regardless of the index's async param — mirrors the +// vector plugins / the retrieval index. +func (Hooks) HandleReindex(ctx compileplugin.CompileContext, indexDefs map[string]*plan.IndexDef, _, merge bool) error { + if merge { + return handleMergeCompact(ctx, indexDefs) + } + return handleCreate(ctx, indexDefs, true) +} + +// handleCreate creates the storage+metadata hidden tables and builds the tag=0 +// base. The `async` param gates only the initial BUILD (sync vs async), NOT DML +// (bm25 DML is always CDC): with async=false (default) or a REINDEX (forceSync) +// the base is built synchronously inline; with async=true on a fresh CREATE the +// build runs at CDC-task start via the InitSQL. Idempotent — it clears any prior +// tag=1 tail first, so the sync path doubles as the REINDEX rebuild body. +func handleCreate(ctx compileplugin.CompileContext, indexDefs map[string]*plan.IndexDef, forceSync bool) error { + storeDef, ok := indexDefs[catalog.Bm25Index_TblType_Storage] + if !ok { + return moerr.NewInternalErrorNoCtx("bm25 storage index definition not found") + } + metaDef, ok := indexDefs[catalog.Bm25Index_TblType_Metadata] + if !ok { + return moerr.NewInternalErrorNoCtx("bm25 metadata index definition not found") + } + + // 1. create the hidden tables. + if info := ctx.IndexInfo(); info != nil { + for _, table := range info.GetIndexTables() { + if err := ctx.BuildIndexTable(table); err != nil { + return err + } + } + } + + originalTableDef := ctx.OriginalTableDef() + qryDatabase := ctx.QryDatabase() + + // 2. CCPR: skip data population when this is a CCPR task transaction on a + // publication-subscribed table (the index data syncs via CCPR instead). + if ctx.IsCCPRTaskTransaction() && ctx.IsTableFromPublication(originalTableDef) { + return nil + } + + capacity, err := resolveBm25Capacity(storeDef.IndexAlgoParams) + if err != nil { + return err + } + sinkerType := ctx.SinkerTypeFromAlgo(catalog.MoIndexBm25Algo.ToString()) + + // The `async` param gates the BUILD only (never DML — bm25 is AlwaysAsync). + async, err := catalog.IsIndexAsync(storeDef.IndexAlgoParams) + if err != nil { + return err + } + + // 3. Drop any prior CDC task first — on REINDEX re-entry it would otherwise + // survive at its old watermark and replay history over the freshly built state. + if err = ctx.DropIndexCdcTask(originalTableDef, qryDatabase, originalTableDef.Name, storeDef.IndexName); err != nil { + return err + } + + // 4a. ASYNC CREATE (async=true, and not a REINDEX): register the CDC task with a + // non-empty InitSQL that clears + builds the tag=0 base from source at task start + // (startFromNow=false → the CDC re-arms at the post-build watermark). CREATE returns + // immediately; the build happens in the background. A REINDEX always takes 4b. + if async && !forceSync { + initSQL, err := genBm25InitSQL(originalTableDef, storeDef, metaDef, qryDatabase, capacity) + if err != nil { + return err + } + if err = ctx.CreateIndexCdcTask(qryDatabase, originalTableDef.Name, + originalTableDef.TblId, storeDef.IndexName, sinkerType, false, initSQL, originalTableDef); err != nil { + return err + } + return registerBm25Idxcron(ctx, storeDef, qryDatabase, originalTableDef) + } + + // 4b. SYNC build (default, or REINDEX forceSync): clear any prior tag=1 tail + // (no-op on a fresh CREATE; a real clear on REINDEX), then build the tag=0 base + // from source inline, then register the CDC task from now (startFromNow=true) — + // the inline build covers the pre-create rows, post-create DML flows into the tail. + cfg := wand.TableConfig{DbName: qryDatabase, IndexTable: storeDef.IndexTableName, MetadataTable: metaDef.IndexTableName} + for _, sql := range wand.DeleteTailSqls(cfg) { + if err = ctx.RunSql(sql); err != nil { + return err + } + } + buildSQLs, err := genBm25BuildFromSourceSQL(originalTableDef, storeDef, metaDef, qryDatabase, capacity) + if err != nil { + return err + } + for _, sql := range buildSQLs { + if err = ctx.RunSql(sql); err != nil { + return err + } + } + if err = ctx.CreateIndexCdcTask(qryDatabase, originalTableDef.Name, + originalTableDef.TblId, storeDef.IndexName, sinkerType, true, "", originalTableDef); err != nil { + return err + } + return registerBm25Idxcron(ctx, storeDef, qryDatabase, originalTableDef) +} + +// registerBm25Idxcron registers the idxcron scheduled-compaction task. Skipped on +// a background re-entry (IdxcronMetadata returns nil) so the existing task row +// persists. +func registerBm25Idxcron(ctx compileplugin.CompileContext, storeDef *plan.IndexDef, qryDatabase string, originalTableDef *plan.TableDef) error { + metadata, err := Hooks{}.IdxcronMetadata(ctx) + if err != nil { + return err + } + if len(metadata) == 0 { + return nil + } + return ctx.RegisterIdxcronUpdate(originalTableDef.TblId, qryDatabase, + originalTableDef.Name, storeDef.IndexName, actionBm25Reindex, metadata) +} + +// genBm25InitSQL builds the ISCP InitSQL — a JSON array of statements run at CDC +// task start — that clears any prior tag=1 tail and builds the tag=0 base from the +// SOURCE rows (the async-CREATE analogue of the inline sync build). +func genBm25InitSQL(originalTableDef *plan.TableDef, storeDef, metaDef *plan.IndexDef, qryDatabase string, capacity int64) (string, error) { + cfg := wand.TableConfig{DbName: qryDatabase, IndexTable: storeDef.IndexTableName, MetadataTable: metaDef.IndexTableName} + sqls := wand.DeleteTailSqls(cfg) + buildSQLs, err := genBm25BuildFromSourceSQL(originalTableDef, storeDef, metaDef, qryDatabase, capacity) + if err != nil { + return "", err + } + sqls = append(sqls, buildSQLs...) + js, err := json.Marshal(sqls) + if err != nil { + return "", err + } + return string(js), nil +} + +// RestoreInitSQL rebuilds the bm25 index from the restored/cloned rows. It runs +// post-commit as the re-armed CDC's InitSQL (startFromNow=true), so it sees the +// committed clone and re-arms the CDC at the post-clone watermark. The rebuild +// discards the block-cloned tag=0 base (which would otherwise be doubled). +func (Hooks) RestoreInitSQL(ctx compileplugin.CompileContext, indexDefs map[string]*plan.IndexDef) (bool, string, error) { + storeDef, ok := indexDefs[catalog.Bm25Index_TblType_Storage] + if !ok { + return false, "", moerr.NewInternalErrorNoCtx("bm25 storage index definition not found") + } + return true, fmt.Sprintf("ALTER TABLE `%s`.`%s` ALTER REINDEX `%s` BM25 FORCE_SYNC", + ctx.QryDatabase(), ctx.OriginalTableDef().Name, storeDef.IndexName), nil +} + +// handleMergeCompact runs incremental compaction: fold the tag=1 CdcTail into +// the tag=0 base + tiered-merge the base, via the standalone bm25_compact TVF — +// no re-tokenize. Capacity comes from the PERSISTED algo_params (pinned at +// CREATE) so a manual MERGE never depends on the triggering session. +func handleMergeCompact(ctx compileplugin.CompileContext, indexDefs map[string]*plan.IndexDef) error { + storeDef, ok := indexDefs[catalog.Bm25Index_TblType_Storage] + if !ok { + return moerr.NewInternalErrorNoCtx("bm25 storage index definition not found") + } + metaDef, ok := indexDefs[catalog.Bm25Index_TblType_Metadata] + if !ok { + return moerr.NewInternalErrorNoCtx("bm25 metadata index definition not found") + } + capacity, err := resolveBm25Capacity(storeDef.IndexAlgoParams) + if err != nil { + return err + } + sql := fmt.Sprintf("SELECT * FROM bm25_compact(%s, %s, %s, %s) AS f", + sqlquote.String(ctx.QryDatabase()), + sqlquote.String(storeDef.IndexTableName), + sqlquote.String(metaDef.IndexTableName), + sqlquote.String(strconv.FormatInt(capacity, 10))) + return ctx.RunSql(sql) +} + +// ValidateReindexParams merges the reindex-time options bm25 honors on a rebuild +// (only max_index_capacity) into the persisted params, and rejects any other +// option (e.g. a vector index's `lists`) with a clear error. +func (Hooks) ValidateReindexParams(old map[string]string, alter compileplugin.ReindexParamUpdate) (map[string]string, error) { + merged := make(map[string]string, len(old)+1) + for k, v := range old { + merged[k] = v + } + for k, v := range alter.Params { + if k != catalog.IndexAlgoParamMaxIndexCapacity { + return nil, moerr.NewNotSupportedNoCtxf("bm25 reindex does not support option %q (only max_index_capacity)", k) + } + merged[k] = v + } + return merged, nil +} + +func (Hooks) HandleDropIndex(compileplugin.CompileContext, map[string]*plan.IndexDef) error { + // CDC-task teardown lands in Phase 4; the generic hidden-table deletion the + // SQL layer performs is sufficient for a sync-only index. + return nil +} + +// IdxcronMetadata — bm25's scheduled compaction reads max_index_capacity from +// the index's PERSISTED algo_params (not a session var), so the metadata blob +// carries no captured vars; it only needs to be non-nil so the idxcron task +// registers. In a background re-entry (not frontend) return nil so the existing +// task's row persists (mirrors BuildIdxcronMetadata's frontend gate). +func (Hooks) IdxcronMetadata(ctx compileplugin.CompileContext) ([]byte, error) { + if !ctx.IsFrontend() { + return nil, nil + } + return []byte("{}"), nil +} + +// resolveBm25Capacity reads max_index_capacity from the index's algo_params, +// defaulting when the WITH option was omitted. +func resolveBm25Capacity(algoParams string) (int64, error) { + flat, err := catalog.IndexParamsStringToMap(algoParams) + if err != nil { + return 0, err + } + if v, ok := flat[catalog.IndexAlgoParamMaxIndexCapacity]; ok && v != "" { + n, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return 0, err + } + if n > 0 { + return n, nil + } + } + return DefaultMaxIndexCapacity, nil +} + +// genBm25BuildFromSourceSQL builds the binary index straight from the SOURCE +// table in one statement: SELECT f.* FROM src CROSS APPLY bm25_create(params, +// cfg{FromSource}, pk, cols…). The create TVF tokenizes each row in-Go (jieba) +// and Add's the tokens; cfg carries FromSource=true and max_index_capacity. +func genBm25BuildFromSourceSQL(originalTableDef *plan.TableDef, storeDef, metaDef *plan.IndexDef, qryDatabase string, capacity int64) ([]string, error) { + const srcAlias = "src" + cfg := wand.TableConfig{ + DbName: qryDatabase, + IndexTable: storeDef.IndexTableName, + MetadataTable: metaDef.IndexTableName, + Capacity: capacity, + FromSource: true, + } + cfgbytes, err := json.Marshal(cfg) + if err != nil { + return nil, err + } + cols := make([]string, 0, len(storeDef.Parts)) + for _, p := range storeDef.Parts { + cols = append(cols, sqlquote.QualifiedIdent(srcAlias, p)) + } + sql := fmt.Sprintf("SELECT f.* FROM %s AS %s CROSS APPLY bm25_create(%s, %s, %s, %s) AS f", + sqlquote.QualifiedIdent(qryDatabase, originalTableDef.Name), + sqlquote.Ident(srcAlias), + sqlquote.String(storeDef.IndexAlgoParams), + sqlquote.String(string(cfgbytes)), + sqlquote.QualifiedIdent(srcAlias, originalTableDef.Pkey.PkeyColName), + strings.Join(cols, ", ")) + return []string{sql}, nil +} diff --git a/pkg/bm25/plugin/idxcron/idxcron.go b/pkg/bm25/plugin/idxcron/idxcron.go new file mode 100644 index 0000000000000..7cc8218234fb7 --- /dev/null +++ b/pkg/bm25/plugin/idxcron/idxcron.go @@ -0,0 +1,180 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package idxcron is the bm25 index's idxcron hook. A bm25 index participates +// in scheduled compaction (SyncDescriptor.IdxcronAction = "bm25_reindex"); the +// cron executor runs `ALTER … REINDEX … BM25 [MERGE] FORCE_SYNC` when Updatable +// returns true — MERGE folds the tag=1 CdcTail into the tag=0 base, or a full +// REBUILD from source once the dead-doc fraction is high (ReindexOption). +package idxcron + +import ( + "fmt" + "os" + "strconv" + "time" + + "github.com/matrixorigin/matrixone/pkg/bm25/wand" + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/sqlquote" + "github.com/matrixorigin/matrixone/pkg/container/vector" + idxcronplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/idxcron" + "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" +) + +// TailChunkThreshold gates the scheduled rebuild on tag=1 CdcTail growth: fire only +// once the tail has accumulated at least this many chunk rows since the last reindex. +// A chunk is ≤ MaxChunkSize (64 KB), so this bounds the delta the search path must +// load+reconcile on top of tag=0 before a compaction folds it in. Default 1024; a +// dev/test deploy can lower it via env MO_IDXCRON_BM25_TAIL_THRESHOLD (e.g. 1) to +// observe a rebuild without generating ~64 MB of tail. (Stage 1 is a full reindex from +// source; Stage 2 will swap the reindex body for tiered merge without changing this gate.) +var TailChunkThreshold = func() int64 { + if v := os.Getenv("MO_IDXCRON_BM25_TAIL_THRESHOLD"); v != "" { + if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 { + return n + } + } + return 1024 +}() + +type Hooks struct{} + +var _ idxcronplugin.Hooks = Hooks{} +var _ idxcronplugin.ReindexOptioner = Hooks{} + +// Updatable fires the scheduled reindex when the tag=1 CdcTail has grown past +// TailChunkThreshold. The framework has already applied the auto_update / hour / +// cadence gates; this adds the WAND-specific "is there enough tail to bother" +// check. A non-retrieval index has no WAND store, so it is skipped here too (a +// defensive backstop — such an index should never have a task registered). +func (Hooks) Updatable(in idxcronplugin.UpdatableInput) (bool, string, error) { + // Cadence backstop (mirrors CuvsUpdatable): skip if rebuilt within the interval. + if in.LastUpdateAt != nil { + last := time.Unix(in.LastUpdateAt.Unix(), 0) + if last.Add(in.Interval).After(time.Now()) { + return false, fmt.Sprintf("within reindex interval (last %s + %v)", + last.Format("2006-01-02 15:04:05"), in.Interval), nil + } + } + + // Locate the retrieval index's WAND chunk-store table. Absent ⇒ not a + // retrieval index (postings/ngram) ⇒ nothing to compact. + var storageTbl string + for _, idx := range in.TableDef.Indexes { + if idx.IndexName == in.IndexName && + idx.IndexAlgoTableType == catalog.Bm25Index_TblType_Storage { + storageTbl = idx.IndexTableName + break + } + } + if storageTbl == "" { + return false, "not a retrieval index (no WAND store)", nil + } + + cfg := wand.TableConfig{DbName: in.TableDef.DbName, IndexTable: storageTbl} + count, err := wand.CountTailChunks(in.Sqlproc, cfg) + if err != nil { + return false, "", err + } + logutil.Infof("[idxcron][wand] Updatable index=%s: tag=1 tail chunks=%d threshold=%d", + in.IndexName, count, TailChunkThreshold) + if count < TailChunkThreshold { + return false, fmt.Sprintf("tag=1 tail chunks %d < threshold %d", count, TailChunkThreshold), nil + } + return true, "", nil +} + +// RebuildDeadPct: when the percentage of dead (deleted-but-not-yet-reclaimed) docs in the +// base exceeds this, the scheduled compaction fires a full REBUILD instead of an incremental +// MERGE — a rebuild reclaims all dead docs AND their tombstones at once, cheaper net than +// folding forever. Hardcoded 30: the alternative (a global merge to reclaim incrementally) +// would load the whole base = O(corpus) resident = OOM, so REBUILD is the accepted reclaim path. +const RebuildDeadPct = 30 + +// ReindexOption picks the scheduled reindex mode (overriding the descriptor's fixed "MERGE"): +// "MERGE" (incremental fold + tiered compaction) normally, or "" (full REBUILD) once the +// dead-doc fraction — 1 - liveSourceRows/baseDocs — exceeds RebuildDeadPct. Cheap: one +// COUNT(*) on the source table + SUM(nrow) on the metadata table (no postings loaded). +func (Hooks) ReindexOption(in idxcronplugin.UpdatableInput) (string, error) { + var metaTbl string + for _, idx := range in.TableDef.Indexes { + if idx.IndexName == in.IndexName && + idx.IndexAlgoTableType == catalog.Bm25Index_TblType_Metadata { + metaTbl = idx.IndexTableName + break + } + } + if metaTbl == "" { + return "MERGE", nil // not a retrieval index / no metadata table — MERGE is a harmless default + } + cfg := wand.TableConfig{DbName: in.TableDef.DbName, MetadataTable: metaTbl} + baseDocs, err := wand.SumBaseNrow(in.Sqlproc, cfg) + if err != nil { + return "", err + } + if baseDocs == 0 { + return "MERGE", nil // no tag=0 base yet (corpus still in the tail) — fold it in + } + liveRows, err := countSourceRows(in.Sqlproc, in.TableDef.DbName, in.TableDef.Name) + if err != nil { + return "", err + } + opt := reindexOptionForCounts(liveRows, baseDocs) + deadPct := int64(0) + if baseDocs > 0 { + deadPct = 100 - liveRows*100/baseDocs + } + logutil.Infof("[idxcron][wand] ReindexOption index=%s: liveRows=%d baseDocs=%d dead=%d%% threshold=%d%% -> %s", + in.IndexName, liveRows, baseDocs, deadPct, RebuildDeadPct, optLabel(opt)) + return opt, nil +} + +// reindexOptionForCounts is the pure decision: "" (full REBUILD) once the dead-doc percentage +// exceeds RebuildDeadPct, else "MERGE" (incremental). Integer arithmetic so the boundary is +// exact (dead% > 30 ⟺ live% < 70 ⟺ liveRows*100 < baseDocs*(100-30)); float 1-live/base tips +// at the exact boundary due to rounding. +func reindexOptionForCounts(liveRows, baseDocs int64) string { + if baseDocs <= 0 { + return "MERGE" + } + if liveRows*100 < baseDocs*(100-RebuildDeadPct) { + return "" // dead % > RebuildDeadPct → full REBUILD + } + return "MERGE" +} + +func optLabel(opt string) string { + if opt == "" { + return "REBUILD" + } + return opt +} + +// countSourceRows returns the source table's live row count (= live docs, since a DELETE +// removes the source row). Compared to SumBaseNrow to estimate the base's dead-doc fraction. +func countSourceRows(sqlproc *sqlexec.SqlProcess, db, table string) (int64, error) { + res, err := sqlexec.RunSql(sqlproc, fmt.Sprintf("SELECT COUNT(*) FROM %s", sqlquote.QualifiedIdent(db, table))) + if err != nil { + return 0, err + } + defer res.Close() + for _, bat := range res.Batches { + if bat != nil && bat.RowCount() > 0 { + return vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[0], 0), nil + } + } + return 0, nil +} diff --git a/pkg/bm25/plugin/iscp/iscp.go b/pkg/bm25/plugin/iscp/iscp.go new file mode 100644 index 0000000000000..8dc470b207085 --- /dev/null +++ b/pkg/bm25/plugin/iscp/iscp.go @@ -0,0 +1,47 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package iscp is bm25's ISCP hook layer. A bm25 index is model-building — +// its CDC sinker builds tag=1 CdcTail delta frames (the WAND writer / +// consumer loop in pkg/iscp), NOT SQL text. Unlike the classic fulltext +// plugin there is no parser branch: a bm25 index is always WAND. +// +// Registered from pkg/indexplugin/iscp/import.go via iscp.Register. +package iscp + +import ( + "context" + + "github.com/matrixorigin/matrixone/pkg/catalog" + iscppkg "github.com/matrixorigin/matrixone/pkg/iscp" + "github.com/matrixorigin/matrixone/pkg/pb/plan" +) + +func init() { + iscppkg.Register(catalog.MoIndexBm25Algo.ToString(), Hooks{}) +} + +// Hooks implements iscp.Hooks for bm25. +type Hooks struct{} + +var _ iscppkg.Hooks = Hooks{} + +func (Hooks) NewSqlWriter(jobID iscppkg.JobID, info *iscppkg.ConsumerInfo, + tabledef *plan.TableDef, indexdefs []*plan.IndexDef) (iscppkg.IndexSqlWriter, error) { + return iscppkg.NewWandSqlWriter(catalog.MoIndexBm25Algo.ToString(), jobID, info, tabledef, indexdefs) +} + +func (Hooks) Run(c *iscppkg.IndexConsumer, ctx context.Context, errch chan error, r iscppkg.DataRetriever) { + iscppkg.RunWand(c, ctx, errch, r) +} diff --git a/pkg/bm25/plugin/plan/plan.go b/pkg/bm25/plugin/plan/plan.go new file mode 100644 index 0000000000000..5ca35d67bfa19 --- /dev/null +++ b/pkg/bm25/plugin/plan/plan.go @@ -0,0 +1,43 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package plan implements the bm25 plugin's plan-layer hooks. +// +// bm25 is queried through MATCH(col) AGAINST('query') — the fulltext-style +// query surface — NOT the vector ORDER BY LIMIT rewrite. So the +// vector ApplyForSort/CanApply hooks are inert (return false/no-op); the +// MATCH → bm25 rewrite lives in pkg/sql/plan/apply_indices_bm25.go (Phase 3). +// The hidden-table schema builder is BuildSecondaryIndexDefs (schema.go). +package plan + +import ( + planplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/plan" +) + +// Compile-time interface check. +var _ planplugin.Hooks = Hooks{} + +// Hooks implements plugin/plan.Hooks for bm25. +type Hooks struct{} + +// CanApply — bm25 does not participate in ORDER BY LIMIT rewrites +// (its query surface is MATCH/AGAINST), so it never applies for sort. +func (Hooks) CanApply(planplugin.PlanBuilder, *planplugin.VectorSortContext, *planplugin.MultiTableIndexRef) (bool, error) { + return false, nil +} + +// ApplyForSort — no-op for bm25 (see CanApply). +func (Hooks) ApplyForSort(_ planplugin.PlanBuilder, _ *planplugin.VectorSortContext, _ *planplugin.MultiTableIndexRef, nodeID int32, _ planplugin.ApplyForSortOpts) (int32, bool, error) { + return nodeID, false, nil +} diff --git a/pkg/bm25/plugin/plan/plan_test.go b/pkg/bm25/plugin/plan/plan_test.go new file mode 100644 index 0000000000000..551043f47e8d9 --- /dev/null +++ b/pkg/bm25/plugin/plan/plan_test.go @@ -0,0 +1,185 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// CPU-side unit coverage for plan.go (inert CanApply/ApplyForSort) and schema.go +// (BuildSecondaryIndexDefs / BuildFullTextIndexDefs). bm25's BVTs run via mo-tester +// (integration), which does not contribute to Go per-package line coverage, so +// without these the package reads ~0% in CI. The pk-type validation tests below are +// also the regression guard for the "unsupported pk silently aborts CDC" fix. +package plan + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/container/types" + planplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/plan" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" +) + +// init wires the planplugin helper vars BuildSecondaryIndexDefs calls on the happy +// path. Production wires them in pkg/sql/plan's init; importing that from a plugin +// test would be a cycle, so we substitute shallow stand-ins (only the shape of the +// returned value matters here). +func init() { + if planplugin.CreateIndexDef == nil { + planplugin.CreateIndexDef = func(_ planplugin.CompilerContext, _ *tree.Index, indexTableName, indexAlgoTableType string, indexParts []string, _ bool) (*plan.IndexDef, error) { + return &plan.IndexDef{ + IndexTableName: indexTableName, + IndexAlgoTableType: indexAlgoTableType, + Parts: indexParts, + }, nil + } + } + if planplugin.MakeHiddenColDefByName == nil { + planplugin.MakeHiddenColDefByName = func(name string) *plan.ColDef { + return &plan.ColDef{Name: name, Typ: plan.Type{Id: int32(types.T_varchar)}} + } + } +} + +type stubCompilerContext struct{ ctx context.Context } + +func (c stubCompilerContext) GetContext() context.Context { return c.ctx } +func (c stubCompilerContext) ResolveVariable(string, bool, bool) (interface{}, error) { + return nil, nil +} + +var _ planplugin.CompilerContext = stubCompilerContext{} + +func newStubCompilerContext() stubCompilerContext { + return stubCompilerContext{ctx: context.Background()} +} + +// bm25ColMap returns a colMap with an int64 pk column and a text column — the shape +// BuildSecondaryIndexDefs expects on the happy path. +func bm25ColMap(pkName, txtName string, pkType types.T) map[string]*plan.ColDef { + return map[string]*plan.ColDef{ + pkName: {Name: pkName, Typ: plan.Type{Id: int32(pkType)}}, + txtName: {Name: txtName, Typ: plan.Type{Id: int32(types.T_text)}}, + } +} + +func indexOn(colName string) *tree.Index { + un := tree.NewUnresolvedName(tree.NewCStr(colName, 0)) + return &tree.Index{KeyParts: []*tree.KeyPart{{ColName: un}}} +} + +// --- plan.go: inert vector hooks ------------------------------------------- + +func TestCanApply_Inert(t *testing.T) { + ok, err := Hooks{}.CanApply(nil, &planplugin.VectorSortContext{}, &planplugin.MultiTableIndexRef{}) + require.NoError(t, err) + require.False(t, ok, "bm25 must not apply for ORDER BY LIMIT rewrites") +} + +func TestApplyForSort_Inert(t *testing.T) { + id, changed, err := Hooks{}.ApplyForSort(nil, &planplugin.VectorSortContext{}, &planplugin.MultiTableIndexRef{}, 42, planplugin.ApplyForSortOpts{}) + require.NoError(t, err) + require.False(t, changed) + require.Equal(t, int32(42), id, "ApplyForSort must return the node id unchanged") +} + +// --- schema.go: BuildSecondaryIndexDefs error paths ------------------------ + +func TestBuildSecondaryIndexDefs_MultiColumn(t *testing.T) { + idx := indexOn("body") + idx.KeyParts = append(idx.KeyParts, &tree.KeyPart{ColName: tree.NewUnresolvedName(tree.NewCStr("body2", 0))}) + _, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), idx, bm25ColMap("id", "body", types.T_int64), nil, "id") + require.Error(t, err) +} + +func TestBuildSecondaryIndexDefs_ColNotExist(t *testing.T) { + _, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOn("nope"), bm25ColMap("id", "body", types.T_int64), nil, "id") + require.Error(t, err) +} + +func TestBuildSecondaryIndexDefs_NotTextColumn(t *testing.T) { + colMap := bm25ColMap("id", "body", types.T_int64) + colMap["body"].Typ.Id = int32(types.T_int64) + _, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOn("body"), colMap, nil, "id") + require.Error(t, err) +} + +func TestBuildSecondaryIndexDefs_DuplicateColumn(t *testing.T) { + existed := []*plan.IndexDef{{ + IndexAlgo: catalog.MoIndexBm25Algo.ToString(), + Parts: []string{"body"}, + }} + _, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOn("body"), bm25ColMap("id", "body", types.T_int64), existed, "id") + require.Error(t, err) +} + +// TestBuildSecondaryIndexDefs_UnsupportedPk: a single-column pk of a type encodePk +// cannot round-trip must be rejected at CREATE (else the CDC sink silently aborts). +func TestBuildSecondaryIndexDefs_UnsupportedPk(t *testing.T) { + for _, pk := range []types.T{ + types.T_int8, types.T_int16, types.T_uint8, types.T_uint16, + types.T_float32, types.T_float64, types.T_bit, types.T_bool, + } { + colMap := bm25ColMap("id", "body", pk) + _, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOn("body"), colMap, nil, "id") + require.Error(t, err, "pk type %s must be rejected at CREATE", pk) + } +} + +// TestBuildSecondaryIndexDefs_SupportedPk: every pk type encodePk handles is accepted +// (mirror of encodePk's switch — keep the two in lockstep). +func TestBuildSecondaryIndexDefs_SupportedPk(t *testing.T) { + for _, pk := range []types.T{ + types.T_int64, types.T_uint64, types.T_int32, types.T_uint32, + types.T_varchar, types.T_char, types.T_text, types.T_datalink, + types.T_uuid, + types.T_date, types.T_datetime, types.T_time, types.T_timestamp, + types.T_decimal64, types.T_decimal128, + } { + colMap := bm25ColMap("id", "body", pk) + idxDefs, tblDefs, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOn("body"), colMap, nil, "id") + require.NoError(t, err, "pk type %s must be accepted", pk) + require.Len(t, idxDefs, 2) + require.Len(t, tblDefs, 2) + } +} + +// TestBuildSecondaryIndexDefs_CompositePkOK: a composite pk is delivered as the +// packed CPrimaryKey varchar, which is NOT in colMap; validation must skip it (not +// reject) so composite-pk tables can be bm25-indexed. +func TestBuildSecondaryIndexDefs_CompositePkOK(t *testing.T) { + colMap := bm25ColMap("id", "body", types.T_int64) + idxDefs, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOn("body"), colMap, nil, catalog.CPrimaryKeyColName) + require.NoError(t, err) + require.Len(t, idxDefs, 2) +} + +func TestBuildSecondaryIndexDefs_OK(t *testing.T) { + idxDefs, tblDefs, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOn("body"), bm25ColMap("id", "body", types.T_int64), nil, "id") + require.NoError(t, err) + require.Len(t, idxDefs, 2) + require.Len(t, tblDefs, 2) + require.Equal(t, catalog.Bm25Index_TblType_Storage, tblDefs[0].TableType) + require.Equal(t, catalog.Bm25Index_TblType_Metadata, tblDefs[1].TableType) + require.NotNil(t, tblDefs[0].Pkey) + require.NotNil(t, tblDefs[1].Pkey) +} + +// --- schema.go: BuildFullTextIndexDefs ------------------------------------- + +func TestBuildFullTextIndexDefs_Unsupported(t *testing.T) { + _, _, err := Hooks{}.BuildFullTextIndexDefs(newStubCompilerContext(), nil, nil, nil, "") + require.Error(t, err) +} diff --git a/pkg/bm25/plugin/plan/schema.go b/pkg/bm25/plugin/plan/schema.go new file mode 100644 index 0000000000000..92ab3dc1f4844 --- /dev/null +++ b/pkg/bm25/plugin/plan/schema.go @@ -0,0 +1,176 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plan + +import ( + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" + planplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/plan" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" + "github.com/matrixorigin/matrixone/pkg/sql/util" +) + +// bm25TextColumn reports whether a column type can be bm25-indexed +// (text-ish types, same set the classic fulltext index accepts). +func bm25TextColumn(id int32) bool { + return id == int32(types.T_text) || id == int32(types.T_char) || + id == int32(types.T_varchar) || id == int32(types.T_json) || + id == int32(types.T_datalink) +} + +// bm25SupportedPkType reports whether a single-column primary key of this type can +// be encoded/decoded by the WAND engine (wand.encodePk / decodePk). It MUST mirror +// encodePk's switch exactly — a pk type accepted at CREATE but not by encodePk would +// silently abort the CDC sink later (the index would stop updating with no visible +// error). A composite primary key is delivered as the packed CPrimaryKey varchar, +// which is covered by the varlena case, so composite pks are always supported. +func bm25SupportedPkType(id int32) bool { + switch types.T(id) { + case types.T_int64, types.T_uint64, types.T_int32, types.T_uint32, + types.T_varchar, types.T_char, types.T_text, types.T_datalink, + types.T_binary, types.T_varbinary, types.T_blob, types.T_json, + types.T_uuid, + types.T_date, types.T_datetime, types.T_time, types.T_timestamp, + types.T_decimal64, types.T_decimal128: + return true + default: + return false + } +} + +// BuildSecondaryIndexDefs constructs the bm25 index def + its two hidden tables +// (storage + metadata) from CREATE INDEX ... USING bm25. bm25 parses to +// *tree.Index and is dispatched here (the vector-plugin path). The two tables +// mirror the HNSW storage/metadata layout: the storage table holds the chunked +// binary (WAND) index blobs, the metadata table one row per sub-index. There is +// no postings table — bm25 builds directly from the source rows. +func (Hooks) BuildSecondaryIndexDefs( + ctx planplugin.CompilerContext, + indexInfo *tree.Index, + colMap map[string]*plan.ColDef, + existedIndexes []*plan.IndexDef, + pkeyName string, +) ([]*plan.IndexDef, []*plan.TableDef, error) { + + // 0. Validate: single text/varchar column, no duplicate bm25 on it. + if len(indexInfo.KeyParts) != 1 { + return nil, nil, moerr.NewNotSupported(ctx.GetContext(), "bm25 index does not support multiple columns") + } + name := indexInfo.KeyParts[0].ColName.ColName() + indexParts := []string{name} + col, ok := colMap[name] + if !ok { + return nil, nil, moerr.NewInvalidInputf(ctx.GetContext(), "column '%s' is not exist", indexInfo.KeyParts[0].ColName.ColNameOrigin()) + } + if !bm25TextColumn(col.Typ.Id) { + return nil, nil, moerr.NewNotSupported(ctx.GetContext(), "bm25 index only supports CHAR/VARCHAR/TEXT/JSON/DATALINK columns") + } + for _, existed := range existedIndexes { + if existed.IndexAlgo == catalog.MoIndexBm25Algo.ToString() && len(existed.Parts) > 0 && existed.Parts[0] == name { + return nil, nil, moerr.NewNotSupported(ctx.GetContext(), "Multiple bm25 indexes are not allowed to use the same column") + } + } + // Reject a primary key whose type the WAND engine cannot encode, at CREATE time + // rather than letting the CDC sink silently abort later. A single-column pk is in + // colMap; a composite pk uses the packed CPrimaryKey varchar (not in colMap, and + // always encodable), so only validate when the pk column is present here. + if pkCol, ok := colMap[pkeyName]; ok && !bm25SupportedPkType(pkCol.Typ.Id) { + return nil, nil, moerr.NewNotSupportedf(ctx.GetContext(), + "bm25 index does not support a primary key of type %s; use an integer, decimal, string, uuid, date/time, or timestamp primary key", + types.T(pkCol.Typ.Id).String()) + } + + // 1. storage (chunk) table: ( index_id VARCHAR, chunk_id INT64, data BLOB, + // tag INT64, PRIMARY KEY (index_id, chunk_id) ) + storeName, err := util.BuildIndexTableName(ctx.GetContext(), false) + if err != nil { + return nil, nil, err + } + storeIdx, err := planplugin.CreateIndexDef(ctx, indexInfo, storeName, catalog.Bm25Index_TblType_Storage, indexParts, false) + if err != nil { + return nil, nil, err + } + storeTbl := &plan.TableDef{ + Name: storeName, + TableType: catalog.Bm25Index_TblType_Storage, + Cols: []*plan.ColDef{ + {Name: catalog.Bm25Index_TblCol_Storage_Index_Id, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_varchar), Width: 128}, Default: &plan.Default{}}, + {Name: catalog.Bm25Index_TblCol_Storage_Chunk_Id, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_int64)}, Default: &plan.Default{}}, + {Name: catalog.Bm25Index_TblCol_Storage_Data, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_blob), Width: 65536}, Default: &plan.Default{}}, + {Name: catalog.Bm25Index_TblCol_Storage_Tag, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_int64)}, Default: &plan.Default{}}, + }, + } + storePk := planplugin.MakeHiddenColDefByName(catalog.CPrimaryKeyColName) + storePk.Alg = plan.CompressType_Lz4 + storePk.Primary = true + storeTbl.Cols = append(storeTbl.Cols, storePk) + storeTbl.Pkey = &plan.PrimaryKeyDef{ + Names: []string{catalog.Bm25Index_TblCol_Storage_Index_Id, catalog.Bm25Index_TblCol_Storage_Chunk_Id}, + PkeyColName: catalog.CPrimaryKeyColName, + CompPkeyCol: storeTbl.Cols[3], // tag col, mirrors HNSW storage layout + } + storeTbl.Defs = append(storeTbl.Defs, &plan.TableDef_DefType{ + Def: &plan.TableDef_DefType_Properties{Properties: &plan.PropertiesDef{Properties: []*plan.Property{ + {Key: catalog.SystemRelAttr_Kind, Value: catalog.Bm25Index_TblType_Storage}, + }}}, + }) + + // 2. metadata table: one row per sub-index. + metaName, err := util.BuildIndexTableName(ctx.GetContext(), false) + if err != nil { + return nil, nil, err + } + metaIdx, err := planplugin.CreateIndexDef(ctx, indexInfo, metaName, catalog.Bm25Index_TblType_Metadata, indexParts, false) + if err != nil { + return nil, nil, err + } + metaTbl := &plan.TableDef{ + Name: metaName, + TableType: catalog.Bm25Index_TblType_Metadata, + Cols: []*plan.ColDef{ + {Name: catalog.Bm25Index_TblCol_Metadata_Index_Id, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_varchar), Width: 128}, Primary: true, Default: &plan.Default{}}, + {Name: catalog.Bm25Index_TblCol_Metadata_Timestamp, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_int64)}, Default: &plan.Default{}}, + {Name: catalog.Bm25Index_TblCol_Metadata_Checksum, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_varchar), Width: types.MaxVarcharLen}, Default: &plan.Default{}}, + {Name: catalog.Bm25Index_TblCol_Metadata_Filesize, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_int64)}, Default: &plan.Default{}}, + {Name: catalog.Bm25Index_TblCol_Metadata_Recency, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_int64)}, Default: &plan.Default{}}, + {Name: catalog.Bm25Index_TblCol_Metadata_Nrow, Alg: plan.CompressType_Lz4, Typ: plan.Type{Id: int32(types.T_int64)}, Default: &plan.Default{}}, + }, + } + metaTbl.Pkey = &plan.PrimaryKeyDef{ + Names: []string{catalog.Bm25Index_TblCol_Metadata_Index_Id}, + PkeyColName: catalog.Bm25Index_TblCol_Metadata_Index_Id, + } + metaTbl.Defs = append(metaTbl.Defs, &plan.TableDef_DefType{ + Def: &plan.TableDef_DefType_Properties{Properties: &plan.PropertiesDef{Properties: []*plan.Property{ + {Key: catalog.SystemRelAttr_Kind, Value: catalog.Bm25Index_TblType_Metadata}, + }}}, + }) + + return []*plan.IndexDef{storeIdx, metaIdx}, []*plan.TableDef{storeTbl, metaTbl}, nil +} + +// BuildFullTextIndexDefs — bm25 is not reached via CREATE FULLTEXT INDEX +// (*tree.FullTextIndex); it uses BuildSecondaryIndexDefs instead. +func (Hooks) BuildFullTextIndexDefs( + _ planplugin.CompilerContext, + _ *tree.FullTextIndex, + _ map[string]*plan.ColDef, + _ []*plan.IndexDef, + _ string, +) ([]*plan.IndexDef, []*plan.TableDef, error) { + return nil, nil, moerr.NewNotSupportedNoCtx("bm25 plugin does not build fulltext indexes") +} diff --git a/pkg/bm25/plugin/plan/tablefunc.go b/pkg/bm25/plugin/plan/tablefunc.go new file mode 100644 index 0000000000000..909b5b9b7b697 --- /dev/null +++ b/pkg/bm25/plugin/plan/tablefunc.go @@ -0,0 +1,119 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plan + +import ( + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" + planplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/plan" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" +) + +// bm25 table functions: build (source rows -> chunk store) and search +// (query -> ranked doc_ids). Registered into the plan-side TVF dispatch the +// same way the vector plugins register hnsw_create / hnsw_search. + +const ( + Bm25CreateFuncName = "bm25_create" + Bm25SearchFuncName = "bm25_search" +) + +var ( + bm25CreateColDefs = []*plan.ColDef{ + {Name: "status", Typ: plan.Type{Id: int32(types.T_int32), Width: 4}}, + } + + bm25SearchColDefs = []*plan.ColDef{ + {Name: "doc_id", Typ: plan.Type{Id: int32(types.T_int64), Width: 8}}, + {Name: "score", Typ: plan.Type{Id: int32(types.T_float64), Width: 8}}, + } +) + +func init() { + planplugin.RegisterTableFunc(Bm25CreateFuncName, buildBm25Create) + planplugin.RegisterTableFunc(Bm25SearchFuncName, buildBm25Search) +} + +// getBm25Params extracts the leading param constant (arg 0). Mirrors the vector +// plugins' getVectorParams: the first arg is a string const carrying the build +// params; it is stripped from the TblFuncExprList and stored on the node. +func getBm25Params(pb planplugin.PlanBuilder, fn *tree.FuncExpr) (string, error) { + if _, ok := fn.Exprs[0].(*tree.NumVal); ok { + return fn.Exprs[0].String(), nil + } + return "", moerr.NewNoConfig(pb.GetContext(), "first parameter must be string") +} + +// buildBm25Create — arg list: [param, TableConfig(JSON), pk, cols...]. +func buildBm25Create(pb planplugin.PlanBuilder, tbl *tree.TableFunction, ctx planplugin.BindContext, exprs []*plan.Expr, children []int32) (int32, error) { + if len(exprs) < 4 { + return 0, moerr.NewInvalidInput(pb.GetContext(), "bm25_create: invalid number of arguments (NARGS < 4)") + } + colDefs := planplugin.DeepCopyColDefList(bm25CreateColDefs) + params, err := getBm25Params(pb, tbl.Func) + if err != nil { + return 0, err + } + exprs = exprs[1:] + + node := &plan.Node{ + NodeType: plan.Node_FUNCTION_SCAN, + Stats: &plan.Stats{}, + TableDef: &plan.TableDef{ + TableType: "func_table", + TblFunc: &plan.TableFunction{ + Name: Bm25CreateFuncName, + Param: []byte(params), + IsSingle: true, + }, + Cols: colDefs, + }, + BindingTags: []int32{pb.GenNewBindTag()}, + TblFuncExprList: exprs, + Children: children, + } + return pb.AppendNode(node, ctx), nil +} + +// buildBm25Search — arg list: [param, TableConfig(JSON), pattern]. +func buildBm25Search(pb planplugin.PlanBuilder, tbl *tree.TableFunction, ctx planplugin.BindContext, exprs []*plan.Expr, children []int32) (int32, error) { + if len(exprs) != 3 { + return 0, moerr.NewInvalidInput(pb.GetContext(), "bm25_search: invalid number of arguments (NARGS != 3)") + } + colDefs := planplugin.DeepCopyColDefList(bm25SearchColDefs) + params, err := getBm25Params(pb, tbl.Func) + if err != nil { + return 0, err + } + exprs = exprs[1:] + + node := &plan.Node{ + NodeType: plan.Node_FUNCTION_SCAN, + Stats: &plan.Stats{}, + TableDef: &plan.TableDef{ + TableType: "func_table", + TblFunc: &plan.TableFunction{ + Name: Bm25SearchFuncName, + Param: []byte(params), + }, + Cols: colDefs, + }, + BindingTags: []int32{pb.GenNewBindTag()}, + TblFuncExprList: exprs, + Children: children, + } + return pb.AppendNode(node, ctx), nil +} diff --git a/pkg/bm25/plugin/plugin.go b/pkg/bm25/plugin/plugin.go new file mode 100644 index 0000000000000..a66c1fe729bbe --- /dev/null +++ b/pkg/bm25/plugin/plugin.go @@ -0,0 +1,77 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package plugin is the bm25 index plugin registration point. +// +// bm25 is a position-free BM25 ranked-retrieval index over a TEXT/VARCHAR +// column, created via `CREATE INDEX ... USING bm25 [WITH PARSER ]` and +// queried via `MATCH(col) AGAINST('query')`. Structurally it follows the +// vector plugins (parsed to *tree.Index, dispatched through +// BuildSecondaryIndexDefs), with the WAND binary engine in pkg/bm25/wand. +// +// # Phase 1b (current) +// +// Skeleton: catalog hooks (runtime/) are fully implemented; compile/ and +// plan/ schema hooks are STUBS (return NYI). Registration is intentionally +// DEFERRED — an unregistered bm25 algo makes `CREATE INDEX ... USING bm25` +// fail cleanly with "unsupported index type" rather than dispatching to the +// NYI stubs. init() is uncommented in Phase 2 once BuildSecondaryIndexDefs +// (hidden tables) and HandleCreateIndex (build) are real. +package plugin + +import ( + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/indexplugin" + catalogplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/catalog" + compileplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/compile" + idxcronplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/idxcron" + planplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/plan" + + bm25compile "github.com/matrixorigin/matrixone/pkg/bm25/plugin/compile" + bm25idxcron "github.com/matrixorigin/matrixone/pkg/bm25/plugin/idxcron" + bm25plan "github.com/matrixorigin/matrixone/pkg/bm25/plugin/plan" + bm25runtime "github.com/matrixorigin/matrixone/pkg/bm25/plugin/runtime" +) + +// Plugin is the bm25 AlgoPlugin. +type Plugin struct { + catalogHooks catalogplugin.Hooks + compileHooks compileplugin.Hooks + planHooks planplugin.Hooks + idxcronHooks idxcronplugin.Hooks +} + +func New() *Plugin { + return &Plugin{ + catalogHooks: bm25runtime.CatalogHooks{}, + compileHooks: bm25compile.Hooks{}, + planHooks: bm25plan.Hooks{}, + idxcronHooks: bm25idxcron.Hooks{}, + } +} + +func (*Plugin) Algo() string { return catalog.MoIndexBm25Algo.ToString() } +func (p *Plugin) Catalog() catalogplugin.Hooks { return p.catalogHooks } +func (p *Plugin) Compile() compileplugin.Hooks { return p.compileHooks } +func (p *Plugin) Plan() planplugin.Hooks { return p.planHooks } +func (p *Plugin) Idxcron() idxcronplugin.Hooks { return p.idxcronHooks } + +// Compile-time check that *Plugin satisfies the AlgoPlugin interface. +var _ plugin.AlgoPlugin = (*Plugin)(nil) + +// init registers bm25 with the global plugin registry. As of Phase 2 the +// create/build path (compile.HandleCreateIndex + plan.BuildSecondaryIndexDefs) +// is real, so CREATE INDEX ... USING bm25 builds an index synchronously from +// source. CDC/idxcron are still off (catalog SyncDescriptor is the zero value). +func init() { plugin.Register(New()) } diff --git a/pkg/bm25/plugin/runtime/runtime.go b/pkg/bm25/plugin/runtime/runtime.go new file mode 100644 index 0000000000000..45764c9f71a0a --- /dev/null +++ b/pkg/bm25/plugin/runtime/runtime.go @@ -0,0 +1,184 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package runtime holds the bm25 index plugin's catalog-side metadata: +// hidden-table types, parameter schema, sync descriptor. +// +// bm25 is a position-free BM25 ranked-retrieval index over a TEXT/VARCHAR +// column. Structurally it follows the vector plugins (created via +// `CREATE INDEX ... USING bm25`, parsed to *tree.Index, dispatched through +// BuildSecondaryIndexDefs) — NOT the fulltext plugin. Its two hidden tables +// (storage + metadata) hold the chunked WAND binary index; there is no +// postings table (the index builds directly from source rows). +package runtime + +import ( + "strconv" + "strings" + + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" + catalogplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/catalog" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" +) + +// actionBm25Reindex is the idxcron action key for bm25's scheduled +// merge-compaction. Inlined here (rather than importing +// pkg/vectorindex/idxcron) to avoid an import cycle, mirroring the ivfflat +// plugin's actionIvfflatReindex. Stays in lock-step with the bm25 arm of +// pkg/vectorindex/idxcron/executor.go. +const actionBm25Reindex = "bm25_reindex" + +// DefaultParser is the tokenizer used when WITH PARSER is omitted. The WAND +// engine's word-id layer is jieba-backed, so gojieba is the default (and the +// only parser wired end-to-end until the word-id layer is generalized). +const DefaultParser = "gojieba" + +// Bm25IndexFlag gates bm25 DDL behind the experimental_bm25_index session var +// (mirrors HNSW's HnswIndexFlag). Both the framework gate (pkg/sql/compile/util.go, +// via ExperimentalFlag()) and the per-plugin HandleCreateIndex gate reference it. +const Bm25IndexFlag = "experimental_bm25_index" + +// supportedParsers is the set of tokenizers a bm25 index accepts. Both the build +// (bm25_create) and query (bm25_search) paths tokenize with the shared jieba +// tokenizer and never read the parser param, so gojieba is the only real tokenizer; +// "default" is an accepted alias for it (omitting WITH PARSER also defaults to +// gojieba). Other parsers (e.g. ngram) are rejected rather than silently tokenized as +// jieba — accepting them would mislead the user into thinking ngrams are in effect. +var supportedParsers = map[string]struct{}{ + "gojieba": {}, + "default": {}, +} + +// Compile-time interface check. +var _ catalogplugin.Hooks = CatalogHooks{} + +// CatalogHooks implements plugin/catalog.Hooks for bm25. +type CatalogHooks struct{} + +// HiddenTableTypes — bm25 uses two hidden tables: the chunked binary index +// store and its metadata. No postings table (the index is built from source). +func (CatalogHooks) HiddenTableTypes() []string { + return []string{ + catalog.Bm25Index_TblType_Storage, + catalog.Bm25Index_TblType_Metadata, + } +} + +// ShouldTruncateHiddenTable — both hidden tables are derived from the source +// rows and must be reset alongside a TRUNCATE of the source table. +func (CatalogHooks) ShouldTruncateHiddenTable(string) bool { return true } + +// AlterTableCloneBehavior — bm25 is CDC-maintained and rebuilds its whole +// binary index from the source rows on the new table (via the re-armed CDC's +// InitSQL), so the unaffected-index clone skips the whole index rather than +// block-copying a base that would then be doubled by the rebuild. +func (CatalogHooks) AlterTableCloneBehavior() catalogplugin.AlterTableCloneBehavior { + return catalogplugin.AlterTableCloneBehavior{SkipWholeIndex: true} +} + +// RestoreBehavior — the restore rebuilds the binary index from the restored +// rows via RestoreInitSQL (ALTER … REINDEX … FORCE_SYNC), so the seeded +// storage+metadata must be emptied before the block-clone appends, else the +// rebuild doubles the tag=0 base. +func (h CatalogHooks) RestoreBehavior() catalogplugin.RestoreBehavior { + return catalogplugin.RestoreBehavior{DeleteBeforeClone: h.HiddenTableTypes()} +} + +// BuildSessionVars — bm25 captures no session vars; max_index_capacity is an +// explicit WITH option persisted in algo_params by ParamsFromTree. +func (CatalogHooks) BuildSessionVars() []string { return nil } + +// DefaultOptions — no WITH(...) clause defaults the tokenizer to gojieba. +func (CatalogHooks) DefaultOptions() map[string]string { + return map[string]string{"parser": DefaultParser} +} + +// ExperimentalFlag — bm25 DDL is gated by experimental_bm25_index. +func (CatalogHooks) ExperimentalFlag() string { return Bm25IndexFlag } + +// SupportedOpTypes — bm25 is text ranking, not a vector metric; no op_types. +func (CatalogHooks) SupportedOpTypes() map[string]string { return nil } + +// SupportedVectorTypes — bm25 has NO vector column (like fulltext). nil is the +// "no vector column" sentinel, so plan-side vector-type validation is skipped. +func (CatalogHooks) SupportedVectorTypes() []types.T { return nil } + +// SupportedPrimaryKeyTypes — no PK-type constraint (any PK). nil = "no constraint". +func (CatalogHooks) SupportedPrimaryKeyTypes() []types.T { return nil } + +// SupportedIncludeColumnTypes — bm25 does not support INCLUDE columns. +func (CatalogHooks) SupportedIncludeColumnTypes() []types.T { return nil } + +// ValidQuantization — bm25 has no quantization; reject any non-empty value. +func (CatalogHooks) ValidQuantization(quant, _ string) error { + if quant != "" { + return moerr.NewNotSupportedNoCtxf("bm25 index does not support quantization") + } + return nil +} + +// SyncDescriptor — bm25 is always CDC-maintained (AlwaysAsync): post-create DML +// flows into the tag=1 CdcTail via the WAND sinker, and a scheduled idxcron +// merge-compaction (action bm25_reindex, token BM25) folds the tail into the +// tag=0 base. It is not lists-aware (no k-means / nlist concept). +func (CatalogHooks) SyncDescriptor() catalogplugin.SyncDescriptor { + return catalogplugin.SyncDescriptor{ + UsesCDC: true, + SinkerType: catalogplugin.SinkerType_IndexSync, + AlwaysAsync: true, + IdxcronAction: actionBm25Reindex, + IdxcronAlgoToken: "BM25", + IdxcronReindexOption: "MERGE", + IdxcronListsAware: false, + } +} + +// ParamsFromTree extracts the WITH(...) options from CREATE INDEX ... USING bm25 +// into the canonical algo_params map. bm25's knobs are the tokenizer (parser), +// the async/idxcron cadence flags, and max_index_capacity. +func (CatalogHooks) ParamsFromTree(idx *tree.Index) (map[string]string, error) { + res := make(map[string]string) + + parser := strings.ToLower(idx.IndexOption.ParserName) + if parser == "" { + parser = DefaultParser + } + if _, ok := supportedParsers[parser]; !ok { + return nil, moerr.NewNotSupportedNoCtxf( + "bm25 parser %q (supported: gojieba, default)", parser) + } + res["parser"] = parser + + if idx.IndexOption.Async { + res[catalog.Async] = "true" + } + if idx.IndexOption.AutoUpdate { + res[catalog.AutoUpdate] = "true" + } + if idx.IndexOption.Day > 0 { + res[catalog.Day] = strconv.FormatInt(idx.IndexOption.Day, 10) + } + if idx.IndexOption.Hour > 0 { + res[catalog.Hour] = strconv.FormatInt(idx.IndexOption.Hour, 10) + } + if idx.IndexOption.Second > 0 { + res[catalog.Second] = strconv.FormatInt(idx.IndexOption.Second, 10) + } + if idx.IndexOption.MaxIndexCapacity > 0 { + res[catalog.IndexAlgoParamMaxIndexCapacity] = strconv.FormatInt(idx.IndexOption.MaxIndexCapacity, 10) + } + return res, nil +} diff --git a/pkg/bm25/plugin/runtime/runtime_test.go b/pkg/bm25/plugin/runtime/runtime_test.go new file mode 100644 index 0000000000000..062832661e6ab --- /dev/null +++ b/pkg/bm25/plugin/runtime/runtime_test.go @@ -0,0 +1,99 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// CPU-side unit coverage for the bm25 catalog hooks (ParamsFromTree, SyncDescriptor, +// ValidQuantization, HiddenTableTypes). These pure functions are otherwise only +// exercised by the integration BVTs, which do not feed Go per-package coverage. +package runtime + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" +) + +func TestParamsFromTree_DefaultParser(t *testing.T) { + res, err := CatalogHooks{}.ParamsFromTree(&tree.Index{IndexOption: &tree.IndexOption{}}) + require.NoError(t, err) + require.Equal(t, DefaultParser, res["parser"]) +} + +func TestParamsFromTree_ExplicitAndFlags(t *testing.T) { + res, err := CatalogHooks{}.ParamsFromTree(&tree.Index{IndexOption: &tree.IndexOption{ + ParserName: "default", + Async: true, + AutoUpdate: true, + Day: 3, + Hour: 4, + Second: 30, + MaxIndexCapacity: 100000, + }}) + require.NoError(t, err) + require.Equal(t, "default", res["parser"]) + require.Equal(t, "true", res[catalog.Async]) + require.Equal(t, "true", res[catalog.AutoUpdate]) + require.Equal(t, "3", res[catalog.Day]) + require.Equal(t, "4", res[catalog.Hour]) + require.Equal(t, "30", res[catalog.Second]) + require.Equal(t, "100000", res[catalog.IndexAlgoParamMaxIndexCapacity]) +} + +func TestParamsFromTree_UnsupportedParser(t *testing.T) { + // bm25 only tokenizes with jieba, so only gojieba/default are accepted; ngram (and + // any other name) is rejected rather than silently tokenized as jieba. + for _, p := range []string{"bogus", "ngram"} { + _, err := CatalogHooks{}.ParamsFromTree(&tree.Index{IndexOption: &tree.IndexOption{ParserName: p}}) + require.Error(t, err, "parser %q must be rejected", p) + } +} + +func TestExperimentalFlag(t *testing.T) { + require.Equal(t, "experimental_bm25_index", CatalogHooks{}.ExperimentalFlag()) + require.Equal(t, Bm25IndexFlag, CatalogHooks{}.ExperimentalFlag()) +} + +func TestParamsFromTree_OmitsUnsetFlags(t *testing.T) { + res, err := CatalogHooks{}.ParamsFromTree(&tree.Index{IndexOption: &tree.IndexOption{ParserName: "gojieba"}}) + require.NoError(t, err) + // zero-valued cadence flags must not appear (so idxcron treats them as unset) + require.NotContains(t, res, catalog.Async) + require.NotContains(t, res, catalog.Day) + require.NotContains(t, res, catalog.Second) + require.NotContains(t, res, catalog.IndexAlgoParamMaxIndexCapacity) +} + +func TestValidQuantization(t *testing.T) { + require.NoError(t, CatalogHooks{}.ValidQuantization("", "")) + require.Error(t, CatalogHooks{}.ValidQuantization("int8", ""), "bm25 has no quantization") +} + +func TestSyncDescriptor(t *testing.T) { + d := CatalogHooks{}.SyncDescriptor() + require.True(t, d.UsesCDC) + require.True(t, d.AlwaysAsync) + require.Equal(t, "MERGE", d.IdxcronReindexOption) + require.Equal(t, "BM25", d.IdxcronAlgoToken) + require.False(t, d.IdxcronListsAware) +} + +func TestHiddenTableTypes(t *testing.T) { + got := CatalogHooks{}.HiddenTableTypes() + require.ElementsMatch(t, []string{ + catalog.Bm25Index_TblType_Storage, + catalog.Bm25Index_TblType_Metadata, + }, got) +} diff --git a/pkg/bm25/wand/compact.go b/pkg/bm25/wand/compact.go new file mode 100644 index 0000000000000..50ad65df8222d --- /dev/null +++ b/pkg/bm25/wand/compact.go @@ -0,0 +1,521 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wand + +import ( + "fmt" + "os" + "time" + + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/sqlquote" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" +) + +// compact.go — model-level primitives for tiered merge-compaction (Stage 2): +// densify a finalized model to its live ords (FilterLive) and split a finalized +// model into capacity-bounded sub-models (Split). Both operate on a model's +// already-finalized (ascending-ord) postings — no re-tokenize — and produce fresh +// Go-heap models, re-finalized. They are the pieces the CompactSegments +// orchestrator chains between ComputeLiveness and Merge. + +// FilterLive returns a new densified model holding only the ords allow.Contains +// reports live, with ords compacted to a fresh 0..M-1 range and every term's +// postings filtered + remapped. allow == nil means "all ords live" (the +// ComputeLiveness fast path) — the receiver is returned unchanged. +// +// Postings are ascending by ord; the live-ord remap is monotonic, so the filtered +// postings stay ascending (finalizeScoring's invariant holds). After +// ComputeLiveness each pk has exactly one owning segment, so FilterLive'ing every +// segment yields pk-DISJOINT models — the precondition Merge requires. +func (m *WandModel) FilterLive(allow Membership) *WandModel { + if allow == nil { + return m + } + remap := make([]int64, m.N) // old ord -> new ord, or -1 if dead + out := NewWandModel(m.Id, m.PkType) + out.overflow = m.overflow // dict shared read-only; Merge reconciles it + var newOrd int64 + for ord := int64(0); ord < m.N; ord++ { + if allow.Contains(ord) { + remap[ord] = newOrd + out.pks = append(out.pks, m.pks[ord]) + out.docLen = append(out.docLen, m.docLen[ord]) + newOrd++ + } else { + remap[ord] = -1 + } + } + out.N = newOrd + // forEachTerm + materialize handles both a build-side receiver and a LOADED one + // (CompactSegments FilterLive's the loaded tail segments): a loaded term's + // compressed blocks expand transiently, a build-side term returns its resident slices. + m.forEachTerm(func(wid int32, tp *termPostings) { + docs := tp.materializeDocIDs() + tfs := tp.materializeTfs() + var stp *termPostings + for i, ord := range docs { + no := remap[ord] + if no < 0 { + continue + } + if stp == nil { + stp = &termPostings{} + } + stp.docIDs = append(stp.docIDs, no) + stp.tfs = append(stp.tfs, tfs[i]) + } + if stp != nil { + out.terms[wid] = stp + } + }) + out.finalizeScoring() + return out +} + +// Split partitions a finalized model into capacity-bounded sub-models by doc-ord +// range (each ≤ capacity docs), mirroring Builder.FinishSegments but on an +// already-built model. capacity <= 0 or N <= capacity returns the receiver +// unchanged. Each sub-model is self-contained (local 0-based ords, copied +// pks/docLen, its own remapped postings) and re-finalized; the overflow dict is +// shared. Used to keep a Merge result ≤ max_index_capacity. +// +// Requires ascending-ord postings (the model invariant after finalizeScoring / +// Merge / FilterLive). Does NOT sort in place — that would mutate off-heap +// C-buffer postings of a loaded model; callers pass Go-heap Merge/FilterLive +// output. +func (m *WandModel) Split(capacity int64) []*WandModel { + n := m.N + if capacity <= 0 || n <= capacity { + return []*WandModel{m} + } + nseg := int((n + capacity - 1) / capacity) + segs := make([]*WandModel, nseg) + for s := 0; s < nseg; s++ { + lo := int64(s) * capacity + hi := lo + capacity + if hi > n { + hi = n + } + seg := NewWandModel(m.Id, m.PkType) + seg.pks = append([]any(nil), m.pks[lo:hi]...) + seg.docLen = append([]int32(nil), m.docLen[lo:hi]...) + seg.overflow = m.overflow + seg.N = hi - lo + segs[s] = seg + } + m.forEachTerm(func(wid int32, tp *termPostings) { + docs := tp.materializeDocIDs() + tfs := tp.materializeTfs() + i, df := 0, len(docs) + for s := 0; s < nseg && i < df; s++ { + hi := int64(s+1) * capacity + start := i + for i < df && docs[i] < hi { + i++ + } + if i == start { + continue + } + lo := int64(s) * capacity + stp := &termPostings{ + docIDs: make([]int64, i-start), + tfs: append([]uint8(nil), tfs[start:i]...), + } + for j := start; j < i; j++ { + stp.docIDs[j-start] = docs[j] - lo // global -> local ord + } + segs[s].terms[wid] = stp + } + }) + for _, seg := range segs { + seg.finalizeScoring() + } + return segs +} + +// CompactSegments folds the visible tag=1 CdcTail into the tag=0 base WITHOUT +// re-tokenizing from source and WITHOUT rewriting the existing base sub-indexes — +// the O(tail) "fold" step of the recency LSM. It runs in the caller's transaction +// (the fulltext_wand_compact TVF, reached by `ALTER … REINDEX … FULLTEXT MERGE`). +// Snapshot isolation makes it atomic: K = MAX(chunk_id) is read within the txn, so +// concurrent sinker appends (chunk_id > K) are invisible and survive the tail delete. +// +// Only the (threshold-bounded) tail is loaded — never the base — so memory is O(tail), +// not O(corpus). Steps: +// 1. Load the tag=1 tail: insert segments + folded delete map + the pk type. +// 2. Live-filter the tail inserts among themselves (dedup by chunk_id, drop those a +// later tail delete supersedes) → Merge into new capacity-split tag=0 sub(s) at +// recency K (metadata.chunk_id = K, above every existing base at recency < K). +// 3. Surviving deletes = tail deletes whose pk is NOT a live tail insert. They must +// still shadow stale copies in the untouched OLD bases (recency < K), so re-frame +// them as ONE tail delete frame at NextTailChunkId (> K). Deletes resolved inside +// the tail (pk re-inserted live) are dropped. +// 4. Delete the folded tail (chunk_id ≤ K). Old base subs are left in place; their +// stale/deleted copies are shadowed by the new sub (recency K) and the surviving +// deletes (recency > K) at query time. A later tiered merge (2b) reclaims the space. +// +// Returns the number of new tag=0 sub-indexes written (0 when the tail held only +// resolved churn / nothing to fold). + +// survivingDeletes returns the tail deletes NOT resolved by a live re-insert among +// the filtered (live) tail segments — the deletes that must still shadow stale copies +// in the untouched old bases. Keys are normalizeKey'd to match ComputeLiveness and +// the `deletes` map: a string-family pk decodes to []byte, which is unhashable as a +// raw map key (would panic) and would never match the normalized delete keys. +func survivingDeletes(filtered []*WandModel, deletes map[any]int64) []DeleteRecord { + livePks := make(map[any]struct{}, len(filtered)) + for _, f := range filtered { + for _, pk := range f.pks { + livePks[normalizeKey(pk)] = struct{}{} + } + } + var surviving []DeleteRecord + for pk := range deletes { + if _, ok := livePks[pk]; !ok { + surviving = append(surviving, DeleteRecord{Pk: pk}) + } + } + return surviving +} + +func CompactSegments(sqlproc *sqlexec.SqlProcess, cfg TableConfig, capacity int64) (int, error) { + // K = MAX tail chunk_id in this snapshot (the prefix we fold + delete). + _, k, emptyTail, err := tailChunkBounds(sqlproc, cfg) + if err != nil { + return 0, err + } + if emptyTail { + return 0, nil // no tail → nothing to fold + } + + tailSegs, deletes, pkType, err := loadTailSegments(sqlproc, cfg) + if err != nil { + return 0, err + } + defer freeSegs(tailSegs) // free off-heap loaded inputs; Merge copies what it keeps + + // Live-filter the tail inserts (dedup by chunk_id + drop tail-deleted); collect + // the surviving pks. After ComputeLiveness each pk has one owner, so the filtered + // models are pk-disjoint — Merge's precondition. + live := ComputeLiveness(tailSegs, deletes) + filtered := make([]*WandModel, 0, len(tailSegs)) + for i, s := range tailSegs { + f := s.FilterLive(live[i]) + if f.N == 0 { + continue // segment fully dead/superseded within the tail + } + filtered = append(filtered, f) + } + if pkType == 0 && len(filtered) > 0 { + pkType = filtered[0].PkType + } + + // Fold the live tail inserts → new base sub(s) at recency K. The id is timestamp- + // unique (disjoint from existing base ids); recency is carried by ChunkId, not id. + ts := time.Now().UnixMicro() + uid := fmt.Sprintf("%s:%d", cfg.IndexTable, ts) + var segs []*WandModel + if len(filtered) > 0 { + merged := Merge(uid, filtered...) + segs = merged.Split(capacity) + for _, s := range segs { + s.Recency = k + } + } + + // Surviving deletes: tail deletes not resolved by a live re-insert. They shadow + // stale copies in the untouched old bases (recency < K). + surviving := survivingDeletes(filtered, deletes) + + // Write the new base sub(s) at recency K. + for i, m := range segs { + m.Id = SubIndexId(uid, i) + sqls, cleanup, e := m.ToInsertSqls(cfg, ts, int(0)) // tag=0 base + if e != nil { + return 0, e + } + if e := runSqlsWithCleanup(sqlproc, sqls, cleanup); e != nil { + return 0, e + } + } + + // Re-frame surviving deletes as ONE tail delete frame. Runs AFTER writing the base + // at recency K, so NextTailChunkId = K+1 (still ≤ K tail present) → the frame lands + // above the new base and every old base; the tail delete below then spares it. + if len(surviving) > 0 { + if pkType == 0 { + return 0, moerr.NewInternalError(sqlproc.GetContext(), + "wand compact: surviving deletes but unknown pk type") + } + if e := appendDeleteFrame(sqlproc, cfg, pkType, surviving); e != nil { + return 0, e + } + } + + // Delete the folded tail prefix (≤ K). Old base subs are left untouched. + for _, s := range DeleteTailChunksByMaxId(cfg, k) { + res, e := sqlexec.RunSql(sqlproc, s) + if e != nil { + return 0, e + } + res.Close() + } + + // Opportunistic tiered merge: coalesce the small fold subs the folds accumulate so the + // sub count (hence query cost) stays bounded. Self-gating — a no-op metadata scan when + // no adjacent small run qualifies. Same txn as the fold, so it rolls back atomically. + if _, e := TieredMergeBases(sqlproc, cfg, capacity); e != nil { + return 0, e + } + return len(segs), nil +} + +// runSqlsWithCleanup runs a group of statements, calling cleanup (temp-file removal) +// after — even on error — so a failed base write never leaks its serialized blob. +func runSqlsWithCleanup(sqlproc *sqlexec.SqlProcess, sqls []string, cleanup func()) error { + if cleanup != nil { + defer cleanup() + } + for _, s := range sqls { + res, e := sqlexec.RunSql(sqlproc, s) + if e != nil { + return e + } + res.Close() + } + return nil +} + +// appendDeleteFrame persists one tag=1 delete frame (the compaction's surviving +// deletes) at NextTailChunkId — the same file→chunk-rows path the CDC sinker uses, +// so it re-loads as an ordinary tail delete frame. +func appendDeleteFrame(sqlproc *sqlexec.SqlProcess, cfg TableConfig, pkType int32, recs []DeleteRecord) error { + framed, err := FrameDeletes(pkType, recs) + if err != nil { + return err + } + fp, err := os.CreateTemp("", "wanddel") + if err != nil { + return err + } + path := fp.Name() + defer func() { fp.Close(); os.Remove(path) }() + if _, err = fp.Write(framed); err != nil { + return err + } + if err = fp.Sync(); err != nil { // durable before load_file reads it + return err + } + start, err := nextTailChunkId(sqlproc, cfg) + if err != nil { + return err + } + for _, s := range TailFileInsertSqls(cfg, start, path, len(framed)) { + res, e := sqlexec.RunSql(sqlproc, s) + if e != nil { + return e + } + res.Close() + } + return nil +} + +// nextTailChunkId runs NextTailChunkIdSql and returns the next free tag=1 append +// position (GREATEST(max tail chunk_id, max base recency)+1). +func nextTailChunkId(sqlproc *sqlexec.SqlProcess, cfg TableConfig) (int64, error) { + res, err := sqlexec.RunSql(sqlproc, NextTailChunkIdSql(cfg)) + if err != nil { + return 0, err + } + defer res.Close() + for _, bat := range res.Batches { + if bat != nil && bat.RowCount() > 0 { + return vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[0], 0), nil + } + } + return 0, nil +} + +// Tiered-merge tuning. A merge run coalesces up to mergeFactor adjacent UNDER-CAPACITY +// subs, capped at maxMergeBytes of resident postings so memory stays bounded. Fullness is +// judged by doc count vs max_index_capacity (not bytes): a sub already at capacity is +// optimal and is never re-merged. +const ( + mergeFactor = 8 + maxMergeBytes = 128 << 20 // 128 MiB resident per merge pass +) + +// baseSubMeta is a tag=0 base sub-index's metadata row (id + recency + serialized size + +// live doc count), used by the tiered merge to pick a batch without loading any postings. +type baseSubMeta struct { + id string + recency int64 + filesize int64 + nrow int64 +} + +// full reports whether the sub is at max_index_capacity — a full sub is never a merge +// candidate, so a MERGE over a pure-insert tail never rewrites the full base. capacity <= 0 +// means "unlimited" (no cap), so nothing is ever full and all subs coalesce. +func (m baseSubMeta) full(capacity int64) bool { return capacity > 0 && m.nrow >= capacity } + +// listBaseSubsByRecency returns the tag=0 base subs ordered by recency (metadata.chunk_id +// ASC, then index_id for a stable order among a fold's capacity-split siblings) — the order +// the tiered merge scans for an adjacent, recency-contiguous run. +func listBaseSubsByRecency(sqlproc *sqlexec.SqlProcess, cfg TableConfig) ([]baseSubMeta, error) { + sql := fmt.Sprintf("SELECT %s, %s, %s, %s FROM %s ORDER BY %s ASC, %s ASC", + catalog.Bm25Index_TblCol_Metadata_Index_Id, catalog.Bm25Index_TblCol_Metadata_Recency, + catalog.Bm25Index_TblCol_Metadata_Filesize, catalog.Bm25Index_TblCol_Metadata_Nrow, + sqlquote.QualifiedIdent(cfg.DbName, cfg.MetadataTable), + catalog.Bm25Index_TblCol_Metadata_Recency, catalog.Bm25Index_TblCol_Metadata_Index_Id) + res, err := sqlexec.RunSql(sqlproc, sql) + if err != nil { + return nil, err + } + defer res.Close() + var metas []baseSubMeta + for _, bat := range res.Batches { + if bat == nil { + continue + } + for i := 0; i < bat.RowCount(); i++ { + metas = append(metas, baseSubMeta{ + id: bat.Vecs[0].GetStringAt(i), + recency: vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[1], i), + filesize: vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[2], i), + nrow: vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[3], i), + }) + } + } + return metas, nil +} + +// selectMergeRun finds the first maximal run of ADJACENT under-capacity subs (in recency +// order) worth merging — capped at mergeFactor subs and maxMergeBytes. It returns [lo,hi) +// with hi-lo ≥ 2, or lo==hi when no run qualifies. A sub already at max_index_capacity is +// full (never a candidate), so a full base is never re-merged. +// +// Adjacency in the recency-sorted list is a correctness requirement, not a heuristic: +// merging emits one sub at the run's MAX recency, so a doc from a lower-recency member is +// "promoted". Because the run skips no sub whose recency lies inside its range, the newest +// copy of every pk in that range is in the run (its promoted copy is the true-newest); every +// excluded sub is strictly older (correctly shadowed by the merged max) or strictly newer +// (correctly shadows it). A non-adjacent pick could leapfrog an excluded middle sub holding +// a newer copy → stale result. (Full subs excluded from a run are always at the run's +// boundary, never interior — a full sub ends the run — so contiguity holds.) +func selectMergeRun(metas []baseSubMeta, capacity int64) (lo, hi int) { + for i := 0; i < len(metas); { + j, sum := i, int64(0) + for j < len(metas) && j-i < mergeFactor && + !metas[j].full(capacity) && sum+metas[j].filesize <= maxMergeBytes { + sum += metas[j].filesize + j++ + } + if j-i >= 2 { + return i, j + } + if j > i { // a single under-capacity sub then a full/over-budget one: resume there + i = j + } else { // metas[i] itself is full: skip it + i++ + } + } + return 0, 0 +} + +// TieredMergeBases coalesces one adjacent, recency-contiguous run of small tag=0 base subs +// into fewer capacity-capped subs — bounding the sub count the fold grows (query cost scales +// with sub count) and reclaiming docs a tail delete or a higher-recency member supersedes. +// Memory is O(run) ≤ maxMergeBytes (never the whole base). The merged sub takes the run's MAX +// recency; the tail is NOT touched (its delete frames still shadow non-merged subs, and are +// re-applied here so a promoted doc is never resurrected past its delete). Returns the number +// of new subs written (0 when no run qualifies). +func TieredMergeBases(sqlproc *sqlexec.SqlProcess, cfg TableConfig, capacity int64) (int, error) { + metas, err := listBaseSubsByRecency(sqlproc, cfg) + if err != nil { + return 0, err + } + lo, hi := selectMergeRun(metas, capacity) + if hi-lo < 2 { + return 0, nil // no adjacent small run worth merging + } + batch := metas[lo:hi] + maxRecency := batch[len(batch)-1].recency // recency-sorted ⇒ last is the max + + subs := make([]*WandModel, 0, len(batch)) + for _, b := range batch { + m, e := LoadFromStorage(sqlproc, cfg, b.id) + if e != nil { + freeSegs(subs) + return 0, e + } + subs = append(subs, m) + } + defer freeSegs(subs) + + // The tail deletes must be re-applied: promoting a doc to maxRecency could lift it past + // a delete frame whose chunk_id sits between the doc's old recency and maxRecency, which + // would resurrect it. Load the tail only for its delete map, then free the insert segs. + tail, deletes, _, err := loadTailSegments(sqlproc, cfg) + if err != nil { + return 0, err + } + freeSegs(tail) + + live := ComputeLiveness(subs, deletes) + filtered := make([]*WandModel, 0, len(subs)) + for i, s := range subs { + f := s.FilterLive(live[i]) + if f.N > 0 { + filtered = append(filtered, f) + } + } + + ts := time.Now().UnixMicro() + uid := fmt.Sprintf("%s:tm:%d", cfg.IndexTable, ts) + var out []*WandModel + if len(filtered) > 0 { + out = Merge(uid, filtered...).Split(capacity) + for _, s := range out { + s.Recency = maxRecency + } + } + + // Write the merged sub(s) at maxRecency, then delete the merged batch subs. New ids + // (uid:tm:ts) are disjoint from the batch ids, so order is immaterial. + for i, m := range out { + m.Id = SubIndexId(uid, i) + sqls, cleanup, e := m.ToInsertSqls(cfg, ts, int(0)) + if e != nil { + return 0, e + } + if e := runSqlsWithCleanup(sqlproc, sqls, cleanup); e != nil { + return 0, e + } + } + for _, b := range batch { + for _, s := range DeleteSqls(cfg, b.id) { + res, e := sqlexec.RunSql(sqlproc, s) + if e != nil { + return 0, e + } + res.Close() + } + } + return len(out), nil +} diff --git a/pkg/bm25/wand/compact_test.go b/pkg/bm25/wand/compact_test.go new file mode 100644 index 0000000000000..101cd8ca8ce29 --- /dev/null +++ b/pkg/bm25/wand/compact_test.go @@ -0,0 +1,179 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wand + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +// The compaction correctness property: FilterLive each segment by its liveness, +// then Merge — the result reproduces exactly the live pk set that +// SearchSegmentsLive returns over the originals (dedup, delete, reinsert), with no +// cross-segment duplicates. This is what lets the compact TVF replace the +// base+tail with a single merged base. +func TestWandCompact_FilterLiveMerge(t *testing.T) { + q := []string{"x"} + segs := []*WandModel{ + buildSeg(t, 1, map[int64][]string{5: {"x"}, 6: {"x"}, 7: {"x"}}), + buildSeg(t, 2, map[int64][]string{5: {"x"}}), // UPDATE pk 5 (newer chunk) + } + deletes := map[any]int64{normalizeKey(int64(6)): 3} // DELETE pk 6 after both inserts + live := ComputeLiveness(segs, deletes) + + // reference live set (SearchSegmentsLive) — expect {5, 7}, pk 6 deleted + want := pkCounts(SearchSegmentsLive(segs, q, 10, nil, live)) + require.Equal(t, map[int64]int{5: 1, 7: 1}, want) + + // compact: FilterLive each → Merge → search the single merged model + filtered := make([]*WandModel, len(segs)) + for i, s := range segs { + filtered[i] = s.FilterLive(live[i]) + } + merged := Merge("compacted", filtered...) + got := pkCounts(SearchSegments([]*WandModel{merged}, q, 10, nil)) + + require.Equal(t, want, got, "compacted result must equal the live set") + require.Equal(t, int64(2), merged.N, "merged holds only the 2 live docs (5,7)") +} + +// Regression: a string-family (varchar/text/…) pk decodes to []byte, which is +// unhashable as a raw map key. survivingDeletes (used by CompactSegments during +// MERGE) must normalizeKey the live pks — otherwise MERGE panics ("hash of +// unhashable type []uint8"), and even for comparable pks a raw key would never +// match the normalized `deletes` keys, mis-classifying every delete as surviving. +func TestWandSurvivingDeletes_StringPk(t *testing.T) { + // A varchar-pk tail: docs alpha/beta/gamma live; deletes of beta (re-inserted + // live) and delta (never re-inserted). pks are []byte, the decodePk form. + b := NewBuilder("seg1", int32(types.T_varchar)) + for _, pk := range [][]byte{[]byte("alpha"), []byte("beta"), []byte("gamma")} { + require.NoError(t, b.Add("x", pk)) + } + filtered := []*WandModel{b.Finish()} + + // deletes map is keyed the way FoldDeleteFrame/ComputeLiveness key it: normalized. + deletes := map[any]int64{ + normalizeKey([]byte("beta")): 5, // resolved: beta is a live insert -> dropped + normalizeKey([]byte("delta")): 5, // surviving: delta is not a live insert + } + + var surviving []DeleteRecord + require.NotPanics(t, func() { surviving = survivingDeletes(filtered, deletes) }) + + require.Len(t, surviving, 1, "only the unresolved delete (delta) survives") + require.Equal(t, normalizeKey([]byte("delta")), normalizeKey(surviving[0].Pk)) +} + +// FilterLive(nil) is the all-live fast path — returns the receiver unchanged. +func TestWandFilterLive_NilIsIdentity(t *testing.T) { + m := buildSeg(t, 1, map[int64][]string{5: {"x"}, 6: {"x"}}) + require.Same(t, m, m.FilterLive(nil)) +} + +// Split partitions a finalized model into capacity-bounded sub-models whose +// combined search results are identical to the unsplit model. +func TestWandSplit_PreservesSearch(t *testing.T) { + docs := map[int64][]string{} + for i := int64(0); i < 10; i++ { + docs[i] = []string{"x", fmt.Sprintf("t%d", i%3)} + } + m := buildSeg(t, 0, docs) // N = 10, finalized single model + q := []string{"x"} + full := pkCounts(SearchSegments([]*WandModel{m}, q, 20, nil)) + require.Len(t, full, 10) + + segs := m.Split(3) + require.Len(t, segs, 4) // ceil(10/3) + var total int64 + for _, s := range segs { + require.LessOrEqual(t, s.N, int64(3), "each sub-model ≤ capacity") + total += s.N + } + require.Equal(t, int64(10), total) + + require.Equal(t, full, pkCounts(SearchSegments(segs, q, 20, nil)), + "split must preserve the search result set") +} + +// Split is a no-op (returns the receiver) when capacity <= 0 or N <= capacity. +func TestWandSplit_NoOpUnderCapacity(t *testing.T) { + m := buildSeg(t, 0, map[int64][]string{1: {"x"}, 2: {"x"}}) + require.Equal(t, []*WandModel{m}, m.Split(0)) + require.Equal(t, []*WandModel{m}, m.Split(100)) +} + +// selectMergeRun picks the first maximal run of ADJACENT under-capacity subs (≥2), capped +// at mergeFactor / maxMergeBytes. Fullness is by doc count vs capacity (a full sub is never +// a candidate) and adjacency is the correctness property (no skipped middle sub) — both +// verified here across full/under-cap interleavings and the caps. +func TestSelectMergeRun(t *testing.T) { + const capacity = int64(100) + // mk builds subs from doc counts; tiny filesize so the byte budget never binds here. + mk := func(nrows ...int64) []baseSubMeta { + metas := make([]baseSubMeta, len(nrows)) + for i, n := range nrows { + metas[i] = baseSubMeta{id: fmt.Sprintf("s%d", i), recency: int64(i), nrow: n, filesize: 1} + } + return metas + } + un, fl := int64(10), capacity // under-capacity vs full (nrow >= capacity) + check := func(name string, metas []baseSubMeta, wantLo, wantHi int) { + lo, hi := selectMergeRun(metas, capacity) + require.Equal(t, [2]int{wantLo, wantHi}, [2]int{lo, hi}, name) + } + + check("empty", nil, 0, 0) + check("single under-cap", mk(un), 0, 0) + check("all full", mk(fl, fl, fl), 0, 0) + check("two under-cap", mk(un, un), 0, 2) + check("full then run", mk(fl, un, un), 1, 3) + check("run then full", mk(un, un, fl), 0, 2) + // first under-cap is alone (followed by a full sub); the real run is the trailing pair. + check("lone under-cap, full, run", mk(un, fl, un, un), 2, 4) + + // mergeFactor cap: 10 under-cap ⇒ first mergeFactor of them. + ten := make([]int64, 10) + for i := range ten { + ten[i] = un + } + check("mergeFactor cap", mk(ten...), 0, mergeFactor) + + // byte-budget cap: under-cap subs but each > half maxMergeBytes ⇒ no adjacent pair fits. + big := int64(maxMergeBytes/2 + 1) + metas := []baseSubMeta{ + {id: "b0", recency: 0, nrow: un, filesize: big}, + {id: "b1", recency: 1, nrow: un, filesize: big}, + } + lo, hi := selectMergeRun(metas, capacity) + require.Equal(t, [2]int{0, 0}, [2]int{lo, hi}, "each pair exceeds the byte budget ⇒ no run") + + // capacity <= 0 (unlimited): nothing is ever full, so even large-nrow subs coalesce. + check2 := func(name string, metas []baseSubMeta, wantLo, wantHi int) { + lo, hi := selectMergeRun(metas, 0) + require.Equal(t, [2]int{wantLo, wantHi}, [2]int{lo, hi}, name) + } + check2("unlimited coalesces all", mk(fl, fl, fl), 0, mergeFactorMin(3)) +} + +func mergeFactorMin(n int) int { + if n < mergeFactor { + return n + } + return mergeFactor +} diff --git a/pkg/bm25/wand/deletes.go b/pkg/bm25/wand/deletes.go new file mode 100644 index 0000000000000..022f56edea5b7 --- /dev/null +++ b/pkg/bm25/wand/deletes.go @@ -0,0 +1,152 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wand + +import ( + "encoding/binary" + "hash/crc32" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" +) + +// The retrieval index's tag=1 "delete log" (see fulltext_wand.md, Phase B). Each +// CDC DELETE/UPSERT emits one DeleteRecord (just the pk). A batch of records is +// appended as one framed tag=1 chunk alongside the tag=1 delta segments in the +// same ft_index store; the frame's chunk_id (its append position in the single +// CdcTail log) is the delete's order — there is NO stored order field. At search +// the frames are decoded and folded, in chunk_id order, into a +// pk -> maxDeleteChunkId map fed to ComputeLiveness. +// +// This is the structural analog of cuVS's deleted-set: delete-then-reinsert / +// UPDATE resolve correctly across immutable segments because a delete only kills +// segments with chunk_id < its frame's chunk_id (see ComputeLiveness). The codec +// is self-contained (no dependency on the GPU-coupled cuVS package), matching +// serialize.go's binary+crc32 style. + +// DeleteRecord is one tombstone: a pk deleted by a CDC batch. Its order is the +// containing frame's chunk_id (assigned at load), not a stored field. +type DeleteRecord struct { + Pk any +} + +const deleteLogMagic uint32 = 0x57440100 // 'W' 'D' 01 00 + +// EncodeDeleteLog serializes delete records into one self-describing, +// CRC32-checked chunk: magic | pkType | count | pks | crc. For a FIXED-width pk +// type (int64/uint64/int32/uint32) each pk is stored as its bare fixed-width bytes +// — NO per-pk length prefix, since pkType in the header already implies the width; +// only a VARLENA pk (varchar/char/text/blob/…) is length-prefixed [pkLen:uint32 pk]. +// No order field — the frame's chunk_id is the order. +func EncodeDeleteLog(pkType int32, recs []DeleteRecord) ([]byte, error) { + var w leBuf + w.u32(deleteLogMagic) + w.i32(pkType) + w.i64(int64(len(recs))) + width, fixed := pkFixedWidth(pkType) + for _, r := range recs { + pkb, err := encodePk(pkType, r.Pk) + if err != nil { + return nil, err + } + if fixed { + if len(pkb) != width { + return nil, moerr.NewInternalErrorNoCtxf("wand delete log: pk width %d != %d for fixed type %d", len(pkb), width, pkType) + } + } else { + w.u32(uint32(len(pkb))) // varlena: length-prefixed + } + w.b.Write(pkb) + } + sum := crc32.ChecksumIEEE(w.b.Bytes()) + w.u32(sum) + return w.b.Bytes(), nil +} + +// DecodeDeleteLog reverses EncodeDeleteLog, validating magic + CRC. Cursor-based +// over the body (no bytes.Reader / binary.Read boxing / per-record buffer alloc). +// deleteLogPkType peeks the pkType word (magic|pkType|…) of a delete-log blob +// without a full decode — the tail loader uses it to surface the pk type of a +// delete-only frame (no insert segment to read PkType from) so compaction can +// re-frame surviving deletes. Returns (0, false) if the blob is too short. +func deleteLogPkType(buf []byte) (int32, bool) { + if len(buf) < 8 { + return 0, false + } + return int32(binary.LittleEndian.Uint32(buf[4:8])), true +} + +func DecodeDeleteLog(buf []byte) ([]DeleteRecord, error) { + if len(buf) < 4+4+8+4 { + return nil, moerr.NewInternalErrorNoCtx("wand delete log: truncated") + } + body := buf[:len(buf)-4] + if crc32.ChecksumIEEE(body) != binary.LittleEndian.Uint32(buf[len(buf)-4:]) { + return nil, moerr.NewInternalErrorNoCtx("wand delete log: checksum mismatch") + } + if binary.LittleEndian.Uint32(body[0:4]) != deleteLogMagic { + return nil, moerr.NewInternalErrorNoCtx("wand delete log: bad magic") + } + pkType := int32(binary.LittleEndian.Uint32(body[4:8])) + n := int64(binary.LittleEndian.Uint64(body[8:16])) + if n < 0 { + return nil, moerr.NewInternalErrorNoCtx("wand delete log: bad count") + } + width, fixed := pkFixedWidth(pkType) + pos := 16 + out := make([]DeleteRecord, 0, n) + for i := int64(0); i < n; i++ { + l := width + if !fixed { + if pos+4 > len(body) { + return nil, moerr.NewInternalErrorNoCtx("wand delete log: truncated pk length") + } + l = int(binary.LittleEndian.Uint32(body[pos:])) + pos += 4 + } + if l < 0 || pos+l > len(body) { + return nil, moerr.NewInternalErrorNoCtx("wand delete log: truncated pk") + } + pk, err := decodePk(pkType, body[pos:pos+l]) // decodePk copies varlena; ints read by value + if err != nil { + return nil, err + } + pos += l + out = append(out, DeleteRecord{Pk: pk}) + } + return out, nil +} + +// FoldDeleteFrame folds one decoded delete frame — all records share the frame's +// chunk_id — into the running pk -> maxDeleteChunkId map ComputeLiveness +// consumes. Keyed by normalizeKey(pk); the max chunk_id wins, so folding frames +// in any order is idempotent and a redelivered DELETE (a later frame at a higher +// chunk_id) only raises the bound. Pass the accumulator across frames (nil to +// start); returns the same map (allocated on first non-empty frame, nil if no +// records were ever folded). +func FoldDeleteFrame(m map[any]int64, recs []DeleteRecord, chunkId int64) map[any]int64 { + if len(recs) == 0 { + return m + } + if m == nil { + m = make(map[any]int64, len(recs)) + } + for _, r := range recs { + k := normalizeKey(r.Pk) + if cur, ok := m[k]; !ok || chunkId > cur { + m[k] = chunkId + } + } + return m +} diff --git a/pkg/bm25/wand/frames.go b/pkg/bm25/wand/frames.go new file mode 100644 index 0000000000000..0c65ec0a71c5e --- /dev/null +++ b/pkg/bm25/wand/frames.go @@ -0,0 +1,248 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wand + +import ( + "bytes" + "fmt" + "io" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + cuvscdc "github.com/matrixorigin/matrixone/pkg/vectorindex/cuvs" +) + +// tag=1 CdcTail frames. WAND reuses cuVS's payload-agnostic FrameCdcChunk +// envelope (magic+version+op-counts+crc32); the payload is WAND-specific — a +// serialized WandModel for an insert segment, or an EncodeDeleteLog blob for a +// delete batch. The two are told apart by the frame's op counts (nInserts>0 is a +// segment, nDeletes>0 is a delete log), which are mutually exclusive for WAND +// frames. The frame's chunk_id — its append position in the single tag=1 CdcTail +// log — is assigned by the storage layer at load, never stored in the frame. +// See fulltext_wand.md "single CdcTail log, chunk_id-ordered". + +// FrameSegment serializes m and wraps it as one insert-segment frame. nInserts is +// set to the segment's doc count so the idxcron tag=1-growth gate can sum delta +// docs from the frame header without deserializing. +func FrameSegment(m *WandModel) ([]byte, error) { + blob, err := m.Serialize() + if err != nil { + return nil, err + } + return cuvscdc.FrameCdcChunk(blob, nil, uint32(m.N), 0, 0), nil +} + +// FrameDeletes wraps one CDC delete batch (pks only) as a delete-log frame; +// nDeletes is the record count. +func FrameDeletes(pkType int32, recs []DeleteRecord) ([]byte, error) { + blob, err := EncodeDeleteLog(pkType, recs) + if err != nil { + return nil, err + } + return cuvscdc.FrameCdcChunk(blob, nil, 0, uint32(len(recs)), 0), nil +} + +// TailFrame is one tag=1 CdcTail entry: the framed bytes carried at its chunk_id. +type TailFrame struct { + Recency int64 + Data []byte +} + +// AssembleFrames decodes tag=1 CdcTail frames — which MUST be pre-sorted by +// ChunkId ascending — into the ordered insert-segment list and the folded +// pk -> max-delete-chunk_id map, ready for ComputeLiveness + SearchSegmentsLive. +// Each returned segment's ChunkId is set to its frame's chunk_id (the recency +// key). The caller owns the returned segments and must Free() them; on any +// framing/decode error the partially-built segments are freed before returning. +func AssembleFrames(frames []TailFrame) (segs []*WandModel, deletes map[any]int64, err error) { + for _, f := range frames { + segs, deletes, _, err = applyTailFrame(f.Data, f.Recency, segs, deletes) + if err != nil { + freeSegs(segs) + return nil, nil, err + } + } + return segs, deletes, nil +} + +// applyTailFrame decodes one framed tag=1 blob carried at chunkId and folds it into +// (segs, deletes): an insert frame → a deserialized segment (its ChunkId set to +// chunkId) appended to segs; a delete frame → folded into the pk→max-delete-chunk_id +// map. Shared by AssembleFrames (in-memory frames) and assembleFramesAt (streaming +// file). On error the caller owns freeing the partial segs. +// applyTailFrame decodes one tail frame into the running (segs, deletes). It also +// returns the frame's pkType (from an insert segment's PkType or a delete log's +// header) so the caller can learn the index pk type even from a delete-only tail; +// pkType is 0 (unknown) when the frame yields neither. +func applyTailFrame(data []byte, chunkId int64, segs []*WandModel, deletes map[any]int64) ([]*WandModel, map[any]int64, int32, error) { + records, _, nInserts, nDeletes, _, uerr := cuvscdc.UnframeCdcChunk(data) + if uerr != nil { + return segs, deletes, 0, uerr + } + switch { + case nInserts > 0: + m, derr := Deserialize(fmt.Sprintf("tail-%d", chunkId), bytes.NewReader(records)) + if derr != nil { + return segs, deletes, 0, derr + } + m.Recency = chunkId + segs = append(segs, m) + return segs, deletes, m.PkType, nil + case nDeletes > 0: + recs, derr := DecodeDeleteLog(records) + if derr != nil { + return segs, deletes, 0, derr + } + deletes = FoldDeleteFrame(deletes, recs, chunkId) + pkType, _ := deleteLogPkType(records) + return segs, deletes, pkType, nil + default: + return segs, deletes, 0, moerr.NewInternalErrorNoCtx("wand tail frame: empty (neither inserts nor deletes)") + } +} + +// assembleFramesAt walks the tag=1 frames from a chunk-placed source (each chunk at +// slot*MaxChunkSize, slot = chunk_id - minChunk; span slots) and decodes each frame +// straight into (segs, deletes) — the STREAMING assembler. It reads only one frame +// at a time (the header to learn the length, then the frame bytes, freed before the +// next), so the whole tail is never resident: peak transient is one frame, not the +// delta. r is the streaming loader's temp file (or a bytes.Reader in tests). +func assembleFramesAt(r io.ReaderAt, minChunk, span int64) (segs []*WandModel, deletes map[any]int64, pkType int32, err error) { + hdr := make([]byte, cuvscdc.CdcHeaderSize) + for slot := int64(0); slot < span; { + off := slot * int64(vectorindex.MaxChunkSize) + if _, e := r.ReadAt(hdr, off); e != nil { + freeSegs(segs) + return nil, nil, 0, e + } + total, e := cuvscdc.CdcFrameLen(hdr) + if e != nil { + freeSegs(segs) + return nil, nil, 0, e + } + buf := make([]byte, total) + if _, e := r.ReadAt(buf, off); e != nil { + freeSegs(segs) + return nil, nil, 0, e + } + var pt int32 + segs, deletes, pt, e = applyTailFrame(buf, minChunk+slot, segs, deletes) + if e != nil { + freeSegs(segs) + return nil, nil, 0, e + } + if pt != 0 { + pkType = pt + } + slot += int64((total + vectorindex.MaxChunkSize - 1) / vectorindex.MaxChunkSize) + } + return segs, deletes, pkType, nil +} + +// freeSegs releases the C-backed buffers of every segment (idempotent). +func freeSegs(segs []*WandModel) { + for _, s := range segs { + s.Free() + } +} + +// TailChunk is one raw tag=1 CdcTail storage row (one MaxChunkSize-bounded piece +// of a frame). A frame larger than the store's data column is split across +// several consecutive chunks; the load path reassembles them. +type TailChunk struct { + Recency int64 + Data []byte +} + +// splitFrameChunks splits a complete frame into MaxChunkSize-bounded storage +// chunks at consecutive chunk_ids from startChunkId. A frame <= MaxChunkSize +// yields a single chunk. (Frames are never empty — a valid frame is >= the +// 44-byte overhead.) +func splitFrameChunks(startChunkId int64, framed []byte) []TailChunk { + out := make([]TailChunk, 0, (len(framed)+vectorindex.MaxChunkSize-1)/vectorindex.MaxChunkSize) + cid := startChunkId + for off := 0; off < len(framed); off += vectorindex.MaxChunkSize { + end := off + vectorindex.MaxChunkSize + if end > len(framed) { + end = len(framed) + } + out = append(out, TailChunk{Recency: cid, Data: framed[off:end]}) + cid++ + } + return out +} + +// orderTailChunks returns the chunks ordered by chunk_id WITHOUT a comparison +// sort: it places each at index (chunk_id - min) in a preallocated slice (O(n)) — +// the same position-not-sort approach the tag=0 loader uses (streamChunksToFile +// WriteAt by offset). This lets loadTailFrames drop `ORDER BY chunk_id`, which +// would force a SQL Sort (full materialization / possible spill) on the load path. +// +// tag=1 chunk_ids are a GAPLESS run — the writer appends consecutive ids and +// compaction deletes a whole low prefix (never a hole) — so [min..max] must span +// exactly len(chunks) ids; a span mismatch means a missing or duplicate chunk +// (corruption), reported rather than silently mis-assembled. +func orderTailChunks(chunks []TailChunk) ([]TailChunk, error) { + if len(chunks) == 0 { + return nil, nil + } + minC, maxC := chunks[0].Recency, chunks[0].Recency + for _, c := range chunks[1:] { + if c.Recency < minC { + minC = c.Recency + } + if c.Recency > maxC { + maxC = c.Recency + } + } + span := maxC - minC + 1 + if span != int64(len(chunks)) { + return nil, moerr.NewInternalErrorNoCtx(fmt.Sprintf( + "wand tail: chunk_id range [%d..%d] spans %d but got %d rows (gap or duplicate)", + minC, maxC, span, len(chunks))) + } + ordered := make([]TailChunk, span) + for _, c := range chunks { + ordered[c.Recency-minC] = c + } + return ordered, nil +} + +// reassembleFrames groups chunk_id-ordered storage rows back into complete +// frames using each frame's self-describing header length (cuVS CdcFrameLen): a +// frame occupies consecutive chunks whose bytes sum to that length, and its +// ordering key is the first chunk's chunk_id. Chunks MUST be pre-sorted by +// ChunkId ascending. +func reassembleFrames(chunks []TailChunk) ([]TailFrame, error) { + var frames []TailFrame + i := 0 + for i < len(chunks) { + total, err := cuvscdc.CdcFrameLen(chunks[i].Data) + if err != nil { + return nil, err + } + firstChunkId := chunks[i].Recency + buf := make([]byte, 0, total) + for len(buf) < total && i < len(chunks) { + buf = append(buf, chunks[i].Data...) + i++ + } + if len(buf) < total { + return nil, moerr.NewInternalErrorNoCtx("wand tail: truncated frame (missing chunk rows)") + } + frames = append(frames, TailFrame{Recency: firstChunkId, Data: buf[:total]}) + } + return frames, nil +} diff --git a/pkg/bm25/wand/frames_test.go b/pkg/bm25/wand/frames_test.go new file mode 100644 index 0000000000000..6d4d5a494f779 --- /dev/null +++ b/pkg/bm25/wand/frames_test.go @@ -0,0 +1,278 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wand + +import ( + "bytes" + "testing" + + "github.com/matrixorigin/matrixone/pkg/vectorindex" + cuvscdc "github.com/matrixorigin/matrixone/pkg/vectorindex/cuvs" +) + +// TestWandFrameSplitReassemble covers the fix for oversized tag=1 frames: a frame +// larger than the store's MaxChunkSize data column is split across several chunk +// rows and reassembled at load, preserving the frame bytes and the ordering key +// (the frame's first chunk_id). +func TestWandFrameSplitReassemble(t *testing.T) { + mk := func(seed byte, payloadLen int) []byte { + p := make([]byte, payloadLen) + for i := range p { + p[i] = seed + byte(i) + } + return cuvscdc.FrameCdcChunk(p, nil, 1, 0, 0) + } + f1 := mk(1, 2*vectorindex.MaxChunkSize+100) // spans 3 chunk rows + f2 := mk(2, 50) // fits in 1 chunk row + + var chunks []TailChunk + cid := int64(0) + for _, f := range [][]byte{f1, f2} { + cs := splitFrameChunks(cid, f) + chunks = append(chunks, cs...) + cid += int64(len(cs)) + } + if len(chunks) != 4 { // f1 -> chunks 0,1,2 ; f2 -> chunk 3 + t.Fatalf("want 4 chunk rows, got %d", len(chunks)) + } + for _, ch := range chunks { + if len(ch.Data) > vectorindex.MaxChunkSize { + t.Fatalf("chunk %d exceeds MaxChunkSize (%d)", ch.Recency, len(ch.Data)) + } + } + + frames, err := reassembleFrames(chunks) + if err != nil { + t.Fatal(err) + } + if len(frames) != 2 { + t.Fatalf("want 2 reassembled frames, got %d", len(frames)) + } + if frames[0].Recency != 0 || !bytes.Equal(frames[0].Data, f1) { + t.Fatalf("frame 0: chunk_id=%d len=%d/%d bytesEqual=%v", frames[0].Recency, len(frames[0].Data), len(f1), bytes.Equal(frames[0].Data, f1)) + } + if frames[1].Recency != 3 || !bytes.Equal(frames[1].Data, f2) { + t.Fatalf("frame 1: chunk_id=%d (want 3)", frames[1].Recency) + } +} + +// TestOrderTailChunks covers the position-not-sort ordering that lets loadTailFrames +// drop `ORDER BY chunk_id`: a shuffled (and offset, post-compaction-style) set of +// chunk rows is placed back into ascending order in O(n), and a missing chunk_id is +// reported instead of silently mis-assembled. +func TestOrderTailChunks(t *testing.T) { + // chunk_ids 5..9 (a post-compaction min>0 run), delivered shuffled. + shuffled := []TailChunk{ + {Recency: 7, Data: []byte{7}}, + {Recency: 5, Data: []byte{5}}, + {Recency: 9, Data: []byte{9}}, + {Recency: 6, Data: []byte{6}}, + {Recency: 8, Data: []byte{8}}, + } + ordered, err := orderTailChunks(shuffled) + if err != nil { + t.Fatal(err) + } + if len(ordered) != 5 { + t.Fatalf("want 5, got %d", len(ordered)) + } + for i, c := range ordered { + if c.Recency != int64(5+i) || c.Data[0] != byte(5+i) { + t.Fatalf("position %d: chunk_id=%d data=%v (want %d)", i, c.Recency, c.Data, 5+i) + } + } + + // empty → nil, no error. + if got, err := orderTailChunks(nil); err != nil || got != nil { + t.Fatalf("empty: got %v, err %v", got, err) + } + + // a gap (missing chunk_id 7) → error, not a wrong assembly. + gap := []TailChunk{{Recency: 5, Data: []byte{5}}, {Recency: 6, Data: []byte{6}}, {Recency: 8, Data: []byte{8}}} + if _, err := orderTailChunks(gap); err == nil { + t.Fatal("expected a gap in chunk_ids to be rejected") + } +} + +// TestWandTailFrames round-trips insert-segment and delete frames through the +// tag=1 CdcTail codec (FrameSegment/FrameDeletes -> AssembleFrames) and asserts +// the assembled segments carry their frame chunk_id and drive correct liveness: +// an UPDATE (same pk in a later segment) dedups to the newest, and a DELETE at a +// higher chunk_id kills the older copy. +func TestWandTailFrames(t *testing.T) { + // segA: docs 5,6 ; segB: doc 5 (an update of pk 5) ; then delete pk 6. + segA := buildSeg(t, 0, map[int64][]string{5: {"x"}, 6: {"x"}}) + segB := buildSeg(t, 0, map[int64][]string{5: {"x"}}) + defer segA.Free() + defer segB.Free() + + fa, err := FrameSegment(segA) + if err != nil { + t.Fatal(err) + } + fb, err := FrameSegment(segB) + if err != nil { + t.Fatal(err) + } + fd, err := FrameDeletes(testPkType, []DeleteRecord{{Pk: int64(6)}}) + if err != nil { + t.Fatal(err) + } + + // Frames in chunk_id order: segA@1, segB@2, delete(6)@3. + frames := []TailFrame{ + {Recency: 1, Data: fa}, + {Recency: 2, Data: fb}, + {Recency: 3, Data: fd}, + } + segs, deletes, err := AssembleFrames(frames) + if err != nil { + t.Fatal(err) + } + defer freeSegs(segs) + + if len(segs) != 2 { + t.Fatalf("want 2 assembled segments, got %d", len(segs)) + } + // chunk_id is assigned from the frame position, not persisted in the blob. + if segs[0].Recency != 1 || segs[1].Recency != 2 { + t.Fatalf("segment chunk_ids not set from frames: %d, %d", segs[0].Recency, segs[1].Recency) + } + if deletes[normalizeKey(int64(6))] != 3 { + t.Fatalf("delete fold wrong: want {6:3}, got %v", deletes) + } + + // Liveness: pk 5 is owned by segB (chunk 2); pk 6 lives only in segA + // (chunk 1) but is deleted at chunk 3 (> 1) → dead. Only pk 5 survives. + live := ComputeLiveness(segs, deletes) + got := pkCounts(SearchSegmentsLive(segs, []string{"x"}, 10, nil, live)) + if got[5] != 1 || len(got) != 1 { + t.Fatalf("assembled-frame search: want {5:1}, got %v", got) + } +} + +// placeFrames lays complete frames into a chunk-slot byte buffer exactly as the +// streaming loader's temp file does: frame i starts at slot firstSlot[i] and +// occupies ceil(len/MaxChunkSize) consecutive slots (each slot MaxChunkSize wide), +// contiguous within the frame. Returns the buffer, total span, and each frame's +// first slot (== its chunk_id when minChunk is 0). +func placeFrames(frames [][]byte) (buf []byte, span int64, firstSlot []int64) { + slot := int64(0) + for _, f := range frames { + firstSlot = append(firstSlot, slot) + slot += int64((len(f) + vectorindex.MaxChunkSize - 1) / vectorindex.MaxChunkSize) + } + span = slot + buf = make([]byte, span*int64(vectorindex.MaxChunkSize)) + for i, f := range frames { + copy(buf[firstSlot[i]*int64(vectorindex.MaxChunkSize):], f) + } + return buf, span, firstSlot +} + +// TestAssembleFramesAtStreaming exercises the production streaming assembler +// (assembleFramesAt): a multi-chunk insert-segment frame (> MaxChunkSize) plus a +// delete frame, laid out by chunk slot as the temp file would be, decoded +// frame-by-frame from an io.ReaderAt — asserting the multi-chunk frame is read +// whole, chunk_id is assigned from the slot, and liveness holds. +func TestAssembleFramesAtStreaming(t *testing.T) { + big := make(map[int64][]string, 6000) + for i := int64(0); i < 6000; i++ { + big[i] = []string{"x"} + } + seg := buildSeg(t, 0, big) + defer seg.Free() + fseg, err := FrameSegment(seg) + if err != nil { + t.Fatal(err) + } + if len(fseg) <= vectorindex.MaxChunkSize { + t.Fatalf("segment frame is not multi-chunk (%d bytes); test needs > MaxChunkSize", len(fseg)) + } + fdel, err := FrameDeletes(testPkType, []DeleteRecord{{Pk: int64(6)}}) + if err != nil { + t.Fatal(err) + } + + buf, span, firstSlot := placeFrames([][]byte{fseg, fdel}) + segs, deletes, _, err := assembleFramesAt(bytes.NewReader(buf), 0, span) + if err != nil { + t.Fatal(err) + } + defer freeSegs(segs) + if len(segs) != 1 { + t.Fatalf("want 1 segment, got %d", len(segs)) + } + if segs[0].N != 6000 || segs[0].Recency != firstSlot[0] { + t.Fatalf("segment: N=%d ChunkId=%d (want 6000, %d)", segs[0].N, segs[0].Recency, firstSlot[0]) + } + if deletes[normalizeKey(int64(6))] != firstSlot[1] { + t.Fatalf("delete fold: %v (want {6:%d})", deletes, firstSlot[1]) + } + // pk 6 deleted at a higher chunk_id than the segment → gone; the rest live. + live := ComputeLiveness(segs, deletes) + got := pkCounts(SearchSegmentsLive(segs, []string{"x"}, 10000, nil, live)) + if got[6] != 0 || len(got) != 5999 { + t.Fatalf("want 5999 live docs (pk 6 deleted), got %d (pk6=%d)", len(got), got[6]) + } +} + +// TestWandTailFramesBadDispatch guards the frame-kind dispatch: a corrupt frame +// is rejected (not silently mis-decoded as the wrong kind). +func TestWandTailFramesBadDispatch(t *testing.T) { + fd, err := FrameDeletes(testPkType, []DeleteRecord{{Pk: int64(1)}}) + if err != nil { + t.Fatal(err) + } + fd[len(fd)-20] ^= 0xff // corrupt inside the framed payload/crc region + if _, _, err := AssembleFrames([]TailFrame{{Recency: 1, Data: fd}}); err == nil { + t.Fatal("expected AssembleFrames to reject a corrupt frame") + } +} + +// TestWandSearchSegsLive exercises the multi-segment load-adapter search core +// (searchSegsLive): a tag=0 base segment (recency below the tail) plus a tag=1 +// delta that updates one doc and adds another, with a delete — asserting +// liveness (base < tail so tail wins) and that a per-segment WHERE prefilter is +// applied against each segment's own pks (no cross-segment ord confusion). +func TestWandSearchSegsLive(t *testing.T) { + base := buildSeg(t, baseRecency, map[int64][]string{1: {"x"}, 2: {"x"}, 3: {"x"}}) + tail := buildSeg(t, 2, map[int64][]string{2: {"x"}, 4: {"x"}}) // pk 2 updated, 4 new + defer base.Free() + defer tail.Free() + segs := []*WandModel{base, tail} + deletes := map[any]int64{normalizeKey(int64(3)): 3} // delete pk 3 at chunk 3 (> base -1) + + wantSet := func(got, want map[int64]int) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("want %v, got %v", want, got) + } + for pk, n := range want { + if got[pk] != n { + t.Fatalf("pk %d: want %d, got %d (full %v)", pk, n, got[pk], got) + } + } + } + + // No filter: pk 2 owned by tail (chunk 2 > base -1); pk 3 deleted → {1,2,4}. + wantSet(pkCounts(searchSegsLive(segs, deletes, []string{"x"}, 10, nil)), map[int64]int{1: 1, 2: 1, 4: 1}) + + // Per-segment prefilter allowing only {1,4} → {1,4}, evaluated on each + // segment's own ord→pk map. + allow := map[int64]bool{1: true, 4: true} + mkAllow := func(m *WandModel) Membership { return &ordMembership{m: m, allowPk: allow} } + wantSet(pkCounts(searchSegsLive(segs, deletes, []string{"x"}, 10, mkAllow)), map[int64]int{1: 1, 4: 1}) +} diff --git a/pkg/bm25/wand/mmap_other.go b/pkg/bm25/wand/mmap_other.go new file mode 100644 index 0000000000000..9b72f988f4e55 --- /dev/null +++ b/pkg/bm25/wand/mmap_other.go @@ -0,0 +1,33 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !darwin && !linux + +package wand + +import ( + "io" + "os" +) + +// mmapReadOnly fallback (non-unix): read the whole file into a Go slice. No page +// cache benefit, but the load/query API is identical; munmap is a no-op. +func mmapReadOnly(f *os.File) ([]byte, error) { + if _, err := f.Seek(0, io.SeekStart); err != nil { + return nil, err + } + return io.ReadAll(f) +} + +func munmap(b []byte) error { return nil } diff --git a/pkg/bm25/wand/mmap_unix.go b/pkg/bm25/wand/mmap_unix.go new file mode 100644 index 0000000000000..f988753482d54 --- /dev/null +++ b/pkg/bm25/wand/mmap_unix.go @@ -0,0 +1,47 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build darwin || linux + +package wand + +import ( + "os" + "syscall" +) + +// mmapReadOnly maps the whole file into a shared, read-only region. One mapping is +// created per cached base segment at load and shared by all concurrent queries — +// reads are plain memory loads (no lock), the kernel serializes page faults, and +// the OS page cache (reclaimable, unlike our old off-heap C-malloc) is the residency +// manager. Unmapped by munmap under the cache's eviction write-lock (no reader in +// flight). Returns nil for a zero-length file. +func mmapReadOnly(f *os.File) ([]byte, error) { + fi, err := f.Stat() + if err != nil { + return nil, err + } + if fi.Size() == 0 { + return nil, nil + } + return syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) +} + +// munmap releases a mapping from mmapReadOnly (no-op on nil/empty). +func munmap(b []byte) error { + if len(b) == 0 { + return nil + } + return syscall.Munmap(b) +} diff --git a/pkg/bm25/wand/multibase_test.go b/pkg/bm25/wand/multibase_test.go new file mode 100644 index 0000000000000..2b3bb91aa4355 --- /dev/null +++ b/pkg/bm25/wand/multibase_test.go @@ -0,0 +1,94 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wand + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestSubIndexIdDistinct: each sub-index of one build gets a unique id under the +// build's uid, so multiple tag=0 bases never collide in the shared store. +func TestSubIndexIdDistinct(t *testing.T) { + uid := "__ft_idx:1700000000000000" + seen := map[string]bool{} + for i := 0; i < 8; i++ { + id := SubIndexId(uid, i) + require.Falsef(t, seen[id], "duplicate sub-index id %q", id) + seen[id] = true + require.True(t, strings.HasPrefix(id, uid+":")) + } + require.Equal(t, "__ft_idx:1700000000000000:0", SubIndexId(uid, 0)) + require.Equal(t, "__ft_idx:1700000000000000:5", SubIndexId(uid, 5)) +} + +// TestDeleteAllBasesSqls: clears every tag=0 chunk + all metadata rows, leaving the +// tag=1 CdcTail untouched. +func TestDeleteAllBasesSqls(t *testing.T) { + cfg := TableConfig{DbName: "db", IndexTable: "idxtbl", MetadataTable: "metatbl"} + sqls := DeleteAllBasesSqls(cfg) + require.Len(t, sqls, 2) + // storage delete is scoped to tag=0 (must NOT be an unqualified DELETE that would + // also wipe the tag=1 tail) + require.Contains(t, sqls[0], "idxtbl") + require.Contains(t, sqls[0], "tag") + require.Contains(t, sqls[0], "= 0") + require.NotContains(t, sqls[0], "metatbl") + // metadata delete removes all base rows + require.Contains(t, sqls[1], "metatbl") + require.NotContains(t, sqls[1], "tag") +} + +// TestWandMultiBaseBuildInsertSqls mirrors the CREATE build's multi-base path: a corpus +// past capacity splits into several sub-models (FinishSegments), each assigned a distinct +// SubIndexId, and each sub-model's INSERTs (metadata + chunks) must carry ONLY its own +// id — so the sub-indexes never collide in the shared store. +func TestWandMultiBaseBuildInsertSqls(t *testing.T) { + b := NewBuilder("seg", testPkType) + for i := int64(1); i <= 6; i++ { + require.NoError(t, b.Add("term", i)) // 6 distinct-pk docs + } + models := b.FinishSegments(2) + require.Len(t, models, 3, "cap=2 over 6 docs => 3 sub-indexes") + + cfg := TableConfig{DbName: "db", IndexTable: "ft_index", MetadataTable: "ft_meta"} + uid := "ft_index:1700000000000000" + ids := make([]string, len(models)) + for i := range models { + ids[i] = SubIndexId(uid, i) + } + for i, m := range models { + m.Id = ids[i] + sqls, cleanup, err := m.ToInsertSqls(cfg, 123, 0) + require.NoError(t, err) + self := "'" + ids[i] + "'" + found := false + for _, s := range sqls { + if strings.Contains(s, self) { + found = true + } + for j, other := range ids { + if j != i { + require.NotContainsf(t, s, "'"+other+"'", + "sub-index %s SQL leaked sibling id %s", ids[i], other) + } + } + } + require.Truef(t, found, "sub-index %s: no INSERT carried its own id", ids[i]) + cleanup() + } +} diff --git a/pkg/bm25/wand/nativepk_test.go b/pkg/bm25/wand/nativepk_test.go new file mode 100644 index 0000000000000..c5e9c839f037b --- /dev/null +++ b/pkg/bm25/wand/nativepk_test.go @@ -0,0 +1,95 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wand + +import ( + "reflect" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +// TestEncodePkNativeTypes round-trips the native fixed-width temporal / decimal pk +// types delivered by the ISCP extractor's ReprNative mode: encodePk -> decodePk must +// reproduce the exact native Go value. +func TestEncodePkNativeTypes(t *testing.T) { + cases := []struct { + name string + pkType types.T + val any + }{ + {"date", types.T_date, types.Date(0x0135_7924)}, + {"datetime", types.T_datetime, types.Datetime(0x0123_4567_89AB_CDEF)}, + {"time", types.T_time, types.Time(-0x0011_2233_4455_6677)}, + {"timestamp", types.T_timestamp, types.Timestamp(0x7FFF_FFFF_FFFF_FFFF)}, + {"decimal64", types.T_decimal64, types.Decimal64(0xDEAD_BEEF_CAFE_F00D)}, + {"decimal128", types.T_decimal128, types.Decimal128{B0_63: 0x0102_0304_0506_0708, B64_127: 0x1112_1314_1516_1718}}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + b, err := encodePk(int32(c.pkType), c.val) + if err != nil { + t.Fatalf("encodePk(%s): %v", c.name, err) + } + got, err := decodePk(int32(c.pkType), b) + if err != nil { + t.Fatalf("decodePk(%s): %v", c.name, err) + } + if !reflect.DeepEqual(got, c.val) { + t.Fatalf("%s round-trip: got %#v (%T), want %#v (%T)", c.name, got, got, c.val, c.val) + } + }) + } +} + +// TestWandCdcNativePkRoundTrip exercises the whole CDC channel blob (WandCdc.Encode -> +// DecodeWandCdc) with native temporal / decimal pks, mirroring what the ISCP sinker +// ships once extractRowFromVector delivers them natively (ReprNative). +func TestWandCdcNativePkRoundTrip(t *testing.T) { + cases := []struct { + name string + pkType types.T + pk any + }{ + {"date", types.T_date, types.Date(0x0135_7924)}, + {"datetime", types.T_datetime, types.Datetime(0x0123_4567_89AB_CDEF)}, + {"timestamp", types.T_timestamp, types.Timestamp(0x7FFF_FFFF_FFFF_FFFF)}, + {"decimal64", types.T_decimal64, types.Decimal64(0xDEAD_BEEF_CAFE_F00D)}, + {"decimal128", types.T_decimal128, types.Decimal128{B0_63: 1000, B64_127: 7}}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + cdc := NewWandCdc(int32(c.pkType)) + cdc.Insert(c.pk, "arbitrary text") + cdc.Delete(c.pk) + blob, err := cdc.Encode() + if err != nil { + t.Fatalf("Encode(%s): %v", c.name, err) + } + got, err := DecodeWandCdc(blob) + if err != nil { + t.Fatalf("DecodeWandCdc(%s): %v", c.name, err) + } + if len(got.Events) != 2 { + t.Fatalf("%s: want 2 events, got %d", c.name, len(got.Events)) + } + for _, e := range got.Events { + if !reflect.DeepEqual(e.Pk, c.pk) { + t.Fatalf("%s pk round-trip: got %#v (%T), want %#v (%T)", c.name, e.Pk, e.Pk, c.pk, c.pk) + } + } + }) + } +} diff --git a/pkg/bm25/wand/review_fixes_test.go b/pkg/bm25/wand/review_fixes_test.go new file mode 100644 index 0000000000000..8468b5856aac3 --- /dev/null +++ b/pkg/bm25/wand/review_fixes_test.go @@ -0,0 +1,69 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wand + +import ( + "bytes" + "encoding/binary" + "hash/crc32" + "testing" + + "github.com/matrixorigin/matrixone/pkg/common/docfilter" + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/stretchr/testify/require" +) + +// TestDocFilterMembershipUuid is the B1 regression: a uuid membership probe must hit +// the filter entry, which is built from the source uuid's RAW 16 bytes. Before the +// fix, Contains re-encoded via encodePk (36-char canonical string) and never matched, +// so a uuid-PK retrieval query with a WHERE prefilter returned zero rows. +func TestDocFilterMembershipUuid(t *testing.T) { + mp := mpool.MustNewZero() + uu := types.Uuid([16]byte{0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0, 8}) + other := types.Uuid([16]byte{9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9}) + + vec := vector.NewVec(types.New(types.T_uuid, 16, 0)) + require.NoError(t, vector.AppendFixed(vec, uu, false, mp)) + fbytes, err := docfilter.Build(vec) + require.NoError(t, err) + filter, err := docfilter.New(fbytes) + require.NoError(t, err) + + m := &WandModel{PkType: int32(types.T_uuid), pks: []any{uu, other}} + dfm := &docFilterMembership{m: m, f: filter} + + require.True(t, dfm.Contains(0), "uuid IN the membership filter must match (B1)") + require.False(t, dfm.Contains(1), "uuid NOT in the filter must not match") +} + +// TestDecodeWandCdcTruncatedPk is the #6 regression: a blob whose internal pkLen +// exceeds the remaining bytes (valid outer CRC, but a truncated body) must return a +// clean error rather than silently zero-filling a corrupt pk. +func TestDecodeWandCdcTruncatedPk(t *testing.T) { + var b bytes.Buffer + _ = binary.Write(&b, binary.LittleEndian, wandCdcMagic) + _ = binary.Write(&b, binary.LittleEndian, int32(types.T_varchar)) + _ = binary.Write(&b, binary.LittleEndian, int64(1)) // one event + b.WriteByte(byte(cdcInsert)) + _ = binary.Write(&b, binary.LittleEndian, uint32(1000)) // pkLen=1000, but no pk bytes follow + // recompute the outer CRC so the truncation is caught by the length guard, not the CRC + sum := crc32.ChecksumIEEE(b.Bytes()) + _ = binary.Write(&b, binary.LittleEndian, sum) + + _, err := DecodeWandCdc(b.Bytes()) + require.Error(t, err, "a pkLen past the buffer end must error, not zero-fill (#6)") +} diff --git a/pkg/bm25/wand/search.go b/pkg/bm25/wand/search.go new file mode 100644 index 0000000000000..4e65c65dcda25 --- /dev/null +++ b/pkg/bm25/wand/search.go @@ -0,0 +1,761 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wand + +import "sort" + +// SearchResult is one ranked hit: the original primary key and its TF-IDF score. +type SearchResult struct { + DocID any + Score float64 +} + +// Membership is the doc-ord allow-set consulted during the WAND walk for +// prefiltering (mirrors cuVS filtered search). It operates on dense int64 doc +// ords so an implementation can be a roaring/cbitmap built once at search setup +// by translating the WHERE filter's pks through the dictionary. nil = unfiltered. +type Membership interface { + Contains(ord int64) bool +} + +const ordEnd = int64(0x7fffffffffffffff) + +// docBitset is a dense per-doc-ord bitset (doc ords are dense in [0, n) per segment), +// 1 bit/doc — ~8× smaller than a []bool. Mirrors fulltext2's docBitset. +type docBitset []uint64 + +func newDocBitset(n int) docBitset { return make(docBitset, (n+63)/64) } +func (b docBitset) set(i int) { b[i>>6] |= uint64(1) << (uint(i) & 63) } +func (b docBitset) clear(i int) { b[i>>6] &^= uint64(1) << (uint(i) & 63) } +func (b docBitset) get(i int) bool { return b[i>>6]&(uint64(1)<<(uint(i)&63)) != 0 } + +// ordAllowSet is a dense per-segment allow-set over ords [0, n) used to carry +// precomputed liveness (owner-by-chunk_id ∩ not-deleted) into the WAND walk via the +// existing Membership interface. A set bit ⇒ the ord is live. +type ordAllowSet struct { + bits docBitset + n int +} + +func (s *ordAllowSet) Contains(ord int64) bool { + return ord >= 0 && ord < int64(s.n) && s.bits.get(int(ord)) +} + +// andMembership is the conjunction of two Membership filters (either may be +// nil = "allow all"); used to AND a WHERE-prefilter with per-segment liveness. +type andMembership struct{ a, b Membership } + +func (m andMembership) Contains(ord int64) bool { + if m.a != nil && !m.a.Contains(ord) { + return false + } + if m.b != nil && !m.b.Contains(ord) { + return false + } + return true +} + +func andAllow(a, b Membership) Membership { + switch { + case a == nil: + return b + case b == nil: + return a + default: + return andMembership{a, b} + } +} + +// ComputeLiveness resolves, once when a segment set is assembled (load time), +// which ord in each segment is the LIVE copy of its pk — the chunk_id-as-identity +// rule that makes CDC delete-then-reinsert / UPDATE correct over immutable +// segments: +// +// - a pk's live copy is the one in the HIGHEST-ChunkId segment that holds it +// (older copies of an UPDATEd pk are superseded — dedup, no duplicate row); +// - that copy is dead iff a delete exists with deleteChunkId > thatSegmentChunkId +// (a delete after the latest insert; a delete before it is superseded). +// +// deletes maps normalizeKey(pk) -> max delete-frame chunk_id for that pk (nil = +// none). The result is parallel to segs: entry i is a Membership over segment i's +// ords (nil ⇒ every ord live, the fast path for a single/compacted segment), +// passed to SearchSegmentsLive. O(total docs), done once per load, not per query. +func ComputeLiveness(segs []*WandModel, deletes map[any]int64) []Membership { + if len(segs) == 0 { + return nil + } + // Fast path: a single segment with no deletes — everything is live. + if len(segs) == 1 && len(deletes) == 0 { + return []Membership{nil} + } + + // owner[pk] = the max segment chunk_id holding pk (the live copy's segment). + owner := make(map[any]int64) + for _, s := range segs { + for _, pk := range s.pks { + k := normalizeKey(pk) + if cur, ok := owner[k]; !ok || s.Recency >= cur { + owner[k] = s.Recency + } + } + } + + out := make([]Membership, len(segs)) + for i, s := range segs { + // Lazily allocate the allow-bitmap only when a dead ord appears — a fully-live + // segment (the common multi-base case with no pending deletes) keeps out[i]=nil and + // costs no O(doc-count) allocation. On the first dead ord, backfill the earlier ords + // (all live so far) so the bitmap stays ord-aligned. + var bits docBitset + for ord, pk := range s.pks { + k := normalizeKey(pk) + live := owner[k] == s.Recency // this segment owns the live copy + if live && deletes != nil { + if dl, ok := deletes[k]; ok && dl > s.Recency { + live = false // deleted after the latest insert + } + } + if !live && bits == nil { + bits = newDocBitset(len(s.pks)) + for j := 0; j < ord; j++ { + bits.set(j) // earlier ords were all live + } + } + if bits != nil && live { + bits.set(ord) + } + } + if bits != nil { + out[i] = &ordAllowSet{bits: bits, n: len(s.pks)} + } + } + return out +} + +// cursor is a term's posting cursor for the Block-Max WAND walk. On a loaded model +// the postings are NOT resident (they live block-compressed in the shared, read-only +// mmap), so the cursor decodes one block at a time into bDocs/bTfs via +// termPostings.fillBlock. The cache is per-cursor, so concurrent queries over the +// same shared model never race; WAND's block-skip means most blocks are never +// decoded. On a build-side model fillBlock copies from the resident flat slices, so +// the same cursor serves both. +type cursor struct { + tp *termPostings + idfSq float64 + weight float64 + maxScore float64 + docLen []int32 + avgDocLen float64 + pos int // global posting index into the term's df postings + curBlk int // block currently decoded into bDocs/bTfs (-1 = none) + blen int // valid entries in bDocs/bTfs + cur int64 // cached curDoc for the current pos (ordEnd when exhausted) + bDocs []int64 // decoded docIDs of curBlk (cap BlockSize) + bTfs []uint8 // decoded tfs of curBlk (cap BlockSize) +} + +func newCursor(tp *termPostings, idfSq, weight, maxScore, avgDocLen float64, docLen []int32) *cursor { + c := &cursor{ + tp: tp, idfSq: idfSq, weight: weight, maxScore: maxScore, + docLen: docLen, avgDocLen: avgDocLen, curBlk: -1, + bDocs: make([]int64, BlockSize), bTfs: make([]uint8, BlockSize), + } + c.refresh() + return c +} + +// ensure decodes the block containing pos into bDocs/bTfs, if not already cached. +// Cheap (a field compare) when the cursor stays within a block. +func (c *cursor) ensure() { + b := c.pos / BlockSize + if b != c.curBlk { + c.blen = c.tp.fillBlock(b, c.bDocs, c.bTfs) + c.curBlk = b + } +} + +// curDoc returns the cached current doc. It is read many times per pivot +// iteration (insertion sort, pivot scan, blockMax, chooseSkip, alignment) while +// the cursor moves at most once, so cur is recomputed only on move (refresh), +// avoiding the df() check, the pos/BlockSize division in ensure, and the modulo +// on every read. +func (c *cursor) curDoc() int64 { return c.cur } + +// refresh recomputes cur after pos changes (advance/skipTo/construction). +func (c *cursor) refresh() { + if c.pos >= c.tp.df() { + c.cur = ordEnd + return + } + c.ensure() + c.cur = c.bDocs[c.pos%BlockSize] +} + +// score is the BM25 contribution at the current posting: +// weight · idf² · bm25Factor(tf, dl, avgdl). +func (c *cursor) score() float64 { + c.ensure() + i := c.pos % BlockSize + ord := c.bDocs[i] + return c.weight * c.idfSq * bm25Factor(float64(c.bTfs[i]), c.docLen[ord], c.avgDocLen) +} +func (c *cursor) advance() { c.pos++; c.refresh() } + +// skipTo advances the cursor to the first doc >= d: locate the block via the +// RESIDENT blockLastDoc (no decode), then binary-search within that one block. +func (c *cursor) skipTo(d int64) { + b := c.blockIndexAt(d) + if b >= c.tp.nblk() { + c.pos = c.tp.df() // past the last posting → exhausted + c.cur = ordEnd + return + } + if b != c.curBlk { + c.blen = c.tp.fillBlock(b, c.bDocs, c.bTfs) + c.curBlk = b + } + // d <= blockLastDoc[b] = bDocs[blen-1], so the lower bound is within the block + // and never moves the cursor backward (d >= current doc). Inline lower-bound + // binary search rather than sort.Search: it avoids the per-call closure + // (skipTo.func1) and the generic sort.Search frame in this hot skip path. + lo, hi := 0, c.blen + for lo < hi { + mid := int(uint(lo+hi) >> 1) + if c.bDocs[mid] < d { + lo = mid + 1 + } else { + hi = mid + } + } + c.pos = b*BlockSize + lo + c.cur = c.bDocs[lo] +} + +// blockIndexAt returns the index of the block (>= the current block) that +// contains doc d (the first block whose last ord >= d). nblk() if d is past +// the cursor's last posting. Uses only the resident blockLastDoc (no block decode). +func (c *cursor) blockIndexAt(d int64) int { + bl := c.tp.blockLastDoc + b := c.pos / BlockSize + for b < len(bl) && bl[b] < d { + b++ + } + return b +} + +// blockMax is the Block-Max score upper bound for the block containing doc d: +// weight·idf²·bm25Factor(blockMaxTf, blockMinDl, avgdl). 0 if d is past the list. +func (c *cursor) blockMax(d int64) float64 { + b := c.blockIndexAt(d) + if b >= len(c.tp.blockLastDoc) { + return 0 + } + return c.weight * c.idfSq * bm25Factor(float64(c.tp.blockMaxTf[b]), c.tp.blockMinDl[b], c.avgDocLen) +} + +// blockEndAt is the last ord of the block containing doc d (the upper edge of the +// region for which blockMax(d) is a valid bound). ordEnd if past the list. +func (c *cursor) blockEndAt(d int64) int64 { + b := c.blockIndexAt(d) + if b >= len(c.tp.blockLastDoc) { + return ordEnd + } + return c.tp.blockLastDoc[b] +} + +// Search runs WAND disjunctive top-K over a single index. Convenience wrapper +// over SearchSegments. +func (m *WandModel) Search(terms []string, limit int, allow Membership) []SearchResult { + return SearchSegments([]*WandModel{m}, terms, limit, allow) +} + +// SearchSegments runs the WAND top-K over the segment set with no liveness +// filtering — valid for a single index or DISJOINT FinishSegments partitions +// (whose pks never collide). For CDC delta segments (where a pk can recur across +// segments) use SearchSegmentsLive with ComputeLiveness, else a re-inserted pk +// would appear once per segment. +func SearchSegments(segs []*WandModel, terms []string, limit int, allow Membership) []SearchResult { + return SearchSegmentsLive(segs, terms, limit, allow, nil) +} + +// SearchSegmentsLive runs WAND disjunctive top-K across one or more index +// segments with CORPUS-GLOBAL BM25 scoring, so the merged top-K is correctly +// ranked even when each segment holds only a slice of the corpus. Global N, +// avgdl and per-term df are aggregated across segments, then each segment's +// Block-Max walk pushes into one shared bounded heap (the running k-th score +// prunes later segments too). limit is K; allow, if non-nil, is the WHERE-clause +// prefilter over doc ords. live, if non-nil, is parallel to segs (from +// ComputeLiveness): live[i] is ANDed with allow for segment i so superseded / +// deleted ords are skipped. A nil live or a nil live[i] means "all ords live". +// corpusStats returns the corpus-global doc count and average doc length over the +// segment set (both include superseded/deleted docs — the accepted stat drift +// until compaction). Query-INDEPENDENT: it depends only on the loaded segments, so +// the search adapter (WandSearch) precomputes it once at Load and passes it to +// searchSegmentsLiveStats, keeping it off the per-query path. +func corpusStats(segs []*WandModel) (gN int64, gAvgDocLen float64) { + var totalDocLen float64 + for _, s := range segs { + gN += s.N + totalDocLen += s.AvgDocLen * float64(s.N) + } + if gN > 0 { + gAvgDocLen = totalDocLen / float64(gN) + } + return gN, gAvgDocLen +} + +// SearchSegmentsLive computes the corpus stats inline and delegates. Callers that +// already hold precomputed stats (the load-cached WandSearch) call +// searchSegmentsLiveStats directly. +func SearchSegmentsLive(segs []*WandModel, terms []string, limit int, allow Membership, live []Membership) []SearchResult { + gN, gAvgDocLen := corpusStats(segs) + return searchSegmentsLiveStats(segs, terms, limit, allow, live, gN, gAvgDocLen) +} + +// searchSegmentsLiveStats is the WAND top-K core with the corpus stats supplied by +// the caller. `live` (per-segment liveness, query-independent) is likewise supplied +// precomputed; only the term-dependent work (weights, per-term df, the walk) runs +// here — so a load-cached adapter pays the O(total-docs) liveness + stats once per +// load, not once per query. +func searchSegmentsLiveStats(segs []*WandModel, terms []string, limit int, allow Membership, live []Membership, gN int64, gAvgDocLen float64) []SearchResult { + if limit <= 0 || len(terms) == 0 || len(segs) == 0 { + return nil + } + if gN <= 0 { + return nil + } + + weights, gdf := queryWeights(segs, terms) + if len(weights) == 0 { + return nil + } + + h := newTopK(limit) + for i, s := range segs { + segAllow := allow + if i < len(live) { + segAllow = andAllow(allow, live[i]) + } + s.searchInto(h, weights, gN, gAvgDocLen, gdf, segAllow) + } + return h.sorted() +} + +// queryWeights builds the dedup'd query-term weights and the corpus-global df per +// word. Resolution of a word to a segment's word-id is done PER SEGMENT: an +// out-of-jieba-dict "overflow" word gets a per-segment id, so a query word can be +// absent from one segment (e.g. the compacted base) yet present in a later +// CDC-delta segment, and independently-built segments may assign it different +// overflow ids. Resolving once against a single segment would drop or mis-map such +// a word (in-dict words resolve to a stable global id, unaffected). +func queryWeights(segs []*WandModel, terms []string) (map[string]float64, map[string]int) { + weights := make(map[string]float64, len(terms)) + for _, t := range terms { + weights[t]++ + } + gdf := make(map[string]int, len(weights)) + for w := range weights { + df := 0 + for _, s := range segs { + if id, ok, err := s.resolveWordID(w); err == nil && ok { + if tp, ok2 := s.lookupTerm(id); ok2 { + df += tp.df() + } + } + } + gdf[w] = df + } + return weights, gdf +} + +// streamBatch is the max rows a streamSink buffers before flushing to emit. +const streamBatch = 8192 + +// streamSink batches (pk, score) results and flushes them to emit in bounded +// chunks, so a no-LIMIT retrieval query returns every matching doc without ever +// materializing them all. On an emit error it records it and stops (the walk +// checks stopped and bails), so a cancelled consumer terminates the walk promptly. +type streamSink struct { + emit func(keys []any, distances []float64) error + keys []any + scores []float64 + err error + stopped bool +} + +func (s *streamSink) push(pk any, score float64) { + if s.stopped { + return + } + s.keys = append(s.keys, pk) + s.scores = append(s.scores, score) + if len(s.keys) >= streamBatch { + s.flush() + } +} + +func (s *streamSink) flush() { + if s.stopped || len(s.keys) == 0 { + return + } + if e := s.emit(s.keys, s.scores); e != nil { + s.err = e + s.stopped = true + return + } + // Hand ownership of the batch to emit; the next batch reallocates on append. + s.keys = nil + s.scores = nil +} + +// streamInto does a plain document-at-a-time OR merge over one segment: it visits +// every matching doc in ord order, scores it (BM25 over the query terms present), +// and pushes it to the sink — no top-K heap, no WAND pruning. The no-LIMIT case +// wants every match; ranking is done by the upstream ORDER BY score node. +func (m *WandModel) streamInto(sink *streamSink, weights map[string]float64, gN int64, gAvgDocLen float64, gdf map[string]int, allow Membership) { + cursors := make([]*cursor, 0, len(weights)) + for word, w := range weights { + id, ok, err := m.resolveWordID(word) + if err != nil || !ok { + continue + } + tp, ok := m.lookupTerm(id) + if !ok { + continue + } + df := gdf[word] + if df <= 0 { + df = tp.df() + } + idf := log10(float64(gN) / float64(df)) + idfSq := idf * idf + cursors = append(cursors, newCursor(tp, idfSq, w, + w*idfSq*bm25Factor(float64(tp.maxTf), tp.minDl, gAvgDocLen), gAvgDocLen, m.docLen)) + } + if len(cursors) == 0 { + return + } + for !sink.stopped { + minDoc := ordEnd + for _, c := range cursors { + if d := c.curDoc(); d < minDoc { + minDoc = d + } + } + if minDoc == ordEnd { + break + } + if allow == nil || allow.Contains(minDoc) { + score := 0.0 + for _, c := range cursors { + if c.curDoc() == minDoc { + score += c.score() + } + } + sink.push(m.PkAt(minDoc), score) + } + for _, c := range cursors { + if c.curDoc() == minDoc { + c.advance() + } + } + } +} + +// streamSegmentsLiveStats is the no-LIMIT streaming counterpart of +// searchSegmentsLiveStats: it walks every segment with liveness + the optional +// WHERE prefilter and emits all matching (pk, score) rows in bounded batches via +// emit, with no top-K heap and no internal sort. +func streamSegmentsLiveStats(segs []*WandModel, terms []string, emit func(keys []any, distances []float64) error, allow Membership, live []Membership, gN int64, gAvgDocLen float64) error { + if len(terms) == 0 || len(segs) == 0 || gN <= 0 { + return nil + } + weights, gdf := queryWeights(segs, terms) + if len(weights) == 0 { + return nil + } + sink := &streamSink{emit: emit} + for i, s := range segs { + segAllow := allow + if i < len(live) { + segAllow = andAllow(allow, live[i]) + } + s.streamInto(sink, weights, gN, gAvgDocLen, gdf, segAllow) + if sink.stopped { + return sink.err + } + } + sink.flush() // final partial batch + return sink.err +} + +// searchSegsLive is a standalone convenience that computes per-segment liveness +// (owner-by-chunk_id ∩ not-deleted) over segs+deletes, optionally ANDs a +// per-segment WHERE prefilter built by mkAllow (nil = unfiltered), and runs the +// corpus-global WAND top-K. mkAllow is called once per segment so a pk-based +// filter resolves against that segment's own ord→pk dictionary — a single filter +// over "ords" would be wrong, since ord i denotes a different pk in each segment. +// +// NB: this recomputes liveness+stats every call. The production load-path adapter +// (WandSearch) does NOT use it — it precomputes liveness+stats at Load and calls +// searchSegmentsLiveStats per query (Phase-C item 3). Kept for tests / one-shot use. +func searchSegsLive(segs []*WandModel, deletes map[any]int64, terms []string, limit int, mkAllow func(*WandModel) Membership) []SearchResult { + live := ComputeLiveness(segs, deletes) + if mkAllow != nil { + if live == nil { + live = make([]Membership, len(segs)) + } + for i, s := range segs { + live[i] = andAllow(mkAllow(s), live[i]) + } + } + return SearchSegmentsLive(segs, terms, limit, nil, live) +} + +// searchInto runs the Block-Max WAND walk over one segment using the supplied +// global stats, pushing (pk, score) into the shared heap h. +func (m *WandModel) searchInto(h *topK, weights map[string]float64, gN int64, gAvgDocLen float64, gdf map[string]int, allow Membership) { + cursors := make([]*cursor, 0, len(weights)) + for word, w := range weights { + id, ok, err := m.resolveWordID(word) + if err != nil || !ok { + continue // word not resolvable in this segment + } + tp, ok := m.lookupTerm(id) + if !ok { + continue // word absent from this segment + } + df := gdf[word] + if df <= 0 { + df = tp.df() + } + idf := log10(float64(gN) / float64(df)) + idfSq := idf * idf + cursors = append(cursors, newCursor(tp, idfSq, w, + w*idfSq*bm25Factor(float64(tp.maxTf), tp.minDl, gAvgDocLen), gAvgDocLen, m.docLen)) + } + if len(cursors) == 0 { + return + } + + for { + live := cursors[:0] + for _, c := range cursors { + if c.curDoc() != ordEnd { + live = append(live, c) + } + } + cursors = live + if len(cursors) == 0 { + break + } + // Keep cursors sorted by curDoc. Insertion sort, NOT sort.Slice: the + // cursors are nearly sorted between iterations (only the skipped cursor + // moved), so this is O(len) here; more importantly sort.Slice boxes the + // slice into an interface{} (runtime.convTslice) and heap-allocates the + // less closure every call — in this hot per-pivot loop that alloc churn + // (and the reflect-based Swapper) dominated the entire query CPU. + for i := 1; i < len(cursors); i++ { + ci := cursors[i] + di := ci.curDoc() + j := i - 1 + for j >= 0 && cursors[j].curDoc() > di { + cursors[j+1] = cursors[j] + j-- + } + cursors[j+1] = ci + } + + theta := -1.0 + if h.full() { + theta = h.min() + } + + // Pivot by term-level max-score upper bounds (classic WAND). + cum := 0.0 + pivot := -1 + for i, c := range cursors { + cum += c.maxScore + if cum > theta { + pivot = i + break + } + } + if pivot < 0 { + break // no remaining doc can beat the current top-K + } + pivotDoc := cursors[pivot].curDoc() + + // Extend the pivot over every cursor also sitting on pivotDoc, so the + // block-max sum and skip bounds account for all of pivotDoc's + // contributors (a cursor beyond the term-UB pivot can still be at + // pivotDoc and add to its score). + for pivot+1 < len(cursors) && cursors[pivot+1].curDoc() == pivotDoc { + pivot++ + } + + // Block-Max refinement: the sum of the per-block upper bounds of + // cursors[0..pivot] for the blocks covering pivotDoc is a valid bound for + // every doc in [pivotDoc, minBlockEnd]. If it can't beat theta, skip the + // whole region instead of evaluating pivotDoc. + blockSum := 0.0 + for i := 0; i <= pivot; i++ { + blockSum += cursors[i].blockMax(pivotDoc) + } + if blockSum <= theta { + next := ordEnd + for i := 0; i <= pivot; i++ { + if e := cursors[i].blockEndAt(pivotDoc); e < next { + next = e + } + } + next++ // first doc beyond the limiting block + if pivot+1 < len(cursors) { + if nd := cursors[pivot+1].curDoc(); nd < next { + next = nd + } + } + if next <= pivotDoc { + // guarantee forward progress: when cursors are aligned at + // pivotDoc (or the next cursor sits on it), the smallest skip + // that still advances is past pivotDoc. + next = pivotDoc + 1 + } + cursors[chooseSkip(cursors, pivot, next)].skipTo(next) + continue + } + + if cursors[0].curDoc() == pivotDoc { + if allow == nil || allow.Contains(pivotDoc) { + score := 0.0 + for _, c := range cursors { + if c.curDoc() == pivotDoc { + score += c.score() + } + } + h.push(m.PkAt(pivotDoc), score) + } + for _, c := range cursors { + if c.curDoc() == pivotDoc { + c.advance() + } + } + } else { + // Not aligned: move a cursor before the pivot up to pivotDoc. + cursors[chooseSkip(cursors, pivot, pivotDoc)].skipTo(pivotDoc) + } + } +} + +// chooseSkip picks a cursor in [0..pivot] whose curDoc < target (so it makes +// progress), preferring the largest term max-score (skip the heaviest list). +// The block-skip / align callers guarantee at least one such cursor exists. +func chooseSkip(cursors []*cursor, pivot int, target int64) int { + best := -1 + var bestScore float64 + for i := 0; i <= pivot; i++ { + if cursors[i].curDoc() < target && (best < 0 || cursors[i].maxScore > bestScore) { + best = i + bestScore = cursors[i].maxScore + } + } + if best < 0 { + best = 0 // defensive; should not happen + } + return best +} + +// --------------------------------------------------------------------------- +// bounded top-K heap on doc ords (keeps the K largest scores; root = minimum) +// --------------------------------------------------------------------------- + +type topKEntry struct { + pk any // original primary key (resolved at push time; segments share one heap) + score float64 +} + +type topK struct { + limit int + entries []topKEntry +} + +func newTopK(limit int) *topK { + capHint := limit + if capHint > 1024 { + capHint = 1024 + } + return &topK{limit: limit, entries: make([]topKEntry, 0, capHint)} +} + +func (h *topK) full() bool { return len(h.entries) >= h.limit } + +func (h *topK) min() float64 { + if len(h.entries) == 0 { + return -1.0 + } + return h.entries[0].score +} + +func (h *topK) push(pk any, score float64) { + if len(h.entries) < h.limit { + h.entries = append(h.entries, topKEntry{pk, score}) + h.siftUp(len(h.entries) - 1) + return + } + if score > h.entries[0].score { + h.entries[0] = topKEntry{pk, score} + h.siftDown(0) + } +} + +func (h *topK) siftUp(i int) { + for i > 0 { + parent := (i - 1) / 2 + if h.entries[i].score >= h.entries[parent].score { + break + } + h.entries[i], h.entries[parent] = h.entries[parent], h.entries[i] + i = parent + } +} + +func (h *topK) siftDown(i int) { + n := len(h.entries) + for { + l := 2*i + 1 + if l >= n { + break + } + s := l + if r := l + 1; r < n && h.entries[r].score < h.entries[l].score { + s = r + } + if h.entries[s].score >= h.entries[i].score { + break + } + h.entries[i], h.entries[s] = h.entries[s], h.entries[i] + i = s + } +} + +// sorted drains the heap into results ordered by score desc (ties arbitrary). +func (h *topK) sorted() []SearchResult { + out := make([]SearchResult, len(h.entries)) + for i, e := range h.entries { + out[i] = SearchResult{DocID: e.pk, Score: e.score} + } + sort.Slice(out, func(i, j int) bool { return out[i].Score > out[j].Score }) + return out +} diff --git a/pkg/bm25/wand/serialize.go b/pkg/bm25/wand/serialize.go new file mode 100644 index 0000000000000..f3bc5d83a41e2 --- /dev/null +++ b/pkg/bm25/wand/serialize.go @@ -0,0 +1,674 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wand + +import ( + "archive/tar" + "bytes" + "encoding/binary" + "fmt" + "hash/crc32" + "io" + "math" + "sort" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/util" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +// Tar member names (cuVS-style multi-member archive). +const ( + memberDocmap = "docmap" // pkType + ord -> pk value + memberTermDict = "termdict" // out-of-dict term -> overflow word-id + memberWandIdx = "wandidx" // version + word-id -> ranking byte offset (resident index) + memberWandRank = "wandrank" // per-term self-contained block-max directory (mmap view, lazy) + memberWandBlk = "wandblk" // per-term docID/tf blocks (delta+varint; mmap view, block-decoded) +) + +// wandFormatV1 is the block-compressed WAND on-disk format: a RANKING directory of +// per-term self-contained Block-Max skip entries reachable by byte offset (the +// wandidx maps word-id -> that offset), plus a BLOCKS section of per-BlockSize-doc +// docID gaps (delta+varint) + raw tfs. Delta+varint shrinks docIDs ~4× and blocking +// them lets the loader keep them on the mmap and random-access one block, instead of +// the old flat, fully-resident postings. This is a fork of fulltext2's +// postingsFormatV6 with the positions section dropped (bm25 is position-free) and +// the FST replaced by the word-id->offset index. The format is free to break (no +// migration); a stale blob is rejected by this version byte. +const wandFormatV1 byte = 1 + +func log10(x float64) float64 { return math.Log10(x) } + +// Checksum returns the CRC32 (IEEE) of the serialized bytes. +func Checksum(b []byte) uint32 { return crc32.ChecksumIEEE(b) } + +// Serialize encodes the model into a tar archive of three members. +func (m *WandModel) Serialize() ([]byte, error) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + + docmap, err := m.encodeDocmap() + if err != nil { + return nil, err + } + if err := writeMember(tw, memberDocmap, docmap); err != nil { + return nil, err + } + if err := writeMember(tw, memberTermDict, m.encodeTermDict()); err != nil { + return nil, err + } + idx, ranking, blocks := m.encodeWand() + if err := writeMember(tw, memberWandIdx, idx); err != nil { + return nil, err + } + if err := writeMember(tw, memberWandRank, ranking); err != nil { + return nil, err + } + if err := writeMember(tw, memberWandBlk, blocks); err != nil { + return nil, err + } + if err := tw.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// Deserialize parses a tar archive produced by Serialize into a LOADED, queryable +// model: the small docmap / termdict members expand resident, while the ranking +// directory + docID/tf blocks stay as VIEWS into the read blob (decoded per term on +// demand, NOT expanded at load). This is the in-memory path (CDC tail frames, tests +// via bytes.Reader); the base-index path (LoadFromStorage) mmaps the file and binds +// the same views to it. The returned model retains the read bytes (ranking/blocks +// slice into them), so the blob is kept alive by GC. For a multi-GB base do NOT use +// this (it would ReadAll onto the Go heap) — use the mmap loader. +func Deserialize(id string, r io.Reader) (*WandModel, error) { + data, err := io.ReadAll(r) + if err != nil { + return nil, err + } + return decodeLoaded(id, data) +} + +// decodeLoaded builds a LOADED model over data (an in-memory blob or an mmap): the +// docmap/termdict expand resident, the ranking directory + blocks are bound as views +// into data, and per-term entries decode lazily (lookupTerm). Callers keep data alive +// (mmapData for a base, the retained slice/GC for a tail). +func decodeLoaded(id string, data []byte) (*WandModel, error) { + m := NewWandModel(id, 0) + docmap, termdict, idx, ranking, blocks, err := sliceMembers(data) + if err != nil { + return nil, err + } + if err := m.decodeDocmap(docmap); err != nil { + return nil, err + } + if err := m.decodeTermDict(termdict); err != nil { + return nil, err + } + if err := m.bindWand(idx, ranking, blocks); err != nil { + return nil, err + } + m.N = int64(len(m.pks)) + m.computeAvgDocLen() // per-term Block-Max stats come from disk (decodeTermEntry), lazily + return m, nil +} + +// sliceMembers returns the docmap / termdict / wandidx / wandrank / wandblk members +// as SLICES into data (zero-copy) by walking the tar and tracking the reader offset, +// so the same code serves an in-memory blob or an mmap'd file. +func sliceMembers(data []byte) (docmap, termdict, idx, ranking, blocks []byte, err error) { + br := bytes.NewReader(data) + tr := tar.NewReader(br) + for { + h, e := tr.Next() + if e == io.EOF { + break + } + if e != nil { + return nil, nil, nil, nil, nil, e + } + off := int64(len(data)) - int64(br.Len()) // tar positions br at the content + if off < 0 || off+h.Size > int64(len(data)) { + return nil, nil, nil, nil, nil, moerr.NewInternalErrorNoCtx("wand: member out of range") + } + seg := data[off : off+h.Size] + switch h.Name { + case memberDocmap: + docmap = seg + case memberTermDict: + termdict = seg + case memberWandIdx: + idx = seg + case memberWandRank: + ranking = seg + case memberWandBlk: + blocks = seg + } + } + return docmap, termdict, idx, ranking, blocks, nil +} + +func writeMember(tw *tar.Writer, name string, data []byte) error { + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(data)), Typeflag: tar.TypeReg}); err != nil { + return err + } + _, err := tw.Write(data) + return err +} + +// leBuf appends little-endian scalars to a bytes.Buffer WITHOUT the per-call heap +// allocation binary.Write incurs: binary.Write's `data any` parameter boxes every +// scalar to the heap, which in the per-term / per-doc serialize loops is millions +// of tiny garbage allocations. PutUintXX into the reused tmp array avoids it (the +// bytes are byte-identical to binary.Write(LittleEndian, ...), so the on-disk +// format is unchanged). Slice writes still go through binary.Write (one buffer +// alloc, no per-element boxing). +type leBuf struct { + b bytes.Buffer + tmp [binary.MaxVarintLen64]byte +} + +func (w *leBuf) u32(v uint32) { binary.LittleEndian.PutUint32(w.tmp[:4], v); w.b.Write(w.tmp[:4]) } +func (w *leBuf) u64(v uint64) { binary.LittleEndian.PutUint64(w.tmp[:8], v); w.b.Write(w.tmp[:8]) } +func (w *leBuf) i32(v int32) { w.u32(uint32(v)) } +func (w *leBuf) i64(v int64) { w.u64(uint64(v)) } + +// uvarint appends v as a LEB128 varint (1–10 bytes) — used for the delta-encoded +// docID gaps and the block directory, where values are small. +func (w *leBuf) uvarint(v uint64) { n := binary.PutUvarint(w.tmp[:], v); w.b.Write(w.tmp[:n]) } + +// encodePkLen writes a length-prefixed pk directly into the buffer — byte-identical +// to `binary.Write(len); Write(encodePk(...))` but without encodePk's per-pk small +// allocation for integer keys. Keep the type switch in sync with encodePk. +func (w *leBuf) encodePkLen(pkType int32, v any) error { + switch types.T(pkType) { + case types.T_int64: + w.u32(8) + w.u64(uint64(v.(int64))) + case types.T_uint64: + w.u32(8) + w.u64(v.(uint64)) + case types.T_int32: + w.u32(4) + w.u32(uint32(v.(int32))) + case types.T_uint32: + w.u32(4) + w.u32(v.(uint32)) + case types.T_varchar, types.T_char, types.T_text, types.T_datalink, + types.T_binary, types.T_varbinary, types.T_blob, types.T_json: + raw := asBytes(v) + w.u32(uint32(len(raw))) + w.b.Write(raw) + default: + // Any other type encodePk handles (e.g. uuid, stored as text) — no integer + // fast path, so length-prefix its encodePk bytes. + pkb, err := encodePk(pkType, v) + if err != nil { + return err + } + w.u32(uint32(len(pkb))) + w.b.Write(pkb) + } + return nil +} + +// ---- docmap: pkType + ord -> pk ---- + +func (m *WandModel) encodeDocmap() ([]byte, error) { + var w leBuf + w.i32(m.PkType) + w.i64(int64(len(m.pks))) + for _, pk := range m.pks { + if err := w.encodePkLen(m.PkType, pk); err != nil { + return nil, err + } + } + // per-doc length (ord-aligned with pks), for BM25. Zero-copy LE bytes (host is + // little-endian — decodeWand reads it back the same way with UnsafeSliceCast); + // avoids binary.Write's temp buffer + the io.Writer boxing that heap-allocates w. + w.b.Write(util.UnsafeSliceToBytes(m.docLen)) + return w.b.Bytes(), nil +} + +func (m *WandModel) decodeDocmap(data []byte) error { + r := bytes.NewReader(data) + if err := binary.Read(r, binary.LittleEndian, &m.PkType); err != nil { + return err + } + var n int64 + if err := binary.Read(r, binary.LittleEndian, &n); err != nil { + return err + } + m.pks = make([]any, n) + for i := int64(0); i < n; i++ { + var l uint32 + if err := binary.Read(r, binary.LittleEndian, &l); err != nil { + return err + } + raw := make([]byte, l) + if _, err := io.ReadFull(r, raw); err != nil { + return err + } + v, err := decodePk(m.PkType, raw) + if err != nil { + return err + } + m.pks[i] = v + } + m.docLen = make([]int32, n) + if err := binary.Read(r, binary.LittleEndian, m.docLen); err != nil { + return err + } + return nil +} + +// ---- termdict: overflow term -> word-id ---- + +func (m *WandModel) encodeTermDict() []byte { + var w leBuf + w.i64(int64(len(m.overflow))) + terms := make([]string, 0, len(m.overflow)) + for t := range m.overflow { + terms = append(terms, t) + } + sort.Strings(terms) // deterministic output + for _, term := range terms { + w.u32(uint32(len(term))) + w.b.WriteString(term) // WriteString avoids the []byte(term) copy + w.i32(m.overflow[term]) + } + return w.b.Bytes() +} + +func (m *WandModel) decodeTermDict(data []byte) error { + if len(data) == 0 { + return nil + } + r := bytes.NewReader(data) + var n int64 + if err := binary.Read(r, binary.LittleEndian, &n); err != nil { + return err + } + for i := int64(0); i < n; i++ { + var l uint32 + if err := binary.Read(r, binary.LittleEndian, &l); err != nil { + return err + } + tb := make([]byte, l) + if _, err := io.ReadFull(r, tb); err != nil { + return err + } + var id int32 + if err := binary.Read(r, binary.LittleEndian, &id); err != nil { + return err + } + m.overflow[string(tb)] = id + } + return nil +} + +// ---- wand: block-compressed postings keyed by int32 word-id ---- + +// encodeWand builds the three WAND members in one pass over the word-id-sorted +// terms (see wandFormatV1): +// +// - IDX (resident): version byte, nterms, then per term {word-id(i32), +// rankingOffset(i64)} — the word-id -> ranking byte-offset directory rebuilt into +// m.termOffsets at load (O(vocabulary) small ints). +// - RANKING (mmap view, lazy): per term a SELF-CONTAINED Block-Max entry at the +// offset the IDX records: df(varint), nblk(varint), blockDataBase(varint, the +// term's absolute offset into BLOCKS), maxTf(byte), minDl(varint), then per block +// {lastDocGap(varint from the previous block's last ord), blockMaxTf(byte), +// blockMinDl(varint), blkByteLen(varint)}. decodeTermEntry decodes ONE entry on +// demand — the resident directory heap is O(query), not O(vocabulary). +// - BLOCKS (mmap view, block-decoded): per term, per BlockSize-doc block the docID +// GAPS (varint, from the previous block's last ord) then the block's raw tf bytes. +// +// Works on a build-side model (terms map) and a loaded one (materializeDocIDs/Tfs + +// the resident block-max stats), and is deterministic given identical postings, so a +// build → load → re-Serialize round-trip is byte-identical. +func (m *WandModel) encodeWand() (idx, ranking, blocks []byte) { + type entry struct { + id int32 + tp *termPostings + } + list := make([]entry, 0, m.NumTerms()) + m.forEachTerm(func(id int32, tp *termPostings) { list = append(list, entry{id, tp}) }) + sort.Slice(list, func(i, j int) bool { return list[i].id < list[j].id }) // deterministic output + + var iw, rw, bw leBuf // index, ranking directory, blocks + iw.b.WriteByte(wandFormatV1) + iw.i64(int64(len(list))) + for _, e := range list { + tp := e.tp + if tp.blockLastDoc == nil { // defensive: a build-side term never finalized + deriveTermStats(tp, m.docLen) + } + docs := tp.materializeDocIDs() + tfs := tp.materializeTfs() + df := len(docs) + nblk := (df + BlockSize - 1) / BlockSize + + iw.i32(e.id) + iw.i64(int64(rw.b.Len())) // ranking offset of this term's self-contained entry + + rw.uvarint(uint64(df)) + rw.uvarint(uint64(nblk)) + rw.uvarint(uint64(bw.b.Len())) // blockDataBase: absolute offset into BLOCKS + rw.b.WriteByte(tp.maxTf) + rw.uvarint(uint64(uint32(tp.minDl))) + var prevLast int64 + for b := 0; b < nblk; b++ { + lo := b * BlockSize + hi := lo + BlockSize + if hi > df { + hi = df + } + blkStart := bw.b.Len() + prev := prevLast + for j := lo; j < hi; j++ { + bw.uvarint(uint64(docs[j] - prev)) + prev = docs[j] + } + bw.b.Write(tfs[lo:hi]) + + rw.uvarint(uint64(tp.blockLastDoc[b] - prevLast)) // lastDocGap + rw.b.WriteByte(tp.blockMaxTf[b]) + rw.uvarint(uint64(uint32(tp.blockMinDl[b]))) + rw.uvarint(uint64(bw.b.Len() - blkStart)) // this block's docID/tf byte length + prevLast = tp.blockLastDoc[b] + } + } + return iw.b.Bytes(), rw.b.Bytes(), bw.b.Bytes() +} + +// bindWand binds a LOADED model to its (mmap'd or in-memory) ranking + blocks +// sections and rebuilds the resident word-id -> ranking-offset map from the IDX +// member, WITHOUT expanding any term. Terms decode lazily via decodeTermEntry. It +// sets m.terms = nil (the build-side map is unused on a loaded model). +func (m *WandModel) bindWand(idx, ranking, blocks []byte) error { + m.terms = nil + m.termOffsets = map[int32]int64{} + m.ranking = ranking + m.blocks = blocks + if len(idx) == 0 { + return nil // no terms (empty index) + } + if len(idx) < 1+8 { + return moerr.NewInternalErrorNoCtx("wand: wandidx blob too short") + } + if idx[0] != wandFormatV1 { + return moerr.NewInternalErrorNoCtx(fmt.Sprintf("wand: unsupported wand format %d", idx[0])) + } + nterms := int64(binary.LittleEndian.Uint64(idx[1:9])) + if nterms < 0 { + return moerr.NewInternalErrorNoCtx("wand: negative nterms") + } + m.termOffsets = make(map[int32]int64, nterms) + off := 9 + for t := int64(0); t < nterms; t++ { + if off+12 > len(idx) { + return moerr.NewInternalErrorNoCtx("wand: wandidx truncated") + } + wid := int32(binary.LittleEndian.Uint32(idx[off:])) + ro := int64(binary.LittleEndian.Uint64(idx[off+4:])) + off += 12 + if ro < 0 || ro >= int64(len(ranking)) { + return moerr.NewInternalErrorNoCtx("wand: ranking offset out of range") + } + m.termOffsets[wid] = ro + } + return nil +} + +// decodeTermEntry lazily decodes ONE term's self-contained directory entry at byte +// offset `off` in m.ranking into a transient termPostings whose blockData is a view +// into m.blocks. Callers hold the result for the query's lifetime (WAND cursor, +// forEachTerm), so the entry decodes at most a few times per query and never persists — +// the resident directory heap is O(query), not O(vocabulary). Returns (nil,false) on a +// corrupt/out-of-bounds entry (defense-in-depth on already-checksummed data). +func (m *WandModel) decodeTermEntry(off int64) (*termPostings, bool) { + r := m.ranking + if off < 0 || off >= int64(len(r)) { + return nil, false + } + p := int(off) + uv := func() (uint64, bool) { + v, n := binary.Uvarint(r[p:]) + if n <= 0 { + return 0, false + } + p += n + return v, true + } + dfu, ok := uv() + if !ok { + return nil, false + } + nblku, ok := uv() + if !ok { + return nil, false + } + baseB, ok := uv() // blockDataBase: absolute offset into m.blocks + if !ok { + return nil, false + } + if p >= len(r) { + return nil, false + } + termMaxTf := r[p] + p++ + minDlu, ok := uv() + if !ok { + return nil, false + } + // A df/nblk larger than the blocks section is impossible (each posting is >= 1 byte). + if dfu > uint64(len(m.blocks)) || nblku > uint64(len(m.blocks)) { + return nil, false + } + df, nblk := int(dfu), int(nblku) + tp := &termPostings{ndoc: df, maxTf: termMaxTf, minDl: int32(minDlu)} + if nblk == 0 { + return tp, true + } + tp.blockLastDoc = make([]int64, nblk) + tp.blockMaxTf = make([]uint8, nblk) + tp.blockMinDl = make([]int32, nblk) + tp.blockOff = make([]int32, nblk+1) // byte offsets RELATIVE to this term's blockData + var prevLast, cb int64 + for b := 0; b < nblk; b++ { + gap, ok := uv() + if !ok { + return nil, false + } + prevLast += int64(gap) + tp.blockLastDoc[b] = prevLast + if p >= len(r) { + return nil, false + } + tp.blockMaxTf[b] = r[p] + p++ + mdl, ok := uv() + if !ok { + return nil, false + } + tp.blockMinDl[b] = int32(mdl) + blkLen, ok := uv() + if !ok || blkLen > uint64(len(m.blocks)) { + return nil, false + } + tp.blockOff[b] = int32(cb) + cb += int64(blkLen) + } + tp.blockOff[nblk] = int32(cb) + if int64(baseB)+cb > int64(len(m.blocks)) { + return nil, false + } + tp.blockData = m.blocks[baseB : int64(baseB)+cb] + return tp, true +} + +// ---- pk codec (by types.T) ---- + +// encodePk serializes a primary-key value to bytes given its type. +func encodePk(pkType int32, v any) ([]byte, error) { + switch types.T(pkType) { + case types.T_int64: + return packUint64(uint64(v.(int64))), nil + case types.T_uint64: + return packUint64(v.(uint64)), nil + case types.T_int32: + return packUint32(uint32(v.(int32))), nil + case types.T_uint32: + return packUint32(v.(uint32)), nil + case types.T_varchar, types.T_char, types.T_text, types.T_datalink, + types.T_binary, types.T_varbinary, types.T_blob, types.T_json: + return asBytes(v), nil + case types.T_uuid: + // Stored as the canonical TEXT form (like a varlena string), because that + // is how the SQL-based CDC delivers a uuid pk (extractRowFromVector -> + // Uuid.String()). The sync build (GetAny) delivers a types.Uuid instead, so + // stringify it — Uuid.String() is deterministic and scale-free, so the same + // uuid stores identically regardless of which path produced it. The + // membership prefilter re-encodes the loaded pk (a types.Uuid) the same way. + switch x := v.(type) { + case string: + return []byte(x), nil + case types.Uuid: + return []byte(x.String()), nil + case []byte: + return append([]byte(nil), x...), nil + default: + return nil, moerr.NewInternalErrorNoCtxf("wand: uuid pk unexpected go type %T", v) + } + // Native fixed-width temporal / decimal pks. Delivered natively by the ISCP + // extractor's ReprNative mode (extractRowFromVector) — NOT as a SQL-display + // string — so they encode as their exact raw bytes (deterministic, reversible) + // rather than a lossy round-trip through Datetime.String2()/Decimal.Format(). + case types.T_date: + return packUint32(uint32(int32(v.(types.Date)))), nil + case types.T_datetime: + return packUint64(uint64(int64(v.(types.Datetime)))), nil + case types.T_time: + return packUint64(uint64(int64(v.(types.Time)))), nil + case types.T_timestamp: + return packUint64(uint64(int64(v.(types.Timestamp)))), nil + case types.T_decimal64: + return packUint64(uint64(v.(types.Decimal64))), nil + case types.T_decimal128: + d := v.(types.Decimal128) + b := make([]byte, 16) + binary.LittleEndian.PutUint64(b[0:8], d.B0_63) + binary.LittleEndian.PutUint64(b[8:16], d.B64_127) + return b, nil + default: + return nil, moerr.NewInternalErrorNoCtxf("wand: unsupported pk type %d", pkType) + } +} + +// pkFixedWidth returns the fixed byte width encodePk emits for a fixed-width pk +// type and true, or (-1, false) for a variable-length (varlena — varchar/char/ +// text/blob/…) type, so callers can distinguish "variable length" (needs a per-pk +// length prefix) from a fixed width. Callers that store many pks (the delete log) +// use it to drop the length prefix for fixed types. +// +// The width comes from types.T.FixedLength() (the canonical source) rather than a +// second hardcoded copy. encodePk handles more than the integer widths (varlena, +// the fixed-width temporal/decimal types, and uuid), but pkFixedWidth deliberately +// fast-paths only the four integer widths: the temporal/decimal types (and uuid, +// which encodePk stores as its variable-looking canonical text form) simply take +// the length-prefixed varlena path below. That is correct — just not maximally +// compact for the fixed-width temporal/decimal pks. Promoting those to fixed-width +// here is a delete-log/docmap on-disk format change and must be done with matching +// encode/decode-symmetry tests, not as an incidental tweak. +func pkFixedWidth(pkType int32) (int, bool) { + switch t := types.T(pkType); t { + case types.T_int64, types.T_uint64, types.T_int32, types.T_uint32: + return t.FixedLength(), true + default: + return -1, false // variable-length (varlena) or length-prefixed fixed type + } +} + +// decodePk reverses encodePk, producing the Go value AppendAny expects. +func decodePk(pkType int32, b []byte) (any, error) { + switch types.T(pkType) { + case types.T_int64: + return int64(binary.LittleEndian.Uint64(b)), nil + case types.T_uint64: + return binary.LittleEndian.Uint64(b), nil + case types.T_int32: + return int32(binary.LittleEndian.Uint32(b)), nil + case types.T_uint32: + return binary.LittleEndian.Uint32(b), nil + case types.T_varchar, types.T_char, types.T_text, types.T_datalink, + types.T_binary, types.T_varbinary, types.T_blob, types.T_json: + return append([]byte(nil), b...), nil + case types.T_uuid: + // Stored as canonical text; parse back to types.Uuid — the search output + // doc_id column is uuid-typed and INNER-JOINed to src.id (apply_indices_ + // fulltext.go), so AppendAny needs a types.Uuid, and a uniform types.Uuid + // keeps normalizeKey consistent across segments and delete frames. + u, err := types.ParseUuid(string(b)) + if err != nil { + return nil, err + } + return u, nil + // Native fixed-width temporal / decimal pks (see encodePk). Reproduce the exact + // native Go value AppendAny / the membership prefilter expect (the doc_id output + // column is the same source type, so a uniform native value keeps normalizeKey + // consistent across segments and delete frames). + case types.T_date: + return types.Date(int32(binary.LittleEndian.Uint32(b))), nil + case types.T_datetime: + return types.Datetime(int64(binary.LittleEndian.Uint64(b))), nil + case types.T_time: + return types.Time(int64(binary.LittleEndian.Uint64(b))), nil + case types.T_timestamp: + return types.Timestamp(int64(binary.LittleEndian.Uint64(b))), nil + case types.T_decimal64: + return types.Decimal64(binary.LittleEndian.Uint64(b)), nil + case types.T_decimal128: + return types.Decimal128{ + B0_63: binary.LittleEndian.Uint64(b[0:8]), + B64_127: binary.LittleEndian.Uint64(b[8:16]), + }, nil + default: + return nil, moerr.NewInternalErrorNoCtxf("wand: unsupported pk type %d", pkType) + } +} + +func packUint64(v uint64) []byte { + b := make([]byte, 8) + binary.LittleEndian.PutUint64(b, v) + return b +} + +func packUint32(v uint32) []byte { + b := make([]byte, 4) + binary.LittleEndian.PutUint32(b, v) + return b +} + +func asBytes(v any) []byte { + switch x := v.(type) { + case []byte: + return x + case string: + return []byte(x) + default: + return nil + } +} diff --git a/pkg/bm25/wand/sink.go b/pkg/bm25/wand/sink.go new file mode 100644 index 0000000000000..5c58f3bbe7da9 --- /dev/null +++ b/pkg/bm25/wand/sink.go @@ -0,0 +1,316 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wand + +import ( + "bytes" + "encoding/binary" + "fmt" + "hash/crc32" + "io" + "strings" + + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/sqlquote" + "github.com/matrixorigin/matrixone/pkg/vectorindex" +) + +// The sink side of Phase B: the ISCP consumer's WandSqlWriter accumulates CDC +// rows into a WandCdc, serializes it through the ISCP channel (Encode), and +// RunWand decodes it (DecodeWandCdc) and STREAMS it through a TailBuilder into +// capacity-capped tag=1 CdcTail segments appended to the store (TailFileInsertSqls). +// (BuildTailFrames below is the non-streaming equivalent, kept for tests / +// small in-memory batches.) The blob is BINARY (typed pk via encodePk) — unlike +// the HNSW JSON path, because a retrieval pk is `any` (int64 OR varchar) and a +// JSON round-trip would corrupt a non-int pk. + +type wandCdcOp byte + +const ( + cdcInsert wandCdcOp = 'I' + cdcUpsert wandCdcOp = 'U' + cdcDelete wandCdcOp = 'D' +) + +// CdcEvent is one source-row mutation: an INSERT/UPSERT carries the row's text +// (tokenized at build), a DELETE carries only the pk. +type CdcEvent struct { + Op wandCdcOp + Pk any + Text string +} + +// WandCdc is the per-flush CDC batch the sinker accumulates and ships as one +// channel blob. +type WandCdc struct { + PkType int32 + Events []CdcEvent +} + +func NewWandCdc(pkType int32) *WandCdc { return &WandCdc{PkType: pkType} } + +func (c *WandCdc) Insert(pk any, text string) { + c.Events = append(c.Events, CdcEvent{cdcInsert, pk, text}) +} +func (c *WandCdc) Upsert(pk any, text string) { + c.Events = append(c.Events, CdcEvent{cdcUpsert, pk, text}) +} +func (c *WandCdc) Delete(pk any) { c.Events = append(c.Events, CdcEvent{cdcDelete, pk, ""}) } +func (c *WandCdc) Len() int { return len(c.Events) } + +const wandCdcMagic uint32 = 0x57440200 // 'W' 'D' 02 00 + +// Encode serializes the batch: magic | pkType | count | +// [op:1 pkLen:u32 pk textLen:u32 text]* | crc32. Self-describing + CRC-checked. +func (c *WandCdc) Encode() ([]byte, error) { + var b bytes.Buffer + _ = binary.Write(&b, binary.LittleEndian, wandCdcMagic) + _ = binary.Write(&b, binary.LittleEndian, c.PkType) + _ = binary.Write(&b, binary.LittleEndian, int64(len(c.Events))) + for _, e := range c.Events { + pkb, err := encodePk(c.PkType, e.Pk) + if err != nil { + return nil, err + } + b.WriteByte(byte(e.Op)) + _ = binary.Write(&b, binary.LittleEndian, uint32(len(pkb))) + b.Write(pkb) + _ = binary.Write(&b, binary.LittleEndian, uint32(len(e.Text))) + b.WriteString(e.Text) + } + sum := crc32.ChecksumIEEE(b.Bytes()) + _ = binary.Write(&b, binary.LittleEndian, sum) + return b.Bytes(), nil +} + +// DecodeWandCdc reverses Encode, validating magic + CRC. +func DecodeWandCdc(buf []byte) (*WandCdc, error) { + if len(buf) < 4+4+8+4 { + return nil, moerr.NewInternalErrorNoCtx("wand cdc: truncated") + } + body := buf[:len(buf)-4] + if crc32.ChecksumIEEE(body) != binary.LittleEndian.Uint32(buf[len(buf)-4:]) { + return nil, moerr.NewInternalErrorNoCtx("wand cdc: checksum mismatch") + } + r := bytes.NewReader(body) + var magic uint32 + _ = binary.Read(r, binary.LittleEndian, &magic) + if magic != wandCdcMagic { + return nil, moerr.NewInternalErrorNoCtx("wand cdc: bad magic") + } + c := &WandCdc{} + if err := binary.Read(r, binary.LittleEndian, &c.PkType); err != nil { + return nil, err + } + var n int64 + if err := binary.Read(r, binary.LittleEndian, &n); err != nil { + return nil, err + } + if n < 0 { + return nil, moerr.NewInternalErrorNoCtx("wand cdc: bad count") + } + c.Events = make([]CdcEvent, 0, n) + for i := int64(0); i < n; i++ { + op, err := r.ReadByte() + if err != nil { + return nil, err + } + pk, err := readLenBytes(r, c.PkType) + if err != nil { + return nil, err + } + text, err := readLenString(r) + if err != nil { + return nil, err + } + c.Events = append(c.Events, CdcEvent{Op: wandCdcOp(op), Pk: pk, Text: text}) + } + return c, nil +} + +func readLenBytes(r *bytes.Reader, pkType int32) (any, error) { + var l uint32 + if err := binary.Read(r, binary.LittleEndian, &l); err != nil { + return nil, err + } + // Bounds-check the length against the bytes actually remaining, then read fully: + // a bare r.Read can short-read (n int64(r.Len()) { + return nil, moerr.NewInternalErrorNoCtx("wand cdc: truncated pk") + } + pkb := make([]byte, l) + if _, err := io.ReadFull(r, pkb); err != nil { + return nil, err + } + return decodePk(pkType, pkb) +} + +func readLenString(r *bytes.Reader) (string, error) { + var l uint32 + if err := binary.Read(r, binary.LittleEndian, &l); err != nil { + return "", err + } + if int64(l) > int64(r.Len()) { + return "", moerr.NewInternalErrorNoCtx("wand cdc: truncated text") + } + sb := make([]byte, l) + if _, err := io.ReadFull(r, sb); err != nil { + return "", err + } + return string(sb), nil +} + +// BuildTailFrames turns one in-memory CDC batch into tag=1 CdcTail frames, +// starting at startChunkId, and returns them plus the next free chunk_id. +// INSERT/UPSERT rows are tokenized (via the injected tokenizer — kept out of this +// package so it stays dependency-light and unit-testable) into `capacity`-capped +// delta segments; DELETE rows become one delete frame. +// +// NON-STREAMING: it builds ALL of cdc's inserts in memory before framing, so the +// production sinker uses the streaming TailBuilder instead (bounded to one open +// segment). BuildTailFrames is kept for tests / small batches. +// +// Ordering: the delete frame is emitted FIRST (lowest chunk_id) so a same-batch +// UPDATE (delivered as DELETE old + INSERT new) resolves correctly — the new +// segment sits at a higher chunk_id, and ComputeLiveness kills only segments +// with chunk_id STRICTLY below the delete, so the fresh copy survives while the +// base copy (chunk_id below the delete) is dropped. +// +// A segment frame larger than MaxChunkSize is split across chunk rows at persist +// and reassembled at load (Bug 1) — capacity no longer needs to keep a segment +// within one storage row; it sizes from max_index_capacity. +func BuildTailFrames(cdc *WandCdc, capacity int64, startChunkId int64, tokenize func(string) []string) ([]TailFrame, int64, error) { + b := NewBuilder(fmt.Sprintf("cdctail-%d", startChunkId), cdc.PkType) + var deletes []DeleteRecord + for _, e := range cdc.Events { + switch e.Op { + case cdcInsert, cdcUpsert: + for _, w := range tokenize(e.Text) { + if err := b.Add(w, e.Pk); err != nil { + return nil, 0, err + } + } + case cdcDelete: + deletes = append(deletes, DeleteRecord{Pk: e.Pk}) + } + } + + segs := b.FinishSegments(capacity) + frames := make([]TailFrame, 0, len(segs)+1) // +1 for the optional deletes frame + chunkId := startChunkId + if len(deletes) > 0 { + frame, err := FrameDeletes(cdc.PkType, deletes) + if err != nil { + return nil, 0, err + } + frames = append(frames, TailFrame{Recency: chunkId, Data: frame}) + chunkId += frameChunkCount(len(frame)) + } + for _, seg := range segs { + if seg.N == 0 { + seg.Free() + continue + } + frame, err := FrameSegment(seg) + seg.Free() + if err != nil { + return nil, 0, err + } + frames = append(frames, TailFrame{Recency: chunkId, Data: frame}) + chunkId += frameChunkCount(len(frame)) + } + return frames, chunkId, nil +} + +// NextTailChunkIdSql returns a SELECT for the next free tag=1 CdcTail chunk_id — +// the monotonic append position the sinker frames at. It is MAX over BOTH the +// tail chunk_ids AND the tag=0 base recencies (metadata.chunk_id) + 1, floored so +// the first append is 1 (base recency 0 = oldest is reserved). Widening past the +// base recencies is what keeps the sequence from resetting after a compaction +// folds the tail into a base at chunk_id K and deletes the tail ≤ K: the next +// append continues at K+1, still newer than that base. +func NextTailChunkIdSql(cfg TableConfig) string { + tailMax := fmt.Sprintf("COALESCE((SELECT MAX(%s) FROM %s WHERE %s = %s AND %s = %d), 0)", + catalog.Bm25Index_TblCol_Storage_Chunk_Id, sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + catalog.Bm25Index_TblCol_Storage_Index_Id, sqlquote.String(vectorindex.CdcTailId), + catalog.Bm25Index_TblCol_Storage_Tag, int(vectorindex.Tag_CdcEvents)) + baseMax := fmt.Sprintf("COALESCE((SELECT MAX(%s) FROM %s), 0)", + catalog.Bm25Index_TblCol_Metadata_Recency, sqlquote.QualifiedIdent(cfg.DbName, cfg.MetadataTable)) + return fmt.Sprintf("SELECT GREATEST(%s, %s) + 1", tailMax, baseMax) +} + +// FrameChunkCount is the exported form: how many MaxChunkSize storage rows a frame +// of frameLen bytes occupies. The streaming sinker uses it to advance chunk_id past +// each spilled segment without holding the framed bytes. +func FrameChunkCount(frameLen int) int64 { return frameChunkCount(frameLen) } + +// frameChunkCount is the number of MaxChunkSize storage rows a frame of this many +// bytes occupies (>= 1). A large segment frame is split across several rows +// because the store's data column is capped at MaxChunkSize (64 KB). +func frameChunkCount(frameLen int) int64 { + n := int64((frameLen + vectorindex.MaxChunkSize - 1) / vectorindex.MaxChunkSize) + if n < 1 { + n = 1 + } + return n +} + +// maxInsertTuples caps VALUES tuples per INSERT (matches HNSW's 2000). Each tuple's +// load_file reads a MaxChunkSize file chunk into memory at execution, so an unbounded +// single INSERT would materialize the whole index at once → OOM / GC pressure. This +// bounds a persist statement to ~maxInsertTuples*MaxChunkSize resident. +const maxInsertTuples = 2000 + +// FileChunkInsertSqls renders the storage INSERTs that read a FILE directly via +// load_file — no hex/unhex, and (for the streaming sinker) no read-back to memory: +// the frame is ALREADY on disk. It splits [0..dataLen) across MaxChunkSize chunk +// rows from startChunkId under (index_id=id, tag), batching <= maxInsertTuples tuples +// per INSERT. Mirrors HNSW's ToSql. The file MUST exist when the INSERT executes, so +// the caller keeps it until the persist txn commits. A frame larger than MaxChunkSize +// is thus split across consecutive chunk_ids and reassembled at load via CdcFrameLen. +func FileChunkInsertSqls(cfg TableConfig, id string, startChunkId int64, path string, dataLen int, tag int) []string { + prefix := fmt.Sprintf("INSERT INTO %s (%s, %s, %s, %s) VALUES ", + sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + catalog.Bm25Index_TblCol_Storage_Index_Id, catalog.Bm25Index_TblCol_Storage_Chunk_Id, + catalog.Bm25Index_TblCol_Storage_Data, catalog.Bm25Index_TblCol_Storage_Tag) + var sqls, vals []string + chunkID := startChunkId + for off := 0; off < dataLen; off += vectorindex.MaxChunkSize { + sz := vectorindex.MaxChunkSize + if off+sz > dataLen { + sz = dataLen - off + } + url := fmt.Sprintf("file://%s?offset=%d&size=%d", path, off, sz) + vals = append(vals, fmt.Sprintf("(%s, %d, load_file(cast(%s as datalink)), %d)", + sqlquote.String(id), chunkID, sqlquote.String(url), tag)) + chunkID++ + if len(vals) == maxInsertTuples { + sqls = append(sqls, prefix+strings.Join(vals, ", ")) + vals = vals[:0] + } + } + if len(vals) > 0 { + sqls = append(sqls, prefix+strings.Join(vals, ", ")) + } + return sqls +} + +// TailFileInsertSqls is FileChunkInsertSqls for the tag=1 CdcTail (index_id = +// CdcTailId, tag = Tag_CdcEvents) — the streaming sinker's spilled frame files. +func TailFileInsertSqls(cfg TableConfig, startChunkId int64, path string, frameLen int) []string { + return FileChunkInsertSqls(cfg, vectorindex.CdcTailId, startChunkId, path, frameLen, int(vectorindex.Tag_CdcEvents)) +} diff --git a/pkg/bm25/wand/sink_test.go b/pkg/bm25/wand/sink_test.go new file mode 100644 index 0000000000000..ad0a1d8e63d5d --- /dev/null +++ b/pkg/bm25/wand/sink_test.go @@ -0,0 +1,124 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wand + +import ( + "strings" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +// TestWandCdcRoundTrip round-trips the binary CDC blob for both int64 and +// varchar pks — the varchar case is the reason the blob is binary (typed +// encodePk) and not JSON, which would corrupt a non-integer pk. +func TestWandCdcRoundTrip(t *testing.T) { + t.Run("int64", func(t *testing.T) { + c := NewWandCdc(testPkType) + c.Insert(int64(1), "营养 早餐") + c.Upsert(int64(2), "视频") + c.Delete(int64(3)) + buf, err := c.Encode() + if err != nil { + t.Fatal(err) + } + got, err := DecodeWandCdc(buf) + if err != nil { + t.Fatal(err) + } + if got.PkType != c.PkType || len(got.Events) != 3 { + t.Fatalf("header/count wrong: %+v", got) + } + want := c.Events + for i, e := range got.Events { + if e.Op != want[i].Op || e.Pk.(int64) != want[i].Pk.(int64) || e.Text != want[i].Text { + t.Fatalf("event %d mismatch: want %+v got %+v", i, want[i], e) + } + } + // corruption is detected. + buf[10] ^= 0xff + if _, err := DecodeWandCdc(buf); err == nil { + t.Fatal("expected checksum mismatch") + } + }) + + t.Run("varchar", func(t *testing.T) { + c := NewWandCdc(int32(types.T_varchar)) + c.Insert([]byte("doc-a"), "hello") + c.Delete([]byte("doc-b")) + buf, err := c.Encode() + if err != nil { + t.Fatal(err) + } + got, err := DecodeWandCdc(buf) + if err != nil { + t.Fatal(err) + } + if len(got.Events) != 2 || + string(got.Events[0].Pk.([]byte)) != "doc-a" || got.Events[0].Text != "hello" || + string(got.Events[1].Pk.([]byte)) != "doc-b" { + t.Fatalf("varchar round-trip wrong: %+v", got.Events) + } + }) +} + +// TestBuildTailFrames drives one CDC batch (2 inserts + a delete) into tag=1 +// frames, checks the delete frame is emitted first (lower chunk_id), then +// assembles them alongside a base segment holding the deleted pk and asserts the +// search reflects the delete + the new docs. +func TestBuildTailFrames(t *testing.T) { + tokenize := func(s string) []string { return strings.Fields(s) } + c := NewWandCdc(testPkType) + c.Insert(int64(1), "x y") + c.Insert(int64(2), "x") + c.Delete(int64(3)) + + frames, next, err := BuildTailFrames(c, 1<<20, 5, tokenize) + if err != nil { + t.Fatal(err) + } + // delete frame @5 (first), one insert segment @6; next free chunk_id = 7. + if next != 7 || len(frames) != 2 || frames[0].Recency != 5 || frames[1].Recency != 6 { + t.Fatalf("frame layout wrong: next=%d frames=%d ids=%v", next, len(frames), + []int64{frames[0].Recency, frames[1].Recency}) + } + + // Assemble the tail alongside a base segment that holds pk 3 → the tail's + // delete (chunk 5 > base -1) drops it; the new docs 1,2 are searchable. + base := buildSeg(t, baseRecency, map[int64][]string{3: {"x"}}) + defer base.Free() + tailSegs, deletes, err := AssembleFrames(frames) + if err != nil { + t.Fatal(err) + } + defer freeSegs(tailSegs) + segs := append([]*WandModel{base}, tailSegs...) + + wantSet := func(got, want map[int64]int) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("want %v, got %v", want, got) + } + for pk, n := range want { + if got[pk] != n { + t.Fatalf("pk %d: want %d, got %d (full %v)", pk, n, got[pk], got) + } + } + } + // "x" matches docs 1,2 (new) but not deleted 3. + wantSet(pkCounts(searchSegsLive(segs, deletes, []string{"x"}, 10, nil)), map[int64]int{1: 1, 2: 1}) + // "y" only tokenized into doc 1. + wantSet(pkCounts(searchSegsLive(segs, deletes, []string{"y"}, 10, nil)), map[int64]int{1: 1}) +} diff --git a/pkg/bm25/wand/storage.go b/pkg/bm25/wand/storage.go new file mode 100644 index 0000000000000..1364950881605 --- /dev/null +++ b/pkg/bm25/wand/storage.go @@ -0,0 +1,611 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wand + +import ( + "context" + "fmt" + "os" + "sync" + + "github.com/google/uuid" + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/sqlquote" + "github.com/matrixorigin/matrixone/pkg/common/system" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/util/executor" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" +) + +// TableConfig is the JSON config passed to the fulltext_wand_create / +// fulltext_wand_search TVFs (const string arg 0). It locates the persistent +// WAND chunk store + metadata table for an index, mirroring +// vectorindex.IndexTableConfig. +type TableConfig struct { + DbName string `json:"db"` + SrcTable string `json:"src"` + IndexTable string `json:"index"` // chunk store (Bm25Index_TblType_Storage) + MetadataTable string `json:"metadata"` // metadata (Bm25Index_TblType_Metadata) + PKey string `json:"pkey"` + // Capacity is max_index_capacity, resolved by the compile layer from the index's + // persisted algo_params (the immutable flat param) and carried to the create-build TVF + // so the base is split at the same value every compaction later reads. 0 ⇒ unset (the + // TVF falls back to the resolver, for older indexes without the flat param). + Capacity int64 `json:"capacity,omitempty"` + // FromSource selects the create-build TVF's input shape. false (default): the TVF + // reads pre-tokenized postings rows (argVecs = [cfg, word, doc_id]). true: the TVF + // reads SOURCE rows (argVecs = [cfg, pk, cols…]) and tokenizes them in-Go — one build + // statement straight off the source, so the postings table is never populated. + FromSource bool `json:"from_source,omitempty"` +} + +// SubIndexId is the index_id for the i-th tag=0 base sub-index of a build identified by +// uid. All of an index's sub-indexes share the one storage + metadata table and are told +// apart by this id (mirrors HNSW's ":"). uid MUST carry a per-build-unique +// component (e.g. the build timestamp) — NOT a deterministic table-derived name — so two +// builds (concurrent across CNs, or a rebuild) never write colliding ids. Load +// enumerates the ids from the metadata table, so the exact form only needs uniqueness. +func SubIndexId(uid string, i int) string { + return fmt.Sprintf("%s:%d", uid, i) +} + +// ToInsertSqls serializes the model, SPILLS it to a temp file, and emits the SQL to +// persist it: one metadata row (timestamp, md5 checksum, filesize) plus the index +// bytes split into <= MaxChunkSize (index_id, chunk_id, data, tag) rows read straight +// from the file via load_file — NO hex/unhex (which doubled the SQL text and had to +// be re-parsed). Mirrors HNSW's ToSql. The returned cleanup MUST be called after the +// SQLs run (they read the temp file at execution) — typically deferred by the caller. +// +// tag selects the storage tier (Phase B): tag=0 = the compacted main index +// (the sync CREATE/REINDEX build and idxcron's merged output); tag=1 = an +// incremental CDC delta segment appended by the ISCP sinker. Both kinds coexist +// in the same ft_index store and are distinguished only by this column. +func (m *WandModel) ToInsertSqls(cfg TableConfig, ts int64, tag int) (sqls []string, cleanup func(), err error) { + buf, err := m.Serialize() + if err != nil { + return nil, nil, err + } + checksum := vectorindex.CheckSumFromBuffer(buf) + filesize := int64(len(buf)) + + fp, err := os.CreateTemp("", "wandbuild") + if err != nil { + return nil, nil, err + } + path := fp.Name() + cleanup = func() { fp.Close(); os.Remove(path) } + if _, err = fp.Write(buf); err != nil { + cleanup() + return nil, nil, err + } + if err = fp.Sync(); err != nil { // durable on disk before load_file reads it + cleanup() + return nil, nil, err + } + + metaTbl := sqlquote.QualifiedIdent(cfg.DbName, cfg.MetadataTable) + sqls = append(sqls, fmt.Sprintf("INSERT INTO %s (%s, %s, %s, %s, %s, %s) VALUES (%s, %d, %s, %d, %d, %d)", + metaTbl, + catalog.Bm25Index_TblCol_Metadata_Index_Id, catalog.Bm25Index_TblCol_Metadata_Timestamp, + catalog.Bm25Index_TblCol_Metadata_Checksum, catalog.Bm25Index_TblCol_Metadata_Filesize, + catalog.Bm25Index_TblCol_Metadata_Recency, catalog.Bm25Index_TblCol_Metadata_Nrow, + sqlquote.String(m.Id), ts, sqlquote.String(checksum), filesize, m.Recency, m.N)) + sqls = append(sqls, FileChunkInsertSqls(cfg, m.Id, 0, path, int(filesize), tag)...) + return sqls, cleanup, nil +} + +// DeleteSqls returns the SQL to remove an index id's chunks + metadata row +// (used before a rebuild so reindex is idempotent). +func DeleteSqls(cfg TableConfig, id string) []string { + return []string{ + fmt.Sprintf("DELETE FROM %s WHERE %s = %s", sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + catalog.Bm25Index_TblCol_Storage_Index_Id, sqlquote.String(id)), + fmt.Sprintf("DELETE FROM %s WHERE %s = %s", sqlquote.QualifiedIdent(cfg.DbName, cfg.MetadataTable), + catalog.Bm25Index_TblCol_Metadata_Index_Id, sqlquote.String(id)), + } +} + +// readMetadata fetches an index id's tag=0 metadata (checksum + filesize). +// found=false means no metadata row exists (a tag=0 base was never built — e.g. +// an index created on an empty table, whose corpus is entirely tag=1 CDC deltas). +func readMetadata(sqlproc *sqlexec.SqlProcess, cfg TableConfig, id string) (checksum string, filesize int64, chunkId int64, found bool, err error) { + metaSQL := fmt.Sprintf("SELECT %s, %s, %s FROM %s WHERE %s = %s", + catalog.Bm25Index_TblCol_Metadata_Checksum, catalog.Bm25Index_TblCol_Metadata_Filesize, + catalog.Bm25Index_TblCol_Metadata_Recency, + sqlquote.QualifiedIdent(cfg.DbName, cfg.MetadataTable), + catalog.Bm25Index_TblCol_Metadata_Index_Id, sqlquote.String(id)) + mres, err := sqlexec.RunSql(sqlproc, metaSQL) + if err != nil { + return "", 0, 0, false, err + } + for _, bat := range mres.Batches { + if bat == nil || bat.RowCount() == 0 { + continue + } + checksum = bat.Vecs[0].GetStringAt(0) + filesize = vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[1], 0) + chunkId = vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[2], 0) + found = true + break + } + mres.Close() + return checksum, filesize, chunkId, found, nil +} + +// DeleteAllBasesSqls removes every tag=0 base sub-index — all tag=0 chunk rows (of all +// sub-index ids) from the storage table plus all metadata rows — so the CREATE build is +// idempotent when several sub-indexes exist. The tag=1 CdcTail is untouched. +func DeleteAllBasesSqls(cfg TableConfig) []string { + return []string{ + fmt.Sprintf("DELETE FROM %s WHERE %s = %d", sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + catalog.Bm25Index_TblCol_Storage_Tag, int(vectorindex.Tag_ModelChunk)), + // WHERE TRUE, not a bare DELETE: a bare DELETE takes MO's truncate fast-path + // (DROP + RECREATE the metadata hidden table object); WHERE TRUE keeps the object. + fmt.Sprintf("DELETE FROM %s WHERE TRUE", sqlquote.QualifiedIdent(cfg.DbName, cfg.MetadataTable)), + } +} + +// DeleteTailSqls removes the entire tag=1 CdcTail (every cdc_tail chunk). Used by a +// REINDEX rebuild, which discards the accumulated delta log and rebuilds tag=0 from +// scratch; the fresh CDC task then starts from the reindex point (startFromNow). The +// tag=0 bases are cleared separately (DeleteAllBasesSqls / the create TVF). +func DeleteTailSqls(cfg TableConfig) []string { + return []string{ + fmt.Sprintf("DELETE FROM %s WHERE %s = %d", sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + catalog.Bm25Index_TblCol_Storage_Tag, int(vectorindex.Tag_CdcEvents)), + } +} + +// DeleteTailChunksByMaxId removes only the tag=1 CdcTail chunk rows with chunk_id +// <= k — the prefix a merge-compaction folded into the new tag=0 base. Chunks with +// chunk_id > k (appended by the sinker after the compaction's txn snapshot) are +// preserved, so `chunk_id` is never renumbered and concurrent appends survive. +func DeleteTailChunksByMaxId(cfg TableConfig, k int64) []string { + return []string{ + fmt.Sprintf("DELETE FROM %s WHERE %s = %s AND %s = %d AND %s <= %d", + sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + catalog.Bm25Index_TblCol_Storage_Index_Id, sqlquote.String(vectorindex.CdcTailId), + catalog.Bm25Index_TblCol_Storage_Tag, int(vectorindex.Tag_CdcEvents), + catalog.Bm25Index_TblCol_Storage_Chunk_Id, k), + } +} + +// CountTailChunks returns the number of tag=1 CdcTail chunk rows — a cheap, +// monotonic proxy for accumulated tail size used by the idxcron reindex gate. +// Chunk count, NOT doc count: an oversized frame is split across several chunk +// rows (see Bug 1 / splitFrameChunks), so per-chunk UnframeCdcChunk cannot run on +// continuation chunks; counting rows is robust and sufficient to gate "how much +// tail has piled up since the last reindex". +func CountTailChunks(sqlproc *sqlexec.SqlProcess, cfg TableConfig) (int64, error) { + sql := fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE %s = %d", + sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + catalog.Bm25Index_TblCol_Storage_Tag, int(vectorindex.Tag_CdcEvents)) + res, err := sqlexec.RunSql(sqlproc, sql) + if err != nil { + return 0, err + } + defer res.Close() + for _, bat := range res.Batches { + if bat == nil || bat.RowCount() == 0 { + continue + } + return vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[0], 0), nil + } + return 0, nil +} + +// SumBaseNrow returns SUM(metadata.nrow) — docs physically present in the tag=0 base subs +// (live + deleted-but-not-yet-reclaimed). A cheap metadata aggregate (no postings loaded); +// idxcron compares it to the source table's live row count to estimate the dead-doc fraction +// and decide MERGE (incremental) vs REBUILD (full reclaim). +func SumBaseNrow(sqlproc *sqlexec.SqlProcess, cfg TableConfig) (int64, error) { + sql := fmt.Sprintf("SELECT COALESCE(SUM(%s), 0) FROM %s", + catalog.Bm25Index_TblCol_Metadata_Nrow, sqlquote.QualifiedIdent(cfg.DbName, cfg.MetadataTable)) + res, err := sqlexec.RunSql(sqlproc, sql) + if err != nil { + return 0, err + } + defer res.Close() + for _, bat := range res.Batches { + if bat == nil || bat.RowCount() == 0 { + continue + } + return vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[0], 0), nil + } + return 0, nil +} + +// LoadAllBases loads every tag=0 base sub-index listed in the metadata table. The +// metadata table is per-fulltext-index, so every row names one of this index's bases +// (mirrors HNSW's LoadMetadata). Returns nil when no base was built (empty-table create +// → CDC-only index). Bases are pk-disjoint, so the caller assigns them one shared +// baseRecency. On any error the partially-loaded bases are freed. +func LoadAllBases(sqlproc *sqlexec.SqlProcess, cfg TableConfig) ([]*WandModel, error) { + idSQL := fmt.Sprintf("SELECT %s FROM %s", + catalog.Bm25Index_TblCol_Metadata_Index_Id, + sqlquote.QualifiedIdent(cfg.DbName, cfg.MetadataTable)) + res, err := sqlexec.RunSql(sqlproc, idSQL) + if err != nil { + return nil, err + } + var ids []string + for _, bat := range res.Batches { + if bat == nil { + continue + } + for i := 0; i < bat.RowCount(); i++ { + ids = append(ids, bat.Vecs[0].GetStringAt(i)) + } + } + res.Close() + + bases := make([]*WandModel, 0, len(ids)) + for _, id := range ids { + m, lerr := LoadFromStorage(sqlproc, cfg, id) + if lerr != nil { + freeSegs(bases) + return nil, lerr + } + bases = append(bases, m) + } + return bases, nil +} + +// LoadFromStorage reads an index's metadata + chunks back from the WAND store, +// verifies the checksum, and binds it into a LOADED model. Chunks are downloaded +// with STREAMING SQL and written by chunk_id offset into a temp file on the fast +// LOCAL (SSD) fileservice (so the mpool only ever holds a chunk or two, never the +// whole index — mirrors HNSW's loadChunk). The file is then MMAP'd read-only and the +// model's ranking directory + docID/tf blocks are kept as VIEWS into the mapping +// (page-cache-backed, reclaimable, shared by all concurrent queries) — the large +// postings are NEVER read onto the Go heap or into off-heap C buffers, and decode one +// block at a time on demand. Errors if the tag=0 metadata is absent. +func LoadFromStorage(sqlproc *sqlexec.SqlProcess, cfg TableConfig, id string) (*WandModel, error) { + checksum, filesize, chunkId, found, err := readMetadata(sqlproc, cfg, id) + if err != nil { + return nil, err + } + if !found { + return nil, moerr.NewInternalError(sqlproc.GetContext(), fmt.Sprintf("wand index %s metadata not found", id)) + } + if filesize <= 0 { + return nil, moerr.NewInternalError(sqlproc.GetContext(), fmt.Sprintf("wand index %s has empty filesize", id)) + } + + // Materialize the segment on the fast LOCAL (SSD) fileservice so mmap page faults + // come off the 2 GB/s mount, not /tmp. path=="" means an anonymous SSD file + // (unlinked; munmap frees the inode) — the same CreateAndRemoveFile the JOIN spill uses. + fp, path, err := createLocalTempFile(sqlproc) + if err != nil { + return nil, err + } + cleanup := func() { + fp.Close() + if path != "" { + os.Remove(path) + } + } + if err = fp.Truncate(filesize); err != nil { + cleanup() + return nil, err + } + if err = streamChunksToFile(sqlproc, cfg, id, filesize, fp); err != nil { + cleanup() + return nil, err + } + + // mmap the file read-only (shared across queries; ranking + blocks are views into + // it, page-cache-backed). The fd is not needed once mapped. + data, err := mmapReadOnly(fp) + fp.Close() + if err != nil { + if path != "" { + os.Remove(path) + } + return nil, err + } + // Checksum the mapped bytes (the anonymous SSD file has no path to CheckSum). + if vectorindex.CheckSumFromBuffer(data) != checksum { + _ = munmap(data) + if path != "" { + os.Remove(path) + } + return nil, moerr.NewInternalError(sqlproc.GetContext(), fmt.Sprintf("wand index %s checksum mismatch", id)) + } + // The model OWNS the mapping (+ path for the /tmp fallback): Free() munmaps and, + // if linked, deletes it. + m, err := decodeLoaded(id, data) + if err != nil { + _ = munmap(data) + if path != "" { + os.Remove(path) + } + return nil, err + } + m.mmapData = data + m.mmapPath = path + // The base's recency key comes from SQL (metadata.chunk_id): 0 = oldest + // full-build base; K = the folded tail chunk_id for a compacted base. Query + // ComputeLiveness dedups bases + tail uniformly by this. + m.Recency = chunkId + return m, nil +} + +// wandLocalDir is bm25's own subdir under the LOCAL fileservice (on the SSD data-dir) +// for mmap segment files — sibling to the JOIN spill's "__spill". +const wandLocalDir = "__bm25wand" + +// createLocalTempFile returns a temp file for the segment's mmap. It prefers the +// LOCAL fileservice's __bm25wand subdir (on the SSD data-dir) via the same +// CreateAndRemoveFile the JOIN spill uses — an ANONYMOUS file (unlinked; the fd + +// mapping keep the inode alive, Free just munmaps, no os.Remove). Falls back to +// os.CreateTemp (/tmp, linked → Free deletes by path) when no process/fileservice is +// attached (tests / one-shot tools). Returns (file, path, err); path=="" for the +// anonymous SSD file. +func createLocalTempFile(sqlproc *sqlexec.SqlProcess) (*os.File, string, error) { + if sqlproc != nil && sqlproc.Proc != nil { + ctx := sqlproc.GetContext() + if local, e := fileservice.Get[fileservice.MutableFileService]( + sqlproc.Proc.Base.FileService, defines.LocalFileServiceName); e == nil { + if e2 := local.EnsureDir(ctx, wandLocalDir); e2 == nil { + if sub, ok := fileservice.SubPath(local, wandLocalDir).(fileservice.MutableFileService); ok { + if f, e3 := sub.CreateAndRemoveFile(ctx, "wandidx_"+uuid.NewString()); e3 == nil { + return f, "", nil + } + } + } + } + } + f, err := os.CreateTemp("", "wandidx") + if err != nil { + return nil, "", err + } + return f, f.Name(), nil +} + +// loadTailSegments streams the tag=1 CdcTail (index_id = CdcTailId) into a temp +// file — each chunk row placed at (chunk_id - min)*MaxChunkSize, bounded memory, +// exactly like the tag=0 loader (streamChunksToFile) — and decodes it frame-by-frame +// (assembleFramesAt) into the ordered segment list + folded delete map. Empty tail +// (no CDC yet) → (nil, nil). +// +// Two things it deliberately avoids: (1) NO `ORDER BY chunk_id` — a SQL sort would +// add a Sort operator (full materialization / possible spill); placement-by-offset +// orders the chunks instead. (2) NO buffering the whole delta — chunks stream to +// disk one batch at a time and frames are decoded one at a time, so the transient +// footprint is a single frame, not the tail. +func loadTailSegments(sqlproc *sqlexec.SqlProcess, cfg TableConfig) ([]*WandModel, map[any]int64, int32, error) { + minC, maxC, empty, err := tailChunkBounds(sqlproc, cfg) + if err != nil { + return nil, nil, 0, err + } + if empty { + return nil, nil, 0, nil + } + // Fail fast if the tail can't fit in the memory budget, rather than letting it + // OOM-kill the CN as it decodes into the Go heap. + if err := checkTailLoadBudget(sqlproc, cfg); err != nil { + return nil, nil, 0, err + } + span := maxC - minC + 1 + filesize := span * int64(vectorindex.MaxChunkSize) + + fp, err := os.CreateTemp("", "wandtail") + if err != nil { + return nil, nil, 0, err + } + path := fp.Name() + defer func() { + fp.Close() + os.Remove(path) + }() + if err = fp.Truncate(filesize); err != nil { + return nil, nil, 0, err + } + + sql := fmt.Sprintf("SELECT %s, %s FROM %s WHERE %s = %s AND %s = %d", + catalog.Bm25Index_TblCol_Storage_Chunk_Id, catalog.Bm25Index_TblCol_Storage_Data, + sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + catalog.Bm25Index_TblCol_Storage_Index_Id, sqlquote.String(vectorindex.CdcTailId), + catalog.Bm25Index_TblCol_Storage_Tag, int(vectorindex.Tag_CdcEvents)) + _, n, err := streamChunkRowsToFile(sqlproc, sql, minC, filesize, fp) + if err != nil { + return nil, nil, 0, err + } + // tag=1 chunk_ids are a gapless run, so [min..max] must span exactly the row + // count; a mismatch is a missing/duplicate chunk (corruption). + if n != span { + return nil, nil, 0, moerr.NewInternalError(sqlproc.GetContext(), + fmt.Sprintf("wand tail: chunk_id range [%d..%d] spans %d but got %d rows (gap or duplicate)", minC, maxC, span, n)) + } + return assembleFramesAt(fp, minC, span) +} + +// tailChunkBounds returns MIN/MAX chunk_id of the tag=1 CdcTail via aggregates (no +// sort). empty=true when there are no tag=1 rows yet (MIN/MAX → NULL). +func tailChunkBounds(sqlproc *sqlexec.SqlProcess, cfg TableConfig) (minC, maxC int64, empty bool, err error) { + sql := fmt.Sprintf("SELECT MIN(%s), MAX(%s) FROM %s WHERE %s = %s AND %s = %d", + catalog.Bm25Index_TblCol_Storage_Chunk_Id, catalog.Bm25Index_TblCol_Storage_Chunk_Id, + sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + catalog.Bm25Index_TblCol_Storage_Index_Id, sqlquote.String(vectorindex.CdcTailId), + catalog.Bm25Index_TblCol_Storage_Tag, int(vectorindex.Tag_CdcEvents)) + res, err := sqlexec.RunSql(sqlproc, sql) + if err != nil { + return 0, 0, false, err + } + defer res.Close() + for _, bat := range res.Batches { + if bat == nil || bat.RowCount() == 0 { + continue + } + if bat.Vecs[0].IsNull(0) { // MIN over no rows → NULL → empty tail + return 0, 0, true, nil + } + minC = vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[0], 0) + maxC = vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[1], 0) + return minC, maxC, false, nil + } + return 0, 0, true, nil +} + +// checkTailLoadBudget fails fast when the CDC tail's stored bytes exceed the CN memory +// budget, returning a clear, actionable error instead of letting the tail load OOM-kill +// the CN (which would take down EVERY query on the node, not just this one). The tail +// decodes into the Go HEAP (frames read from the streamed temp file), so its stored size +// — SUM(LENGTH(data)), the ACTUAL bytes, not the padded span*MaxChunkSize file — is a +// good proxy for the load footprint. The mmap'd base is reclaimable OS page cache and is +// deliberately NOT counted (it cannot cause an OOM-kill). Budget = MemoryTotal*0.8 - live +// Go heap; MemoryTotal is cgroup-aware, and MemoryGolang already includes any tails +// currently resident, so this correctly gates an INCREMENTAL load. The 0.8 headroom +// absorbs the (small, post-mmap) C-allocator/query-mpool usage the formula omits. +func checkTailLoadBudget(sqlproc *sqlexec.SqlProcess, cfg TableConfig) error { + sql := fmt.Sprintf("SELECT COALESCE(SUM(LENGTH(%s)), 0) FROM %s WHERE %s = %s AND %s = %d", + catalog.Bm25Index_TblCol_Storage_Data, sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + catalog.Bm25Index_TblCol_Storage_Index_Id, sqlquote.String(vectorindex.CdcTailId), + catalog.Bm25Index_TblCol_Storage_Tag, int(vectorindex.Tag_CdcEvents)) + res, err := sqlexec.RunSql(sqlproc, sql) + if err != nil { + return err + } + defer res.Close() + var need int64 + for _, bat := range res.Batches { + if bat == nil || bat.RowCount() == 0 { + continue + } + need = vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[0], 0) + } + avail := int64(system.MemoryTotal())*8/10 - int64(system.MemoryGolang()) + if need > avail { + return moerr.NewInternalError(sqlproc.GetContext(), fmt.Sprintf( + "bm25 CDC tail for %s.%s needs ~%d MB to load but only ~%d MB is free "+ + "(MemoryTotal*0.8 - Go heap); compact it (ALTER ... REINDEX) or increase CN memory", + cfg.DbName, cfg.IndexTable, need>>20, avail>>20)) + } + return nil +} + +// streamChunksToFile streams a tag=0 index's chunk rows and writes each at +// chunk_id*MaxChunkSize into fp; the assembled bytes must fill filesize exactly. +func streamChunksToFile(sqlproc *sqlexec.SqlProcess, cfg TableConfig, id string, filesize int64, fp *os.File) error { + sql := fmt.Sprintf("SELECT %s, %s FROM %s WHERE %s = %s", + catalog.Bm25Index_TblCol_Storage_Chunk_Id, catalog.Bm25Index_TblCol_Storage_Data, + sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), + catalog.Bm25Index_TblCol_Storage_Index_Id, sqlquote.String(id)) + written, _, err := streamChunkRowsToFile(sqlproc, sql, 0, filesize, fp) + if err != nil { + return err + } + if written != filesize { + return moerr.NewInternalError(sqlproc.GetContext(), + fmt.Sprintf("wand index %s incomplete: wrote %d of %d bytes", id, written, filesize)) + } + return nil +} + +// streamChunkRowsToFile streams the (chunk_id, data) rows returned by sql and writes +// each at (chunk_id - baseChunk)*MaxChunkSize into fp, bounding the mpool to the +// stream buffer (never the whole index). bound is the file extent for the range +// check. Returns bytes written and the chunk-row count (callers use whichever fits +// their completeness check: tag=0 wants written == filesize; tag=1 — which has +// partial-tail holes — wants count == span). +func streamChunkRowsToFile(sqlproc *sqlexec.SqlProcess, sql string, baseChunk, bound int64, fp *os.File) (written, nchunks int64, err error) { + streamCh := make(chan executor.Result, 2) + errorCh := make(chan error, 2) + ctx, cancel := context.WithCancelCause(sqlproc.GetTopContext()) + defer cancel(nil) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer func() { + close(streamCh) + wg.Done() + }() + if _, e := sqlexec.RunStreamingSql(ctx, sqlproc, sql, streamCh, errorCh); e != nil { + errorCh <- e + } + }() + + var loopErr error + closed := false + for !closed { + select { + case res, ok := <-streamCh: + if !ok { + closed = true + break + } + for _, bat := range res.Batches { + if bat == nil || bat.RowCount() == 0 { + continue + } + cids := vector.MustFixedColNoTypeCheck[int64](bat.Vecs[0]) + for i, cid := range cids { + data := bat.Vecs[1].GetRawBytesAt(i) + off := (cid - baseChunk) * int64(vectorindex.MaxChunkSize) + if off < 0 || off+int64(len(data)) > bound { + loopErr = moerr.NewInternalError(sqlproc.GetContext(), + fmt.Sprintf("wand chunk_id %d out of range [base %d, bound %d]", cid, baseChunk, bound)) + break + } + if _, e := fp.WriteAt(data, off); e != nil { + loopErr = e + break + } + written += int64(len(data)) + nchunks++ + } + if loopErr != nil { + break + } + } + res.Close() + if loopErr != nil { + closed = true + } + case e := <-errorCh: + loopErr = e + closed = true + case <-ctx.Done(): + loopErr = context.Cause(ctx) + closed = true + } + } + + if loopErr != nil { + cancel(loopErr) + } + // drain any remaining results so the producer can exit cleanly + for res := range streamCh { + res.Close() + } + wg.Wait() + if loopErr == nil { + select { + case e := <-errorCh: + loopErr = e + default: + } + } + if loopErr != nil { + return 0, 0, loopErr + } + return written, nchunks, nil +} diff --git a/pkg/bm25/wand/tailbuild.go b/pkg/bm25/wand/tailbuild.go new file mode 100644 index 0000000000000..1a2fc345e55a0 --- /dev/null +++ b/pkg/bm25/wand/tailbuild.go @@ -0,0 +1,158 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wand + +import ( + "fmt" + "os" + "path/filepath" +) + +// defaultTailCapacity floors the streaming segment cap when the caller passes a +// non-positive capacity (should not happen — the sinker sources it from +// max_index_capacity, default 1M — but a 0 would disable sealing and reintroduce +// the OOM). Kept in step with iscp.defaultWandCapacity. +const defaultTailCapacity int64 = 1000000 + +// TailSegment is one sealed CDC segment spilled to a temp file: the path to its +// framed bytes (FrameSegment output) and the frame length (so the persist step can +// advance chunk_id without re-reading the file first). +type TailSegment struct { + Path string + FrameLen int +} + +// TailBuilder streams CDC insert/upsert rows into capacity-capped WAND segments, +// spilling each sealed segment's framed bytes to a temp file the moment it fills — +// so the sinker's peak memory is ONE open segment's postings, not the whole CDC +// stream. This is the fix for RunWand buffering every event: an 88M-row initial +// sync would OOM holding all (pk, text) events in RAM before building. +// +// It mirrors hnsw's HnswSync stream-and-spill: Update() rolls to a new model and +// Unload()s full ones to files mid-stream; Save() persists them at close. Here +// AddBatch() rolls+spills sealed segments; Finish() returns the spilled segment +// files + accumulated deletes for the caller to persist in one txn. Deletes stay a +// single record batch (small — pk only). NOT safe for concurrent use; Cleanup() +// must be called (defer) to remove the temp files. +type TailBuilder struct { + pkType int32 + capacity int64 + tokenize func(string) []string + dir string + seq int + cur *Builder // current open segment (nil until the first insert row) + segs []TailSegment + deletes []DeleteRecord +} + +// NewTailBuilder creates a streaming tail builder backed by a private temp dir. +func NewTailBuilder(pkType int32, capacity int64, tokenize func(string) []string) (*TailBuilder, error) { + if capacity < 1 { + capacity = defaultTailCapacity + } + dir, err := os.MkdirTemp("", "wandtail") + if err != nil { + return nil, err + } + return &TailBuilder{pkType: pkType, capacity: capacity, tokenize: tokenize, dir: dir}, nil +} + +// AddBatch streams one decoded CDC batch: insert/upsert rows are tokenized into the +// open segment (sealed + spilled once it reaches `capacity` docs), deletes are +// collected. Same tokenizer as the search side, so build/query tokens match. +func (t *TailBuilder) AddBatch(cdc *WandCdc) error { + for i := range cdc.Events { + e := &cdc.Events[i] + switch e.Op { + case cdcInsert, cdcUpsert: + if t.cur == nil { + t.cur = NewBuilder(fmt.Sprintf("cdctail-%d", t.seq), t.pkType) + } + for _, w := range t.tokenize(e.Text) { + if err := t.cur.Add(w, e.Pk); err != nil { + return err + } + } + // Seal at doc boundaries: len(pks) is the distinct-doc count so far. + if int64(len(t.cur.model.pks)) >= t.capacity { + if err := t.seal(); err != nil { + return err + } + } + case cdcDelete: + t.deletes = append(t.deletes, DeleteRecord{Pk: e.Pk}) + } + } + return nil +} + +// seal finalizes the open segment, frames it, and spills the framed bytes to a temp +// file (freeing the segment's C/Go buffers). A no-op if there's no open segment; an +// all-empty segment (every row had no searchable tokens) is dropped, not spilled. +func (t *TailBuilder) seal() error { + if t.cur == nil { + return nil + } + model := t.cur.Finish() + t.cur = nil + if model.N == 0 { + model.Free() + return nil + } + framed, err := FrameSegment(model) + model.Free() + if err != nil { + return err + } + path := filepath.Join(t.dir, fmt.Sprintf("seg-%d.frame", t.seq)) + t.seq++ + if err := os.WriteFile(path, framed, 0o600); err != nil { + return err + } + t.segs = append(t.segs, TailSegment{Path: path, FrameLen: len(framed)}) + return nil +} + +// Finish seals the final open segment, frames + spills the accumulated delete batch, +// and returns ALL spilled frame files in chunk_id order: the DELETE frame FIRST (so a +// same-batch UPSERT's new insert segment — at a higher chunk_id — supersedes the +// deleted base copy under ComputeLiveness), then the insert segments. Each is a +// temp file the caller persists via load_file. chunk_id is assigned by the caller at +// persist. Cleanup must be called afterwards. +func (t *TailBuilder) Finish() ([]TailSegment, error) { + if err := t.seal(); err != nil { + return nil, err + } + if len(t.deletes) == 0 { + return t.segs, nil + } + framed, err := FrameDeletes(t.pkType, t.deletes) + if err != nil { + return nil, err + } + path := filepath.Join(t.dir, "delete.frame") + if err := os.WriteFile(path, framed, 0o600); err != nil { + return nil, err + } + return append([]TailSegment{{Path: path, FrameLen: len(framed)}}, t.segs...), nil +} + +// Cleanup removes the temp dir and all spilled segment files. Idempotent. +func (t *TailBuilder) Cleanup() { + if t.dir != "" { + os.RemoveAll(t.dir) + t.dir = "" + } +} diff --git a/pkg/bm25/wand/tailbuild_test.go b/pkg/bm25/wand/tailbuild_test.go new file mode 100644 index 0000000000000..f7261ff92f898 --- /dev/null +++ b/pkg/bm25/wand/tailbuild_test.go @@ -0,0 +1,149 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wand + +import ( + "os" + "strings" + "testing" +) + +// TestTailBuilderStreamsCappedSegments drives docs through the streaming sinker +// builder across several batches and asserts it (1) seals capacity-capped segments +// spilled to files (peak memory = one open segment, the fix for buffering all +// events), (2) collects deletes, and (3) the spilled framed files reassemble — +// exactly as the load path does (split at MaxChunkSize + reassembleFrames + +// AssembleFrames) — into searchable segments covering every doc. +func TestTailBuilderStreamsCappedSegments(t *testing.T) { + tokenize := func(s string) []string { return strings.Fields(s) } + tb, err := NewTailBuilder(testPkType, 3, tokenize) // capacity 3 → multiple segments + if err != nil { + t.Fatal(err) + } + defer tb.Cleanup() + + mk := func(pairs ...any) *WandCdc { + c := NewWandCdc(testPkType) + for i := 0; i < len(pairs); i += 2 { + c.Insert(int64(pairs[i].(int)), pairs[i+1].(string)) + } + return c + } + // 7 insert docs across 3 batches → ceil(7/3) = 3 sealed segments ([1,2,3], + // [4,5,6], [7]); a batch also carries a delete of an unrelated pk. + if err := tb.AddBatch(mk(1, "x y", 2, "x", 3, "y")); err != nil { + t.Fatal(err) + } + if err := tb.AddBatch(mk(4, "x", 5, "z", 6, "x y z")); err != nil { + t.Fatal(err) + } + last := mk(7, "x") + last.Delete(int64(99)) + if err := tb.AddBatch(last); err != nil { + t.Fatal(err) + } + + segs, err := tb.Finish() + if err != nil { + t.Fatal(err) + } + // 3 capacity-capped insert segments + 1 delete frame (spilled FIRST) = 4 files. + if len(segs) != 4 { + t.Fatalf("want 4 spilled files (1 delete + 3 segments), got %d", len(segs)) + } + + // Reassemble the spilled files exactly as loadTailFrames does. + var chunks []TailChunk + cid := int64(0) + for _, seg := range segs { + framed, e := os.ReadFile(seg.Path) + if e != nil { + t.Fatal(e) + } + if len(framed) != seg.FrameLen { + t.Fatalf("FrameLen %d != spilled file size %d", seg.FrameLen, len(framed)) + } + cs := splitFrameChunks(cid, framed) + chunks = append(chunks, cs...) + cid += int64(len(cs)) + } + frames, err := reassembleFrames(chunks) + if err != nil { + t.Fatal(err) + } + models, delMap, err := AssembleFrames(frames) + if err != nil { + t.Fatal(err) + } + defer freeSegs(models) + if len(models) != 3 { + t.Fatalf("want 3 reassembled models, got %d", len(models)) + } + for i, m := range models { + if m.N > 3 { + t.Fatalf("segment %d exceeds capacity: N=%d", i, m.N) + } + } + if _, ok := delMap[normalizeKey(int64(99))]; !ok { + t.Fatalf("delete pk 99 not folded from the spilled delete frame: %v", delMap) + } + + live := ComputeLiveness(models, delMap) + got := pkCounts(SearchSegmentsLive(models, []string{"x"}, 100, nil, live)) + want := map[int64]int{1: 1, 2: 1, 4: 1, 6: 1, 7: 1} // docs containing "x" + if len(got) != len(want) { + t.Fatalf("search x: want %v, got %v", want, got) + } + for pk := range want { + if got[pk] != 1 { + t.Fatalf("search x: pk %d missing (got %v)", pk, got) + } + } +} + +// TestTailBuilderEmptyAndCleanup covers the corner cases: an all-delete stream +// yields only the spilled delete frame (no insert segments), a segment whose rows +// have no searchable tokens is dropped (not spilled), and Cleanup removes the dir. +func TestTailBuilderEmptyAndCleanup(t *testing.T) { + tokenize := func(s string) []string { return strings.Fields(s) } + tb, err := NewTailBuilder(testPkType, 100, tokenize) + if err != nil { + t.Fatal(err) + } + + c := NewWandCdc(testPkType) + c.Insert(int64(1), "") // no tokens → contributes no doc + c.Delete(int64(7)) + if err := tb.AddBatch(c); err != nil { + t.Fatal(err) + } + segs, err := tb.Finish() + if err != nil { + t.Fatal(err) + } + // no insert segments; just the spilled delete frame. + if len(segs) != 1 { + t.Fatalf("all-delete/empty-token stream should spill 1 file (the delete frame), got %d", len(segs)) + } + + dir := tb.dir + if _, e := os.Stat(dir); e != nil { + t.Fatalf("temp dir should exist before Cleanup: %v", e) + } + tb.Cleanup() + if _, e := os.Stat(dir); !os.IsNotExist(e) { + t.Fatalf("Cleanup should remove temp dir, stat err=%v", e) + } +} diff --git a/pkg/bm25/wand/uuidpk_test.go b/pkg/bm25/wand/uuidpk_test.go new file mode 100644 index 0000000000000..8d35efbc37711 --- /dev/null +++ b/pkg/bm25/wand/uuidpk_test.go @@ -0,0 +1,132 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wand + +import ( + "bytes" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +const uuidPkType = int32(types.T_uuid) + +// TestWandUuidPkEncodeReps: a uuid pk arrives as a CDC STRING (extractRowFromVector +// -> Uuid.String()) OR as a sync-build types.Uuid (GetAny). Both MUST encode to the +// SAME stored bytes and decode back to types.Uuid (what AppendAny + the doc_id -> +// src.id INNER JOIN need). This is exactly the representation the earlier generic +// codec crashed on. +func TestWandUuidPkEncodeReps(t *testing.T) { + utext := "0195e0c8-1234-7890-abcd-000000000001" + u, err := types.ParseUuid(utext) + if err != nil { + t.Fatal(err) + } + + bStr, err := encodePk(uuidPkType, utext) // CDC representation (string) + if err != nil { + t.Fatal(err) + } + bVal, err := encodePk(uuidPkType, u) // sync representation (types.Uuid) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(bStr, bVal) { + t.Fatalf("uuid encode differs by rep: string=%q value=%q", bStr, bVal) + } + + got, err := decodePk(uuidPkType, bStr) + if err != nil { + t.Fatal(err) + } + gu, ok := got.(types.Uuid) + if !ok { + t.Fatalf("decodePk returned %T, want types.Uuid", got) + } + if gu != u { + t.Fatalf("decodePk uuid mismatch: got %v want %v", gu, u) + } + // normalizeKey must agree so a string-delivered delete matches a Uuid segment pk. + if normalizeKey(got) != normalizeKey(u) { + t.Fatal("normalizeKey(decoded) != normalizeKey(types.Uuid)") + } +} + +// TestWandUuidPkRoundTrip builds a segment whose pks are delivered as STRINGS (the +// CDC form), serializes + deserializes it, and confirms search returns the correct +// docs with types.Uuid doc-ids — the full path that must not panic. Also checks a +// string-delivered delete folds to the same key as the reloaded Uuid pk (liveness). +func TestWandUuidPkRoundTrip(t *testing.T) { + u1s := "00000000-0000-0000-0000-000000000001" + u2s := "00000000-0000-0000-0000-000000000002" + u1, _ := types.ParseUuid(u1s) + u2, _ := types.ParseUuid(u2s) + + b := NewBuilder("uuidtest", uuidPkType) + adds := []struct { + w string + pk any + }{{"x", u1s}, {"x", u2s}, {"y", u1s}} // string pks, the CDC representation + for _, a := range adds { + if err := b.Add(a.w, a.pk); err != nil { + t.Fatal(err) + } + } + m := b.Finish() + + blob, err := m.Serialize() + if err != nil { + t.Fatal(err) + } + m2, err := Deserialize("uuidtest", bytes.NewReader(blob)) + if err != nil { + t.Fatal(err) + } + defer m2.Free() + if m2.N != 2 { + t.Fatalf("N=%d want 2", m2.N) + } + + res := SearchSegments([]*WandModel{m2}, []string{"x"}, 10, nil) + if len(res) != 2 { + t.Fatalf("search x: %d results want 2", len(res)) + } + got := map[types.Uuid]bool{} + for _, r := range res { + u, ok := r.DocID.(types.Uuid) + if !ok { + t.Fatalf("DocID is %T, want types.Uuid (needed for AppendAny + src join)", r.DocID) + } + got[u] = true + } + if !got[u1] || !got[u2] { + t.Fatalf("search returned wrong uuids: %v", got) + } + + // A DELETE delivered as the CDC string must fold to the SAME key as the reloaded + // types.Uuid segment pk, or liveness would never match it. + dl, err := EncodeDeleteLog(uuidPkType, []DeleteRecord{{Pk: u1s}}) + if err != nil { + t.Fatal(err) + } + drecs, err := DecodeDeleteLog(dl) + if err != nil { + t.Fatal(err) + } + dm := FoldDeleteFrame(nil, drecs, 3) + if dm[normalizeKey(u1)] != 3 { + t.Fatalf("string-delivered uuid delete did not fold to the types.Uuid key: %v", dm) + } +} diff --git a/pkg/bm25/wand/wand.go b/pkg/bm25/wand/wand.go new file mode 100644 index 0000000000000..8a73768b63abb --- /dev/null +++ b/pkg/bm25/wand/wand.go @@ -0,0 +1,649 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package wand implements an in-memory, doc-ordered, skippable posting +// structure answering disjunctive (OR) top-K fulltext queries with the WAND / +// Block-Max WAND family, instead of materializing the whole match set and +// feeding it through a SQL ORDER BY ... LIMIT sort. It backs the `retrieval` +// fulltext parser / IN RETRIEVAL MODE. +// +// Internals are pure integers for speed/compactness: +// - doc id -> dense int64 "ord" (map[any]int64 dictionary, like fulltext.go's +// normalizeDocID; any PK type supported, []byte normalized to string keys). +// - word -> int32 word-id: jieba dictionary words use their global line-id +// (tokenizer.WordID); out-of-dict tokens get per-index overflow ids +// (>= tokenizer.DictWordIDLimit). +// +// Scoring is MatrixOne's default BM25: weight * idf^2 * bm25Factor(tf, dl, avgdl) +// with idf = log10(N/df), matching fulltext.go ALGO_BM25. +// +// The serialized form is a tar archive (see serialize.go) with members: +// docmap (pkType + ord->pk), termdict (overflow word->id), wand (postings). +package wand + +import ( + "encoding/binary" + "math" + "os" + "sort" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/monlp/tokenizer" +) + +const ( + // MaxCappedTf mirrors fulltext.cappedTfExpr (cap tf at 255 so it fits a + // uint8). The builder accumulates occurrence counts and caps here. + MaxCappedTf = 255 + + // BlockSize is the number of postings per Block-Max skip block. + BlockSize = 128 + + // BM25 parameters — match fulltext.BM25_K1 / BM25_B (the default score). + bm25K1 = 1.5 + bm25B = 0.75 +) + +// bm25Factor is the BM25 tf component: tf·(k1+1)/(tf + k1·(1-b+b·dl/avgdl)). +// The full per-term contribution is weight·idf²·bm25Factor (MatrixOne's BM25). +func bm25Factor(tf float64, dl int32, avgDocLen float64) float64 { + norm := 1.0 + if avgDocLen > 0 { + norm = 1.0 - bm25B + bm25B*float64(dl)/avgDocLen + } + return tf * (bm25K1 + 1) / (tf + bm25K1*norm) +} + +// termPostings is the posting list for one word-id, ordered by doc ord. It has +// two representations (mirrors fulltext2's termPostings): +// +// - BUILD-side: docIDs/tfs are resident Go slices; df == len(docIDs). Produced by +// the Builder, Merge, FilterLive, Split. blockLastDoc/blockMaxTf/blockMinDl are +// derived by finalizeScoring. +// - LOADED-side (Deserialize / mmap): docIDs/tfs are nil — the postings live +// block-compressed in blockData (a view into the mmap/blob, one BlockSize-doc +// block at a time: docID gaps as delta+varint from the previous block's last +// ord, then raw tf bytes). ndoc is the authoritative df. The block-max directory +// (blockLastDoc/blockMaxTf/blockMinDl + blockOff) is decoded RESIDENT per term on +// demand (decodeTermEntry) — never re-derived from the compressed blocks. The +// WAND cursor decodes only the blocks its block-max walk lands on (fillBlock). +type termPostings struct { + docIDs []int64 // BUILD-side doc ords, ascending, len == df; nil when LOADED + tfs []uint8 // BUILD-side parallel capped tf; nil when LOADED + + // ndoc is the document frequency on a LOADED posting list (docIDs not expanded); + // on a build-side list it equals len(docIDs). + ndoc int + + // LOADED-side compressed docID/tf blocks (a view into the segment's mmap/blob) + // plus the per-block byte offsets within blockData (len nblk+1, cumulative). nil + // on a build-side list. + blockData []byte + blockOff []int32 + + // Term-level score upper-bound inputs, idf-AND-avgdl-FREE so the bound stays + // valid under a global idf/avgdl (segments, incremental). The term UB is + // weight·idf²·bm25Factor(maxTf, minDl, avgdl), computed at query time. + maxTf uint8 // max tf over all postings + minDl int32 // min doc length over all postings + + // Block-Max skip-block metadata (one entry per ceil(df/BlockSize)). Also + // idf/avgdl-free. block UB = weight·idf²·bm25Factor(blockMaxTf, blockMinDl, + // avgdl). Derived by finalizeScoring (build) or decodeTermEntry (loaded). + blockLastDoc []int64 // max (last) ord in each block (ascending → last) + blockMaxTf []uint8 // max tf in each block + blockMinDl []int32 // min doc length in each block +} + +// df is the document frequency (build-side authoritative len; loaded-side ndoc). +func (p *termPostings) df() int { + if p.docIDs != nil { + return len(p.docIDs) + } + return p.ndoc +} + +// nblk is the number of Block-Max skip blocks (== ceil(df/BlockSize)). +func (p *termPostings) nblk() int { return len(p.blockLastDoc) } + +// blockLen is the number of postings in block b (BlockSize, or the last remainder). +func (p *termPostings) blockLen(b int) int { + n := p.df() - b*BlockSize + if n > BlockSize { + n = BlockSize + } + return n +} + +// fillBlock decodes block b's docIDs and tfs into outDocs/outTfs (each cap >= +// BlockSize) and returns the block length. Build-side copies the flat slices; +// loaded-side varint-decodes the block from blockData (the mmap view): docID gaps +// accumulate from the previous block's last ord (resident blockLastDoc[b-1]), then +// the raw tf bytes follow. The WAND cursor calls this once per block it lands on; +// materializeDocIDs/Tfs call it per block to rebuild the flat arrays. +func (p *termPostings) fillBlock(b int, outDocs []int64, outTfs []uint8) int { + blen := p.blockLen(b) + if p.docIDs != nil { // build-side: copy from the flat arrays + lo := b * BlockSize + copy(outDocs[:blen], p.docIDs[lo:lo+blen]) + copy(outTfs[:blen], p.tfs[lo:lo+blen]) + return blen + } + data := p.blockData[p.blockOff[b]:p.blockOff[b+1]] + var prev int64 + if b > 0 { + prev = p.blockLastDoc[b-1] + } + off := 0 + for i := 0; i < blen; i++ { + g, n := binary.Uvarint(data[off:]) + off += n + prev += int64(g) + outDocs[i] = prev + } + copy(outTfs[:blen], data[off:off+blen]) + return blen +} + +// materializeDocIDs returns this term's full ascending doc ords (df entries): the +// build-side flat slice as-is, or a transient decode of every loaded block. Cold +// paths (Merge / FilterLive / Split / re-Serialize) that scan all postings call +// this ONCE; WAND ranking never does (it decodes only the blocks its walk lands on). +func (p *termPostings) materializeDocIDs() []int64 { + if p.docIDs != nil { + return p.docIDs + } + out := make([]int64, p.ndoc) + var scratch [BlockSize]uint8 + for b := 0; b < p.nblk(); b++ { + lo := b * BlockSize + p.fillBlock(b, out[lo:], scratch[:]) + } + return out +} + +// materializeTfs returns this term's full capped tf bytes (df entries): the +// build-side flat slice as-is, or a transient decode of every loaded block. +func (p *termPostings) materializeTfs() []uint8 { + if p.tfs != nil { + return p.tfs + } + out := make([]uint8, p.ndoc) + var scratch [BlockSize]int64 + for b := 0; b < p.nblk(); b++ { + lo := b * BlockSize + p.fillBlock(b, scratch[:], out[lo:]) + } + return out +} + +// WandModel is the loadable in-memory index (one segment). +type WandModel struct { + Id string + N int64 // number of documents (= len(pks)) + PkType int32 // types.T of the source primary key, for output decode + membership + AvgDocLen float64 // average doc length (derived from DocLen), for BM25 + + // Recency is the segment's ordering key for liveness. For a tag=1 CdcTail delta it is + // the frame's append position (storage chunk_id, NOT an ISCP LSN; see fulltext_wand.md + // "single CdcTail log, chunk_id-ordered"); for a tag=0 base sub it is metadata.recency + // (0 = full-build/oldest, K = folded). When the same pk lands in multiple segments + // (UPDATE / reinsert / a stale base copy), only the highest-Recency copy is live (see + // ComputeLiveness). Named distinctly from the storage table's chunk_id (a physical + // chunk position within a blob), which is an unrelated concept. + Recency int64 + + pks []any // ord -> original pk value (for output via AppendAny) + docLen []int32 // ord -> document length (token count), for BM25 + terms map[int32]*termPostings // BUILD-side word-id -> postings; nil on a LOADED model + overflow map[string]int32 // out-of-dict term -> overflow word-id (query resolution) + + // LOADED-side term dict (set by decodeLoaded / bindWand; nil on a build-side + // model). termOffsets maps word-id -> the BYTE OFFSET of that term's + // self-contained directory entry in `ranking`; a loaded model does NOT expand any + // term at load — lookupTerm decodes just the touched term's directory entry from + // `ranking` on demand and points its blockData at `blocks` — so the resident + // directory heap is O(the current query), not O(vocabulary). `ranking`/`blocks` + // are views into the mmap/blob (kept alive by mmapData or GC). The build-side + // `terms` map is left nil. + termOffsets map[int32]int64 + ranking, blocks []byte + + // mmapData is the shared read-only mmap of a base segment's on-disk file: the + // ranking directory and the compressed docID/tf blocks are views into it + // (page-cache-backed, reclaimable, shared by all concurrent queries — no copy, no + // off-heap). mmapPath is that file (empty for the anonymous SSD file, whose inode + // is freed by munmap). Both are released by Free() under the cache's eviction + // write-lock. nil on a build-side or in-memory (tail) model, whose bytes are + // GC-managed Go slices. + mmapData []byte + mmapPath string +} + +// Free releases a loaded model's mmap (and its backing file, if linked). Safe on a +// build-side / in-memory tail model (nil mmapData → no-op) and idempotent. After +// Free the model must not be searched; the VectorIndexCache holds the write lock +// when calling Destroy. The loaded posting blocks (blockData) are views into +// mmapData, so munmap reclaims them — there is no off-heap buffer to deallocate. +func (m *WandModel) Free() { + if m.mmapData != nil { + _ = munmap(m.mmapData) + m.mmapData = nil + } + if m.mmapPath != "" { + _ = os.Remove(m.mmapPath) + m.mmapPath = "" + } + m.ranking, m.blocks = nil, nil + m.termOffsets = nil + m.terms = nil +} + +// lookupTerm resolves a word-id to its posting list, transparently across the two +// representations: the build-side terms map, or a lazy per-term decode of the loaded +// directory entry (decodeTermEntry). Returns (nil,false) if the word-id is absent. +func (m *WandModel) lookupTerm(id int32) (*termPostings, bool) { + if m.termOffsets != nil { // loaded + off, ok := m.termOffsets[id] + if !ok { + return nil, false + } + return m.decodeTermEntry(off) + } + tp, ok := m.terms[id] + return tp, ok +} + +// forEachTerm calls fn for every (word-id, posting-list), whether build-side (terms +// map) or loaded-side (lazy directory decode). Used by the cold full-scan paths +// (Merge / FilterLive / Split / re-Serialize). On a loaded model each tp is a +// transient decode whose blockData views the mmap. +func (m *WandModel) forEachTerm(fn func(int32, *termPostings)) { + if m.termOffsets != nil { // loaded + for id, off := range m.termOffsets { + if tp, ok := m.decodeTermEntry(off); ok { + fn(id, tp) + } + } + return + } + for id, tp := range m.terms { + fn(id, tp) + } +} + +// NewWandModel returns an empty model. +func NewWandModel(id string, pkType int32) *WandModel { + return &WandModel{ + Id: id, + PkType: pkType, + terms: make(map[int32]*termPostings), + overflow: make(map[string]int32), + } +} + +// computeAvgDocLen sets AvgDocLen from docLen. Called by both finalizeScoring +// (build) and decodeLoaded (load) — each model carries its own AvgDocLen, which +// corpusStats aggregates across segments. +func (m *WandModel) computeAvgDocLen() { + var sum int64 + for _, dl := range m.docLen { + sum += int64(dl) + } + if len(m.docLen) > 0 { + m.AvgDocLen = float64(sum) / float64(len(m.docLen)) + } +} + +// finalizeScoring derives AvgDocLen and every BUILD-side term's max BM25 factor + +// per-term Block-Max skip-block stats. Called by the builder's Finish and by +// Merge/FilterLive/Split (all build-side). A LOADED model reads the block-max +// directory back from disk (decodeTermEntry) instead — it never calls this. +func (m *WandModel) finalizeScoring() { + m.computeAvgDocLen() + for _, tp := range m.terms { + deriveTermStats(tp, m.docLen) + } +} + +// deriveTermStats fills one BUILD-side posting list's term-level maxTf/minDl and its +// per-block Block-Max skip metadata (blockLastDoc/blockMaxTf/blockMinDl), one entry +// per ceil(df/BlockSize), from its resident docIDs/tfs + docLen. Raw / idf-avgdl-free. +func deriveTermStats(tp *termPostings, docLen []int32) { + df := len(tp.docIDs) + tp.ndoc = df + nblk := (df + BlockSize - 1) / BlockSize + tp.blockLastDoc = make([]int64, nblk) + tp.blockMaxTf = make([]uint8, nblk) + tp.blockMinDl = make([]int32, nblk) + var termMaxTf uint8 + termMinDl := int32(math.MaxInt32) + for b := 0; b < nblk; b++ { + lo := b * BlockSize + hi := lo + BlockSize + if hi > df { + hi = df + } + var maxTf uint8 + minDl := int32(math.MaxInt32) + for i := lo; i < hi; i++ { + if tp.tfs[i] > maxTf { + maxTf = tp.tfs[i] + } + if dl := docLen[tp.docIDs[i]]; dl < minDl { + minDl = dl + } + } + tp.blockLastDoc[b] = tp.docIDs[hi-1] // ascending → last is max + tp.blockMaxTf[b] = maxTf + tp.blockMinDl[b] = minDl + if maxTf > termMaxTf { + termMaxTf = maxTf + } + if minDl < termMinDl { + termMinDl = minDl + } + } + tp.maxTf = termMaxTf + tp.minDl = termMinDl +} + +// Merge combines several index segments (disjoint document sets) into one +// segment — the compaction primitive for incremental indexing. Segment i's docs +// are appended after the previous segments (ords re-based), so each term's +// concatenated postings stay globally sorted. Overflow word-ids are reconciled +// by word into a single dictionary (dictionary word-ids < DictWordIDLimit are +// global and unchanged). The result is finalized (block/term stats + avgdl) and +// self-contained. Callers must pass segments with disjoint pk sets. +func Merge(id string, segs ...*WandModel) *WandModel { + m := NewWandModel(id, 0) + if len(segs) > 0 { + m.PkType = segs[0].PkType + } + var nextOverflow int32 // next free per-corpus overflow offset + var base int64 // ord offset for the current segment + + for _, s := range segs { + // Reconcile this segment's overflow ids into the merged dictionary. + var remap map[int32]int32 + if len(s.overflow) > 0 { + remap = make(map[int32]int32, len(s.overflow)) + for word, sid := range s.overflow { + mid, ok := m.overflow[word] + if !ok { + mid = tokenizer.DictWordIDLimit + nextOverflow + nextOverflow++ + m.overflow[word] = mid + } + remap[sid] = mid + } + } + + m.pks = append(m.pks, s.pks...) + m.docLen = append(m.docLen, s.docLen...) + + // forEachTerm handles both a build-side input (terms map) and a LOADED input + // (lazy directory decode) — CompactSegments feeds FilterLive'd loaded tail + // segments here. materializeDocIDs/Tfs expand a loaded term's compressed blocks + // transiently; a build-side term returns its resident slices. + s.forEachTerm(func(wid int32, tp *termPostings) { + mwid := wid + if wid >= tokenizer.DictWordIDLimit { + mwid = remap[wid] + } + mtp := m.terms[mwid] + if mtp == nil { + mtp = &termPostings{} + m.terms[mwid] = mtp + } + docs := tp.materializeDocIDs() + tfs := tp.materializeTfs() + for i, ord := range docs { + mtp.docIDs = append(mtp.docIDs, ord+base) + mtp.tfs = append(mtp.tfs, tfs[i]) + } + }) + base += s.N + } + + m.N = base + m.finalizeScoring() + return m +} + +// NumTerms returns the number of distinct word-ids in the index (build- or +// loaded-side). +func (m *WandModel) NumTerms() int { + if m.termOffsets != nil { + return len(m.termOffsets) + } + return len(m.terms) +} + +// PkAt returns the original pk value for a doc ord (for output). +func (m *WandModel) PkAt(ord int64) any { + if ord < 0 || ord >= int64(len(m.pks)) { + return nil + } + return m.pks[ord] +} + +// resolveWordID maps a query/build word to its word-id. ok is false when the +// word is neither a dictionary word nor (for queries) a known overflow term. +func (m *WandModel) resolveWordID(word string) (int32, bool, error) { + id, ok, err := tokenizer.WordID(word) + if err != nil { + return 0, false, err + } + if ok { + return id, true, nil + } + oid, ok := m.overflow[word] + return oid, ok, nil +} + +// --------------------------------------------------------------------------- +// Build +// --------------------------------------------------------------------------- + +// Builder accumulates postings — one Add per (word, doc) occurrence, in any +// order — and produces a WandModel. tf per (word, doc) is the occurrence count +// (capped). doc ords and overflow word-ids are assigned on first sight. +type Builder struct { + model *WandModel + ordMap map[any]int64 // normalized pk -> ord + overflowNext int32 // next overflow word-id offset + posOf map[int32]map[int64]int // word-id -> ord -> index in termPostings +} + +// NewBuilder creates a Builder for an index id and source pk type (types.T). +func NewBuilder(id string, pkType int32) *Builder { + return &Builder{ + model: NewWandModel(id, pkType), + ordMap: make(map[any]int64), + posOf: make(map[int32]map[int64]int), + } +} + +// normalizeKey converts a pk to a comparable map key ([]byte -> string), like +// fulltext.go's normalizeDocID. +func normalizeKey(pk any) any { + if b, ok := pk.([]byte); ok { + return string(b) + } + return pk +} + +// copyPk returns a value safe to retain ([]byte is copied; the source buffer may +// be reused by the caller). +func copyPk(pk any) any { + if b, ok := pk.([]byte); ok { + c := make([]byte, len(b)) + copy(c, b) + return c + } + return pk +} + +// docOrd returns the dense ord for a pk, assigning one on first sight. +func (b *Builder) docOrd(pk any) int64 { + key := normalizeKey(pk) + if o, ok := b.ordMap[key]; ok { + return o + } + o := int64(len(b.model.pks)) + b.ordMap[key] = o + b.model.pks = append(b.model.pks, copyPk(pk)) + b.model.docLen = append(b.model.docLen, 0) + return o +} + +// wordID returns the word-id for a build-time word, assigning an overflow id for +// out-of-dictionary tokens. +func (b *Builder) wordID(word string) (int32, error) { + id, ok, err := tokenizer.WordID(word) + if err != nil { + return 0, err + } + if ok { + return id, nil + } + if oid, ok := b.model.overflow[word]; ok { + return oid, nil + } + oid := tokenizer.DictWordIDLimit + b.overflowNext + b.overflowNext++ + b.model.overflow[word] = oid + return oid, nil +} + +// Add records one (word, doc) occurrence (any order). tf is accumulated per +// (word-id, ord), capped at MaxCappedTf. +func (b *Builder) Add(word string, pk any) error { + if word == "" { + return moerr.NewInternalErrorNoCtx("wand builder: empty word") + } + id, err := b.wordID(word) + if err != nil { + return err + } + ord := b.docOrd(pk) + b.model.docLen[ord]++ // one token occurrence contributes to this doc's length + + tp := b.model.terms[id] + if tp == nil { + tp = &termPostings{} + b.model.terms[id] = tp + b.posOf[id] = make(map[int64]int) + } + if pos, dup := b.posOf[id][ord]; dup { + if tp.tfs[pos] < MaxCappedTf { + tp.tfs[pos]++ + } + } else { + b.posOf[id][ord] = len(tp.docIDs) + tp.docIDs = append(tp.docIDs, ord) + tp.tfs = append(tp.tfs, 1) + } + return nil +} + +// Finish produces a single-segment index (no capacity limit). +func (b *Builder) Finish() *WandModel { + return b.FinishSegments(0)[0] +} + +// FinishSegments finalizes the build into one or more index segments, each +// holding at most `capacity` documents (by doc-ord range). capacity <= 0 means +// no limit → a single segment. Each segment is self-contained (local 0-based +// ords, its own pks/docLen/postings) and scored corpus-globally at query time by +// SearchSegments. Mirrors HNSW's multi-mini-index rollover. +func (b *Builder) FinishSegments(capacity int64) []*WandModel { + full := b.model + for _, tp := range full.terms { + sortPostings(tp) // global ascending order, so range-splits are contiguous + } + n := int64(len(full.pks)) + + if capacity <= 0 || n <= capacity { + full.N = n + full.finalizeScoring() + return []*WandModel{full} + } + + nseg := int((n + capacity - 1) / capacity) + segs := make([]*WandModel, nseg) + for s := 0; s < nseg; s++ { + lo := int64(s) * capacity + hi := lo + capacity + if hi > n { + hi = n + } + seg := NewWandModel(full.Id, full.PkType) + seg.pks = full.pks[lo:hi] // build-side view; serialized independently + seg.docLen = full.docLen[lo:hi] // local ord i == global ord lo+i + seg.overflow = full.overflow // identical dict across segments + seg.N = hi - lo + segs[s] = seg + } + + // Partition each term's (globally-sorted) postings into segment ranges, + // remapping global ords to per-segment local ords. + for wid, tp := range full.terms { + i, df := 0, len(tp.docIDs) + for s := 0; s < nseg && i < df; s++ { + lo := int64(s) * capacity + hi := lo + capacity + start := i + for i < df && tp.docIDs[i] < hi { + i++ + } + if i == start { + continue + } + stp := &termPostings{ + docIDs: make([]int64, i-start), + tfs: append([]uint8(nil), tp.tfs[start:i]...), + } + for j := start; j < i; j++ { + stp.docIDs[j-start] = tp.docIDs[j] - lo // local ord + } + segs[s].terms[wid] = stp + } + } + + for _, seg := range segs { + seg.finalizeScoring() + } + return segs +} + +func sortPostings(tp *termPostings) { + if sort.SliceIsSorted(tp.docIDs, func(i, j int) bool { return tp.docIDs[i] < tp.docIDs[j] }) { + return + } + idx := make([]int, len(tp.docIDs)) + for i := range idx { + idx[i] = i + } + sort.Slice(idx, func(i, j int) bool { return tp.docIDs[idx[i]] < tp.docIDs[idx[j]] }) + docs := make([]int64, len(idx)) + tfs := make([]uint8, len(idx)) + for i, j := range idx { + docs[i] = tp.docIDs[j] + tfs[i] = tp.tfs[j] + } + tp.docIDs = docs + tp.tfs = tfs +} diff --git a/pkg/bm25/wand/wand_test.go b/pkg/bm25/wand/wand_test.go new file mode 100644 index 0000000000000..8b1a8620f34f3 --- /dev/null +++ b/pkg/bm25/wand/wand_test.go @@ -0,0 +1,747 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wand + +import ( + "bytes" + "fmt" + "math" + "math/rand" + "sort" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +const testPkType = int32(types.T_int64) + +// baseRecency — the recency of a full-build base in tests: 0 (oldest; the tail +// starts at 1 under option (ii)). Production reads each base's recency from +// metadata.chunk_id; these liveness tests just need base < tail. +const baseRecency int64 = 0 + +// corpus mirrors what was fed to the Builder, for the brute-force reference. +type corpus struct { + docTf map[string]map[int64]int // term -> pk -> total tf (uncapped) + docLen map[int64]int // pk -> document length (token count) + pks map[int64]bool +} + +func cappedTf(raw int) float64 { + if raw > MaxCappedTf { + raw = MaxCappedTf + } + return float64(raw) +} + +func (c *corpus) avgDocLen() float64 { + if len(c.pks) == 0 { + return 0 + } + var sum int + for _, dl := range c.docLen { + sum += dl + } + return float64(sum) / float64(len(c.pks)) +} + +// bruteForce computes the exact BM25 top-K: score(d) = Σ w_t·idf(t)²·factor, +// idf = log10(N/df), factor = bm25Factor(cappedTf, dl, avgdl) — MatrixOne's +// default BM25, matching the WAND walk. +func bruteForce(c *corpus, terms []string, limit int, allowPk map[int64]bool) []SearchResult { + n := len(c.pks) + avgdl := c.avgDocLen() + weights := map[string]float64{} + for _, t := range terms { + weights[t]++ + } + scores := map[int64]float64{} + for t, w := range weights { + dm, ok := c.docTf[t] + if !ok { + continue + } + df := len(dm) + idf := math.Log10(float64(n) / float64(df)) + idfSq := idf * idf + for d, raw := range dm { + if allowPk != nil && !allowPk[d] { + continue + } + scores[d] += w * idfSq * bm25Factor(cappedTf(raw), int32(c.docLen[d]), avgdl) + } + } + out := make([]SearchResult, 0, len(scores)) + for d, s := range scores { + out = append(out, SearchResult{DocID: d, Score: s}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Score != out[j].Score { + return out[i].Score > out[j].Score + } + return out[i].DocID.(int64) < out[j].DocID.(int64) + }) + if len(out) > limit { + out = out[:limit] + } + return out +} + +func recompute(c *corpus, terms []string, pk int64) float64 { + n := len(c.pks) + avgdl := c.avgDocLen() + weights := map[string]float64{} + for _, t := range terms { + weights[t]++ + } + s := 0.0 + for t, w := range weights { + dm, ok := c.docTf[t] + if !ok { + continue + } + if raw, ok := dm[pk]; ok { + df := len(dm) + idf := math.Log10(float64(n) / float64(df)) + s += w * idf * idf * bm25Factor(cappedTf(raw), int32(c.docLen[pk]), avgdl) + } + } + return s +} + +// buildModelAndCorpus generates a random corpus, feeds it to the Builder one +// occurrence at a time (Add per token), and returns the model + reference. +func buildModelAndCorpus(t *testing.T, rng *rand.Rand, nDocs, nTerms, maxPostings int) (*WandModel, *corpus) { + t.Helper() + terms := make([]string, nTerms) + for i := range terms { + terms[i] = fmt.Sprintf("term%04d", i) // out-of-dict → exercises overflow ids + } + c := &corpus{docTf: map[string]map[int64]int{}, docLen: map[int64]int{}, pks: map[int64]bool{}} + b := NewBuilder("test", testPkType) + for _, term := range terms { + k := 1 + rng.Intn(maxPostings) + for j := 0; j < k; j++ { + d := int64(rng.Intn(nDocs)) + tf := 1 + rng.Intn(6) + for o := 0; o < tf; o++ { + if err := b.Add(term, d); err != nil { + t.Fatalf("Add: %v", err) + } + } + if c.docTf[term] == nil { + c.docTf[term] = map[int64]int{} + } + c.docTf[term][d] += tf + c.docLen[d] += tf // every token contributes to doc length + c.pks[d] = true + } + } + return b.Finish(), c +} + +func assertTopKEqual(t *testing.T, label string, got, want []SearchResult, c *corpus, terms []string) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("%s: result count got %d want %d", label, len(got), len(want)) + } + for i := range got { + if math.Abs(got[i].Score-want[i].Score) > 1e-9 { + t.Fatalf("%s: rank %d score got %.12g want %.12g", label, i, got[i].Score, want[i].Score) + } + } + seen := map[int64]bool{} + for _, r := range got { + pk := r.DocID.(int64) + if seen[pk] { + t.Fatalf("%s: duplicate doc-id %d", label, pk) + } + seen[pk] = true + if exp := recompute(c, terms, pk); math.Abs(exp-r.Score) > 1e-9 { + t.Fatalf("%s: doc %d score got %.12g recomputed %.12g", label, pk, r.Score, exp) + } + } +} + +func TestWandDifferential(t *testing.T) { + rng := rand.New(rand.NewSource(20260619)) + allTerms := make([]string, 40) + for i := range allTerms { + allTerms[i] = fmt.Sprintf("term%04d", i) + } + for iter := 0; iter < 40; iter++ { + m, c := buildModelAndCorpus(t, rng, 300+rng.Intn(400), 40, 100) + nq := 1 + rng.Intn(6) + q := make([]string, nq) + for i := range q { + if rng.Intn(10) == 0 { + q[i] = "absent_term" + } else { + q[i] = allTerms[rng.Intn(len(allTerms))] + } + } + limit := 1 + rng.Intn(20) + got := m.Search(q, limit, nil) + want := bruteForce(c, q, limit, nil) + assertTopKEqual(t, fmt.Sprintf("iter=%d q=%v lim=%d", iter, q, limit), got, want, c, q) + } +} + +// TestWandSegments verifies that splitting a corpus into capacity-bounded +// segments and merging with corpus-global stats yields the SAME top-K as the +// single-index brute force. +func TestWandSegments(t *testing.T) { + rng := rand.New(rand.NewSource(7)) + allTerms := make([]string, 30) + for i := range allTerms { + allTerms[i] = fmt.Sprintf("term%04d", i) + } + for iter := 0; iter < 20; iter++ { + nDocs := 300 + rng.Intn(400) + c := &corpus{docTf: map[string]map[int64]int{}, docLen: map[int64]int{}, pks: map[int64]bool{}} + b := NewBuilder("seg", testPkType) + for _, term := range allTerms { + k := 1 + rng.Intn(80) + for j := 0; j < k; j++ { + d := int64(rng.Intn(nDocs)) + tf := 1 + rng.Intn(6) + for o := 0; o < tf; o++ { + if err := b.Add(term, d); err != nil { + t.Fatal(err) + } + } + if c.docTf[term] == nil { + c.docTf[term] = map[int64]int{} + } + c.docTf[term][d] += tf + c.docLen[d] += tf + c.pks[d] = true + } + } + capacity := int64(50 + rng.Intn(200)) + segs := b.FinishSegments(capacity) + + nq := 1 + rng.Intn(5) + q := make([]string, nq) + for i := range q { + q[i] = allTerms[rng.Intn(len(allTerms))] + } + limit := 1 + rng.Intn(15) + got := SearchSegments(segs, q, limit, nil) + want := bruteForce(c, q, limit, nil) + assertTopKEqual(t, fmt.Sprintf("seg iter=%d nseg=%d cap=%d", iter, len(segs), capacity), got, want, c, q) + } +} + +// TestWandMerge verifies Merge of two independently-built, disjoint-doc indexes +// (with conflicting overflow word-ids) equals the brute force over the union. +func TestWandMerge(t *testing.T) { + rng := rand.New(rand.NewSource(123)) + terms := make([]string, 30) + for i := range terms { + terms[i] = fmt.Sprintf("term%04d", i) + } + for iter := 0; iter < 20; iter++ { + c := &corpus{docTf: map[string]map[int64]int{}, docLen: map[int64]int{}, pks: map[int64]bool{}} + // order drives overflow-id assignment; reverse it for B so the two + // builds assign different ids to the same words → exercises reconcile. + fill := func(b *Builder, order []string, pkBase, n int) { + for _, term := range order { + k := 1 + rng.Intn(40) + for j := 0; j < k; j++ { + d := int64(pkBase + rng.Intn(n)) + tf := 1 + rng.Intn(5) + for o := 0; o < tf; o++ { + if err := b.Add(term, d); err != nil { + t.Fatal(err) + } + } + if c.docTf[term] == nil { + c.docTf[term] = map[int64]int{} + } + c.docTf[term][d] += tf + c.docLen[d] += tf + c.pks[d] = true + } + } + } + nA, nB := 100+rng.Intn(200), 100+rng.Intn(200) + rev := make([]string, len(terms)) + for i := range terms { + rev[len(terms)-1-i] = terms[i] + } + ba, bb := NewBuilder("a", testPkType), NewBuilder("b", testPkType) + fill(ba, terms, 0, nA) + fill(bb, rev, nA, nB) + merged := Merge("m", ba.Finish(), bb.Finish()) + + nq := 1 + rng.Intn(5) + q := make([]string, nq) + for i := range q { + q[i] = terms[rng.Intn(len(terms))] + } + limit := 1 + rng.Intn(15) + got := merged.Search(q, limit, nil) + want := bruteForce(c, q, limit, nil) + assertTopKEqual(t, fmt.Sprintf("merge iter=%d N=%d", iter, merged.N), got, want, c, q) + } +} + +// buildSeg builds a one-flush delta segment with the given chunk_id (its append +// position in the tag=1 CdcTail log). docs maps pk -> the terms occurring in +// that doc (one tf each). +func buildSeg(t *testing.T, chunkId int64, docs map[int64][]string) *WandModel { + b := NewBuilder(fmt.Sprintf("seg%d", chunkId), testPkType) + for pk, terms := range docs { + for _, term := range terms { + if err := b.Add(term, pk); err != nil { + t.Fatal(err) + } + } + } + m := b.Finish() + m.Recency = chunkId + return m +} + +// pkCounts returns pk -> number of times it appears in the results (so a value +// > 1 flags a cross-segment duplicate). +func pkCounts(res []SearchResult) map[int64]int { + out := map[int64]int{} + for _, r := range res { + out[r.DocID.(int64)]++ + } + return out +} + +// TestWandLiveness exercises the chunk_id-as-identity rule (ComputeLiveness + +// SearchSegmentsLive) that makes CDC delete-then-reinsert / UPDATE correct over +// immutable segments. Assertions are on the LIVE pk SET (dedup / delete / +// reinsert), not exact scores: global N/df/avgdl intentionally still include +// superseded+deleted docs until compaction (accepted stat drift), so scores +// drift but membership must be exact. +func TestWandLiveness(t *testing.T) { + q := []string{"x"} + + // 1. UPDATE = same pk in two segments → newest-chunk_id wins, exactly one row. + t.Run("dedup_update", func(t *testing.T) { + segs := []*WandModel{ + buildSeg(t, 1, map[int64][]string{5: {"x"}, 6: {"x"}}), + buildSeg(t, 2, map[int64][]string{5: {"x"}}), // pk 5 updated + } + live := ComputeLiveness(segs, nil) + got := pkCounts(SearchSegmentsLive(segs, q, 10, nil, live)) + if got[5] != 1 || got[6] != 1 || len(got) != 2 { + t.Fatalf("dedup: want {5:1,6:1}, got %v", got) + } + // Without liveness the stale copy leaks → pk 5 appears twice (this is + // exactly why the search adapter must use ComputeLiveness). + dup := pkCounts(SearchSegments(segs, q, 10, nil)) + if dup[5] != 2 { + t.Fatalf("expected the no-liveness path to duplicate pk 5, got %v", dup) + } + }) + + // 2. DELETE then reINSERT at a higher chunk_id → live again. + t.Run("delete_then_reinsert", func(t *testing.T) { + segs := []*WandModel{ + buildSeg(t, 1, map[int64][]string{5: {"x"}}), + buildSeg(t, 3, map[int64][]string{5: {"x"}}), // reinsert at chunk_id 3 + } + deletes := map[any]int64{normalizeKey(int64(5)): 2} // delete at chunk_id 2 < 3 + got := pkCounts(SearchSegmentsLive(segs, q, 10, nil, ComputeLiveness(segs, deletes))) + if got[5] != 1 || len(got) != 1 { + t.Fatalf("delete-then-reinsert: want {5:1}, got %v", got) + } + }) + + // 3. DELETE after the latest insert → gone. + t.Run("delete_after_insert", func(t *testing.T) { + segs := []*WandModel{ + buildSeg(t, 1, map[int64][]string{5: {"x"}}), + buildSeg(t, 3, map[int64][]string{5: {"x"}}), + } + deletes := map[any]int64{normalizeKey(int64(5)): 4} // delete at chunk_id 4 > 3 + got := pkCounts(SearchSegmentsLive(segs, q, 10, nil, ComputeLiveness(segs, deletes))) + if len(got) != 0 { + t.Fatalf("delete-after-insert: want empty, got %v", got) + } + }) + + // 4. Pure DELETE of one pk among several. + t.Run("pure_delete", func(t *testing.T) { + segs := []*WandModel{buildSeg(t, 1, map[int64][]string{5: {"x"}, 6: {"x"}, 7: {"x"}})} + deletes := map[any]int64{normalizeKey(int64(6)): 2} + got := pkCounts(SearchSegmentsLive(segs, q, 10, nil, ComputeLiveness(segs, deletes))) + if got[5] != 1 || got[7] != 1 || got[6] != 0 || len(got) != 2 { + t.Fatalf("pure-delete: want {5,7}, got %v", got) + } + }) + + // 5. Mixed: insert base, update one + insert one in a delta, delete one. + t.Run("mixed", func(t *testing.T) { + segs := []*WandModel{ + buildSeg(t, 1, map[int64][]string{1: {"x"}, 2: {"x"}, 3: {"x"}}), + buildSeg(t, 2, map[int64][]string{2: {"x"}, 4: {"x"}}), // 2 updated, 4 new + } + deletes := map[any]int64{normalizeKey(int64(3)): 2} // delete 3 + got := pkCounts(SearchSegmentsLive(segs, q, 10, nil, ComputeLiveness(segs, deletes))) + want := map[int64]int{1: 1, 2: 1, 4: 1} + if len(got) != len(want) { + t.Fatalf("mixed: want %v, got %v", want, got) + } + for pk, n := range want { + if got[pk] != n { + t.Fatalf("mixed: pk %d want %d, got %d (full %v)", pk, n, got[pk], got) + } + } + }) + + // 6. No tag=0 base (Bug 2): an index created on an empty table has no + // compacted-main segment — its corpus is entirely tag=1 CDC deltas (chunk_id + // >= 0, never the baseRecency -1). Liveness/search must work with tail-only + // segments: dedup across deltas and honor a delete, with no base present. + t.Run("no_base_tail_only", func(t *testing.T) { + segs := []*WandModel{ + buildSeg(t, 0, map[int64][]string{5: {"x"}, 6: {"x"}}), // first CDC delta + buildSeg(t, 1, map[int64][]string{5: {"x"}, 7: {"x"}}), // 5 updated, 7 new + } + deletes := map[any]int64{normalizeKey(int64(6)): 2} // delete 6 at chunk 2 + got := pkCounts(SearchSegmentsLive(segs, q, 10, nil, ComputeLiveness(segs, deletes))) + want := map[int64]int{5: 1, 7: 1} + if len(got) != len(want) { + t.Fatalf("no-base: want %v, got %v", want, got) + } + for pk, n := range want { + if got[pk] != n { + t.Fatalf("no-base: pk %d want %d, got %d (full %v)", pk, n, got[pk], got) + } + } + }) +} + +// TestWandSearchCachedLivenessStats proves item 3 (load-time liveness/stats +// caching): precomputing ComputeLiveness + corpusStats once and feeding them to +// searchSegmentsLiveStats yields results IDENTICAL to recomputing per query +// (searchSegsLive) — both unfiltered and with a per-query WHERE prefilter — and +// the per-query filter-combine never mutates the cached liveness slice (a later +// unfiltered query still matches the recompute oracle). +func TestWandSearchCachedLivenessStats(t *testing.T) { + base := buildSeg(t, baseRecency, map[int64][]string{1: {"x"}, 2: {"x"}, 3: {"x"}}) + tail := buildSeg(t, 2, map[int64][]string{2: {"x"}, 4: {"x"}}) // 2 updated, 4 new + defer base.Free() + defer tail.Free() + segs := []*WandModel{base, tail} + deletes := map[any]int64{normalizeKey(int64(3)): 3} // delete pk 3 at chunk 3 + + // Precompute once, exactly as WandSearch.Load does. + cachedLive := ComputeLiveness(segs, deletes) + gN, gAvg := corpusStats(segs) + + // The WandSearch.Search combine: a FRESH slice when filtered, never touching cachedLive. + cachedSearch := func(mkAllow func(*WandModel) Membership) []SearchResult { + live := cachedLive + if mkAllow != nil { + live = make([]Membership, len(segs)) + for i, s := range segs { + var b Membership + if i < len(cachedLive) { + b = cachedLive[i] + } + live[i] = andAllow(mkAllow(s), b) + } + } + return searchSegmentsLiveStats(segs, []string{"x"}, 10, nil, live, gN, gAvg) + } + eq := func(name string, got, want []SearchResult) { + t.Helper() + g, w := pkCounts(got), pkCounts(want) + if len(g) != len(w) { + t.Fatalf("%s: cached %v vs recompute %v", name, g, w) + } + for pk, n := range w { + if g[pk] != n { + t.Fatalf("%s: pk %d cached %d vs recompute %d (%v/%v)", name, pk, g[pk], n, g, w) + } + } + } + + // unfiltered: cached == recompute ({1,2,4}; 3 deleted) + eq("unfiltered", cachedSearch(nil), searchSegsLive(segs, deletes, []string{"x"}, 10, nil)) + + allowA := map[int64]bool{1: true, 4: true} + mkA := func(m *WandModel) Membership { return &ordMembership{m: m, allowPk: allowA} } + eq("filterA", cachedSearch(mkA), searchSegsLive(segs, deletes, []string{"x"}, 10, mkA)) + + allowB := map[int64]bool{2: true} + mkB := func(m *WandModel) Membership { return &ordMembership{m: m, allowPk: allowB} } + eq("filterB", cachedSearch(mkB), searchSegsLive(segs, deletes, []string{"x"}, 10, mkB)) + + // unfiltered again: cachedLive must be intact after the filtered calls. + eq("unfiltered-again", cachedSearch(nil), searchSegsLive(segs, deletes, []string{"x"}, 10, nil)) +} + +// TestWandToInsertSqlsTag checks the tag column threads through: tag=0 for the +// compacted main index, tag=1 for a CDC delta segment. +func TestWandToInsertSqlsTag(t *testing.T) { + b := NewBuilder("seg-1", testPkType) + for _, term := range []string{"a", "b", "营养"} { + if err := b.Add(term, int64(1)); err != nil { + t.Fatal(err) + } + } + m := b.Finish() + cfg := TableConfig{DbName: "db", IndexTable: "ft_index", MetadataTable: "ft_meta"} + + for _, tag := range []int{0, 1} { + sqls, cleanup, err := m.ToInsertSqls(cfg, 123, tag) + if err != nil { + t.Fatal(err) + } + defer cleanup() + // the chunk INSERT(s) must carry the requested tag, and the wrong tag + // must not appear. + want := fmt.Sprintf(", %d)", tag) + bad := fmt.Sprintf(", %d)", 1-tag) + found := false + for _, s := range sqls { + if bytes.Contains([]byte(s), []byte("ft_index")) { + if !bytes.Contains([]byte(s), []byte(want)) { + t.Fatalf("tag=%d: chunk insert missing %q: %s", tag, want, s) + } + if bytes.Contains([]byte(s), []byte(bad)) { + t.Fatalf("tag=%d: chunk insert has wrong tag %q: %s", tag, bad, s) + } + found = true + } + } + if !found { + t.Fatalf("tag=%d: no ft_index chunk insert generated", tag) + } + } +} + +// TestWandDeleteLogRoundTrip checks the tag=1 delete-log codec round-trips for +// int64 and varchar PKs, validates CRC, and folds to the max-LSN map. +func TestWandDeleteLogRoundTrip(t *testing.T) { + t.Run("int64", func(t *testing.T) { + recs := []DeleteRecord{{Pk: int64(5)}, {Pk: int64(9)}, {Pk: int64(5)}} + buf, err := EncodeDeleteLog(int32(types.T_int64), recs) + if err != nil { + t.Fatal(err) + } + got, err := DecodeDeleteLog(buf) + if err != nil { + t.Fatal(err) + } + if len(got) != len(recs) { + t.Fatalf("want %d recs, got %d", len(recs), len(got)) + } + for i := range recs { + if got[i].Pk.(int64) != recs[i].Pk.(int64) { + t.Fatalf("rec %d mismatch: want %v got %v", i, recs[i], got[i]) + } + } + // fold by frame chunk_id: a later frame raises the bound, an earlier + // (redelivered) frame is a no-op. pk 5 at chunk_id 4, then redelivered + // at 2 → stays 4; pk 9 re-deleted at 7. + m := FoldDeleteFrame(nil, got, 4) + m = FoldDeleteFrame(m, []DeleteRecord{{Pk: int64(9)}}, 7) + m = FoldDeleteFrame(m, []DeleteRecord{{Pk: int64(5)}}, 2) + if m[normalizeKey(int64(5))] != 4 || m[normalizeKey(int64(9))] != 7 { + t.Fatalf("FoldDeleteFrame fold wrong: %v", m) + } + }) + + t.Run("varchar", func(t *testing.T) { + recs := []DeleteRecord{{Pk: []byte("doc-a")}, {Pk: []byte("doc-b")}} + buf, err := EncodeDeleteLog(int32(types.T_varchar), recs) + if err != nil { + t.Fatal(err) + } + got, err := DecodeDeleteLog(buf) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || string(got[0].Pk.([]byte)) != "doc-a" || string(got[1].Pk.([]byte)) != "doc-b" { + t.Fatalf("varchar round-trip wrong: %v", got) + } + m := FoldDeleteFrame(nil, got, 5) + if m[normalizeKey([]byte("doc-a"))] != 5 || m[normalizeKey([]byte("doc-b"))] != 5 { + t.Fatalf("varchar fold wrong: %v", m) + } + }) + + t.Run("corruption_detected", func(t *testing.T) { + buf, err := EncodeDeleteLog(int32(types.T_int64), []DeleteRecord{{Pk: int64(1)}}) + if err != nil { + t.Fatal(err) + } + buf[8] ^= 0xff // flip a byte in the body + if _, err := DecodeDeleteLog(buf); err == nil { + t.Fatal("expected checksum mismatch error") + } + }) + + t.Run("empty", func(t *testing.T) { + buf, err := EncodeDeleteLog(int32(types.T_int64), nil) + if err != nil { + t.Fatal(err) + } + got, err := DecodeDeleteLog(buf) + if err != nil || len(got) != 0 { + t.Fatalf("empty round-trip: got %v err %v", got, err) + } + if FoldDeleteFrame(nil, got, 0) != nil { + t.Fatal("empty fold should stay nil") + } + }) +} + +// ordMembership filters by pk, evaluated on doc ords via the model's pk map. +type ordMembership struct { + m *WandModel + allowPk map[int64]bool +} + +func (o ordMembership) Contains(ord int64) bool { + pk, _ := o.m.PkAt(ord).(int64) + return o.allowPk[pk] +} + +func TestWandPrefilter(t *testing.T) { + rng := rand.New(rand.NewSource(0xBEEF)) + allTerms := make([]string, 30) + for i := range allTerms { + allTerms[i] = fmt.Sprintf("term%04d", i) + } + for iter := 0; iter < 25; iter++ { + nDocs := 300 + rng.Intn(300) + m, c := buildModelAndCorpus(t, rng, nDocs, 30, 80) + allowPk := map[int64]bool{} + for d := range c.pks { + if rng.Intn(10) < 3 { + allowPk[d] = true + } + } + nq := 1 + rng.Intn(5) + q := make([]string, nq) + for i := range q { + q[i] = allTerms[rng.Intn(len(allTerms))] + } + limit := 1 + rng.Intn(15) + got := m.Search(q, limit, ordMembership{m, allowPk}) + want := bruteForce(c, q, limit, allowPk) + for _, r := range got { + if !allowPk[r.DocID.(int64)] { + t.Fatalf("iter=%d prefilter returned disallowed doc %d", iter, r.DocID.(int64)) + } + } + assertTopKEqual(t, fmt.Sprintf("prefilter iter=%d", iter), got, want, c, q) + } +} + +func TestWandSerializeRoundTrip(t *testing.T) { + rng := rand.New(rand.NewSource(99)) + m, c := buildModelAndCorpus(t, rng, 1000, 50, 400) + + buf, err := m.Serialize() + if err != nil { + t.Fatalf("Serialize: %v", err) + } + sum := Checksum(buf) + + m2, err := Deserialize("test", bytes.NewReader(buf)) + if err != nil { + t.Fatalf("Deserialize: %v", err) + } + if m2.N != m.N || m2.NumTerms() != m.NumTerms() || m2.PkType != m.PkType { + t.Fatalf("mismatch N %d/%d terms %d/%d pkType %d/%d", m2.N, m.N, m2.NumTerms(), m.NumTerms(), m2.PkType, m.PkType) + } + if buf2, _ := m2.Serialize(); Checksum(buf2) != sum { + t.Fatalf("checksum not stable across round-trip") + } + + q := []string{"term0001", "term0002", "term0010", "term0025"} + r1 := m.Search(q, 20, nil) + r2 := m2.Search(q, 20, nil) + assertTopKEqual(t, "reloaded", r2, bruteForce(c, q, 20, nil), c, q) + if len(r1) != len(r2) { + t.Fatalf("reloaded count differs %d/%d", len(r1), len(r2)) + } +} + +func TestWandEdgeCases(t *testing.T) { + b := NewBuilder("e", testPkType) + // term "a": docs 1,3,5 ; term "b": docs 2,3 ; plus an in-dict Chinese term. + add := func(w string, d int64, n int) { + for i := 0; i < n; i++ { + if err := b.Add(w, d); err != nil { + t.Fatal(err) + } + } + } + add("a", 1, 2) + add("a", 3, 1) + add("a", 5, 9) + add("b", 2, 1) + add("b", 3, 4) + add("营养", 5, 1) // dictionary word → global word-id + m := b.Finish() + + if r := m.Search(nil, 5, nil); r != nil { + t.Fatalf("empty query should be nil") + } + if r := m.Search([]string{"a"}, 0, nil); r != nil { + t.Fatalf("limit 0 should be nil") + } + if r := m.Search([]string{"absent"}, 5, nil); len(r) != 0 { + t.Fatalf("absent query should be empty") + } + if r := m.Search([]string{"营养"}, 5, nil); len(r) != 1 || r[0].DocID.(int64) != 5 { + t.Fatalf("in-dict term search failed: %v", r) + } + + // round-trip with a mix of overflow + dictionary terms must preserve results. + buf, err := m.Serialize() + if err != nil { + t.Fatal(err) + } + m2, err := Deserialize("e", bytes.NewReader(buf)) + if err != nil { + t.Fatal(err) + } + r1 := m.Search([]string{"a", "b", "营养"}, 10, nil) + r2 := m2.Search([]string{"a", "b", "营养"}, 10, nil) + if len(r1) != len(r2) || len(r1) != 4 { // docs 1,2,3,5 + t.Fatalf("expected 4 docs both, got %d/%d", len(r1), len(r2)) + } + + // tf is capped at 255: doc 1 gets "z" 300 times → scored as tf 255. + cb := NewBuilder("cap", testPkType) + for i := 0; i < 300; i++ { + _ = cb.Add("z", int64(1)) + } + _ = cb.Add("z", int64(2)) // df(z)=2 + _ = cb.Add("filler", int64(3)) // N=3 > df → idf > 0 + cm := cb.Finish() + avg := (300.0 + 1 + 1) / 3 + idf := math.Log10(3.0 / 2.0) + wantTop := idf * idf * bm25Factor(255, 300, avg) // capped tf, dl=300 + res := cm.Search([]string{"z"}, 1, nil) + if len(res) != 1 || res[0].DocID.(int64) != 1 || math.Abs(res[0].Score-wantTop) > 1e-9 { + t.Fatalf("tf cap: got %v, want top doc 1 score %.12g", res, wantTop) + } +} diff --git a/pkg/bm25/wand/wandsearch.go b/pkg/bm25/wand/wandsearch.go new file mode 100644 index 0000000000000..10a9839ddc5c1 --- /dev/null +++ b/pkg/bm25/wand/wandsearch.go @@ -0,0 +1,244 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wand + +import ( + "encoding/binary" + "math" + + "github.com/matrixorigin/matrixone/pkg/common/docfilter" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + veccache "github.com/matrixorigin/matrixone/pkg/vectorindex/cache" + "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" +) + +// WandQuery is the query payload passed through VectorIndexCache.Search to a +// WandSearch: the jieba-tokenized query terms (duplicates → weight) plus an +// optional serialized docfilter membership payload (the WHERE-clause prefilter +// pushed down as a runtime filter, exactly as fulltext_index_scan receives it). +type WandQuery struct { + Terms []string + FilterBytes []byte +} + +// docFilterMembership applies a docfilter.MembershipFilter (built from the +// WHERE-clause pks) to the WAND walk: a candidate doc ord is allowed iff its pk +// bytes pass the filter. For integer PKs this is the C int64 cbitmap +// (mo_cbitmap_contain); for other PKs a bloom (false positives removed by the +// downstream join to the filtered source). +type docFilterMembership struct { + m *WandModel + f docfilter.MembershipFilter + scratch [8]byte // reused encode buffer for the hot integer-PK path (Test copies out) +} + +func (d *docFilterMembership) Contains(ord int64) bool { + v := d.m.PkAt(ord) + // Contains runs once per candidate on the Block-Max walk hot path. For the common + // integer PKs, encode straight into a reused scratch buffer (byte-identical to + // packUint*) instead of allocating a fresh slice per call via encodePk. Test reads + // the bytes synchronously and does not retain them, so reuse is safe. + var raw []byte + switch types.T(d.m.PkType) { + case types.T_int64: + binary.LittleEndian.PutUint64(d.scratch[:], uint64(v.(int64))) + raw = d.scratch[:8] + case types.T_uint64: + binary.LittleEndian.PutUint64(d.scratch[:], v.(uint64)) + raw = d.scratch[:8] + case types.T_int32: + binary.LittleEndian.PutUint32(d.scratch[:4], uint32(v.(int32))) + raw = d.scratch[:4] + case types.T_uint32: + binary.LittleEndian.PutUint32(d.scratch[:4], v.(uint32)) + raw = d.scratch[:4] + case types.T_uuid: + // The membership filter (docfilter.buildBloomBytes -> CBloomFilter.addFixedVector) + // hashes each source uuid as its RAW 16 bytes (typeSize=16). Probe with the same + // raw bytes — NOT encodePk(uuid), which is the 36-char canonical string and would + // never hit the same bloom cell, rejecting every candidate (B1). + u := v.(types.Uuid) + raw = u[:] + default: + var err error + if raw, err = encodePk(d.m.PkType, v); err != nil { + return false + } + } + return d.f.Test(raw) +} + +// WandSearch adapts the loaded WAND segments to veccache.VectorIndexSearchIf so a +// retrieval index shares the VectorIndexCache (load-once, RW-shared, TTL +// eviction) with the vector plugins. The index is keyed in the cache by its +// storage table name. +type WandSearch struct { + cfg TableConfig + // segs holds the tag=0 base sub-indexes (each carrying its metadata.chunk_id + // recency — 0 for a full build, K for a compacted one) followed by the tag=1 + // CdcTail delta segments (ChunkId = frame chunk_id). An index created on an + // empty table has no tag=0 base, so segs may hold only tail segments (or be empty). + // deletes is pk -> max delete-frame chunk_id from the tag=1 log. + segs []*WandModel + deletes map[any]int64 + // Precomputed at Load (query-independent): per-segment liveness and the corpus + // stats. ComputeLiveness is O(total docs); computing it here (once per cache + // load) instead of per query is item 3 of the Phase-C scaling plan. live is + // parallel to segs (nil ⇒ every ord live). + live []Membership + gN int64 + gAvgDocLen float64 + // loaded distinguishes "never loaded" (Search errors) from "loaded but empty" + // (an index with no docs yet → Search returns zero rows, not an error). + loaded bool +} + +var _ veccache.VectorIndexSearchIf = (*WandSearch)(nil) + +// NewWandSearch returns an unloaded search handle; the cache calls Load before +// the first Search. +func NewWandSearch(cfg TableConfig) *WandSearch { + return &WandSearch{cfg: cfg} +} + +// Load reads the index from the WAND chunk store: the tag=0 compacted-main +// segment (offset-reassembled blob under its own index_id) plus the tag=1 +// CdcTail delta frames (one complete frame per chunk_id, in append order), +// assembled into the ordered segment set + delete map searched with liveness. +func (s *WandSearch) Load(sqlproc *sqlexec.SqlProcess) error { + // The tag=0 base may be several capacity-bounded sub-indexes, one, or none (an + // index created on an empty table has only tag=1 CDC deltas). Load them all. + bases, err := LoadAllBases(sqlproc, s.cfg) + if err != nil { + return err + } + // Each base carries its recency key (model.Recency) from metadata.chunk_id — 0 + // for a full-build base (oldest, below the tail which starts at 1), K for a + // folded/merged base — so ComputeLiveness dedups bases + tail uniformly. + tail, deletes, _, err := loadTailSegments(sqlproc, s.cfg) + if err != nil { + freeSegs(bases) + return err + } + s.segs = append(bases, tail...) + s.deletes = deletes + // Precompute the query-independent liveness + corpus stats once here, so the + // per-query path (Search) skips the O(total-docs) ComputeLiveness scan. + s.live = ComputeLiveness(s.segs, s.deletes) + s.gN, s.gAvgDocLen = corpusStats(s.segs) + s.loaded = true + return nil +} + +// Search runs WAND top-K and returns ([]any doc-ids of the source pk type, +// []float64 scores). +func (s *WandSearch) Search(proc *sqlexec.SqlProcess, query any, rt vectorindex.RuntimeConfig) (keys any, distances []float64, err error) { + if !s.loaded { + return nil, nil, moerr.NewInternalError(proc.GetContext(), "wand index not loaded") + } + if len(s.segs) == 0 { + // A loaded but empty index (no docs yet) matches nothing. + return []any{}, []float64{}, nil + } + q, ok := query.(WandQuery) + if !ok { + return nil, nil, moerr.NewInternalError(proc.GetContext(), "wand search: invalid query payload") + } + // rt.Limit is uint; a value past MaxInt32 (e.g. an absurd pushed LIMIT) would + // wrap negative in int(...) and get clamped to 1, silently truncating the + // top-K. Clamp such values to "effectively all" instead of wrapping. + limit := int(rt.Limit) + if rt.Limit > uint(math.MaxInt32) { + limit = math.MaxInt32 + } else if limit <= 0 { + limit = 1 + } + // The WHERE prefilter is pk-based, so it must resolve against each segment's + // own ord→pk dictionary — build one membership per segment. + var mkAllow func(*WandModel) Membership + if len(q.FilterBytes) > 0 { + f, ferr := docfilter.New(q.FilterBytes) + if ferr != nil { + return nil, nil, ferr + } + defer f.Free() + mkAllow = func(m *WandModel) Membership { return &docFilterMembership{m: m, f: f} } + } + // Combine the load-cached liveness with the per-query WHERE prefilter. When + // there's a filter, build a FRESH slice (never mutate the cached s.live, which + // is shared across all queries between reloads). + live := s.live + if mkAllow != nil { + live = make([]Membership, len(s.segs)) + for i, seg := range s.segs { + var base Membership + if i < len(s.live) { + base = s.live[i] + } + live[i] = andAllow(mkAllow(seg), base) + } + } + // STREAMING no-LIMIT path: when the caller passes an Emit callback (the TVF + // does this only for a query with no pushed LIMIT), yield every matching doc in + // bounded batches — no top-K heap, no internal sort. Ranking is done by the + // upstream ORDER BY score node. Results are handed off through Emit, so return + // empty keys/distances. + if rt.Emit != nil { + if e := streamSegmentsLiveStats(s.segs, q.Terms, rt.Emit, nil, live, s.gN, s.gAvgDocLen); e != nil { + return nil, nil, e + } + return []any{}, []float64{}, nil + } + + res := searchSegmentsLiveStats(s.segs, q.Terms, limit, nil, live, s.gN, s.gAvgDocLen) + keysOut := make([]any, len(res)) + dist := make([]float64, len(res)) + for i, r := range res { + keysOut[i] = r.DocID + dist[i] = r.Score + } + return keysOut, dist, nil +} + +// SearchFloat32 is unsupported (fulltext scores are float64; the vector +// float32 fast-path does not apply). +func (s *WandSearch) SearchFloat32(proc *sqlexec.SqlProcess, query any, rt vectorindex.RuntimeConfig, outKeys []int64, outDists []float32) error { + return moerr.NewInternalError(proc.GetContext(), "wand search: SearchFloat32 not supported") +} + +// UpdateConfig refreshes the table config from a freshly-built search handle +// (the cache passes the newest one on each call). +func (s *WandSearch) UpdateConfig(newalgo veccache.VectorIndexSearchIf) error { + if n, ok := newalgo.(*WandSearch); ok { + s.cfg = n.cfg + } + return nil +} + +// Destroy frees the off-heap (C-allocated) postings and drops the model. The +// cache holds the write lock around this, so no search is in flight. +func (s *WandSearch) Destroy() { + logutil.Debugf("[wand] WandSearch.Destroy: freeing %d cached segments for index=%s", len(s.segs), s.cfg.IndexTable) + freeSegs(s.segs) + s.segs = nil + s.deletes = nil + s.live = nil + s.gN = 0 + s.gAvgDocLen = 0 + s.loaded = false +} diff --git a/pkg/catalog/secondary_index_utils.go b/pkg/catalog/secondary_index_utils.go index eff5aad41435f..cba2d6b030f3e 100644 --- a/pkg/catalog/secondary_index_utils.go +++ b/pkg/catalog/secondary_index_utils.go @@ -38,6 +38,7 @@ const ( MoIndexHnswAlgo = tree.INDEX_TYPE_HNSW // used for HNSW Index on Vector/Array columns MoIndexCagraAlgo = tree.INDEX_TYPE_CAGRA // used for CAGRA Index on Vector/Array columns MoIndexIvfpqAlgo = tree.INDEX_TYPE_IVFPQ // used for IVFPQ Index on Vector/Array columns + MoIndexBm25Algo = tree.INDEX_TYPE_BM25 // used for BM25 ranked-retrieval Index on TEXT/VARCHAR columns ) // ToLower is used for before comparing AlgoType and IndexAlgoParamOpType. Reason why they are strings @@ -107,6 +108,7 @@ const ( AutoUpdate = "auto_update" Day = "day" Hour = "hour" + Second = "second" DistributionMode = "distribution_mode" Quantization = "quantization" BitsPerCode = "bits_per_code" diff --git a/pkg/catalog/types.go b/pkg/catalog/types.go index c7ee8630790f3..1d274ae7b0906 100644 --- a/pkg/catalog/types.go +++ b/pkg/catalog/types.go @@ -385,6 +385,13 @@ const ( SystemSI_IVFFLAT_TblCol_Metadata_key = "__mo_index_key" SystemSI_IVFFLAT_TblCol_Metadata_val = "__mo_index_val" + // IVF_FLAT MetadataTable - well-known keys (rows in the key/val metadata table) + // QuantizeMin/QuantizeMax store the trained int8 scalar-quantizer bounds + // (cuVS-style asymmetric): [min,max] is mapped to the full int8 range [-128,127] + // via q(x)=round(x*mul+add). Entries and the query use the same transform. + SystemSI_IVFFLAT_Metadata_QuantizeMin = "quantize_min" + SystemSI_IVFFLAT_Metadata_QuantizeMax = "quantize_max" + // IVF_FLAT Centroids - Column names SystemSI_IVFFLAT_TblCol_Centroids_version = "__mo_index_centroid_version" SystemSI_IVFFLAT_TblCol_Centroids_id = "__mo_index_centroid_id" @@ -405,6 +412,37 @@ const ( FullTextIndex_TabCol_Id = "doc_id" FullTextIndex_TabCol_Position = "pos" + /************ 3b. BM25 Index **************/ + + // BM25 ranked-retrieval index: a chunked binary (WAND) index store + metadata, + // HNSW-style. A bm25 index (CREATE INDEX ... USING bm25) has these two hidden + // tables (no postings table — it builds directly from the source rows); both + // share the IndexName and are distinguished by IndexAlgoTableType. + // NOTE: IndexAlgoTableType is stored in a varchar(11) catalog column, so these + // must be <= 11 chars (cf. "cagra_index"/"ivfpq_index"). + Bm25Index_TblType_Metadata = "bm25_meta" + Bm25Index_TblType_Storage = "bm25_index" + + Bm25Index_TblCol_Storage_Index_Id = "index_id" + Bm25Index_TblCol_Storage_Chunk_Id = "chunk_id" + Bm25Index_TblCol_Storage_Data = "data" + Bm25Index_TblCol_Storage_Tag = "tag" + + Bm25Index_TblCol_Metadata_Index_Id = "index_id" + Bm25Index_TblCol_Metadata_Timestamp = "timestamp" + Bm25Index_TblCol_Metadata_Checksum = "checksum" + Bm25Index_TblCol_Metadata_Filesize = "filesize" + // Recency is the tag=0 base sub-index's recency key (0 for a full-build base = oldest; + // K = max folded tail chunk_id for a compacted base). ComputeLiveness dedups bases + + // tail uniformly by this; NextTailChunkId reads MAX over it too. Named distinctly from + // the storage table's chunk_id (a physical chunk position within a sub's blob), which + // is a different concept that happened to share the column name. + Bm25Index_TblCol_Metadata_Recency = "recency" + // Nrow is the sub-index's live doc count. Tiered merge reads it (without loading the + // sub) to skip subs already at max_index_capacity — a "full" sub is optimal and is + // never re-merged, so a MERGE over a pure-insert tail never rewrites the full base. + Bm25Index_TblCol_Metadata_Nrow = "nrow" + /************ 4. HNSW Index *************/ // HNSW Table Types diff --git a/pkg/compare/arraycompare.go b/pkg/compare/arraycompare.go index 885278268cf1c..3ac0695297ba8 100644 --- a/pkg/compare/arraycompare.go +++ b/pkg/compare/arraycompare.go @@ -59,6 +59,14 @@ func (c arrayCompare) Compare(veci, vecj int, vi, vj int64) int { return types.CompareArrayFromBytes[float32](_x, _y, c.desc) case types.T_array_float64: return types.CompareArrayFromBytes[float64](_x, _y, c.desc) + case types.T_array_bf16: + return types.CompareArrayElementFromBytes[types.BF16](_x, _y, c.desc) + case types.T_array_float16: + return types.CompareArrayElementFromBytes[types.Float16](_x, _y, c.desc) + case types.T_array_int8: + return types.CompareArrayElementFromBytes[int8](_x, _y, c.desc) + case types.T_array_uint8: + return types.CompareArrayElementFromBytes[uint8](_x, _y, c.desc) default: panic("Compare Not supported") } diff --git a/pkg/compare/arraycompare_narrow_test.go b/pkg/compare/arraycompare_narrow_test.go new file mode 100644 index 0000000000000..a78ead4a3eebc --- /dev/null +++ b/pkg/compare/arraycompare_narrow_test.go @@ -0,0 +1,92 @@ +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compare + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/stretchr/testify/require" +) + +// TestArrayCompareNarrow covers the bf16 / f16 / int8 / uint8 switch arms of +// arrayCompare.Compare, which testutil.NewVector cannot build (it only supports +// float32/float64 arrays). Two rows where the first sorts before the second let +// us assert asc/desc symmetry and the equal-element zero case. +func TestArrayCompareNarrow(t *testing.T) { + mp := mpool.MustNewZero() + + buildBF16 := func() (a, b *vector.Vector) { + return newArrayVec(t, mp, types.T_array_bf16, + [][]types.BF16{{types.BF16FromFloat32(1), types.BF16FromFloat32(2)}}), + newArrayVec(t, mp, types.T_array_bf16, + [][]types.BF16{{types.BF16FromFloat32(1), types.BF16FromFloat32(3)}}) + } + buildF16 := func() (a, b *vector.Vector) { + return newArrayVec(t, mp, types.T_array_float16, + [][]types.Float16{{types.Float16FromFloat32(1), types.Float16FromFloat32(2)}}), + newArrayVec(t, mp, types.T_array_float16, + [][]types.Float16{{types.Float16FromFloat32(1), types.Float16FromFloat32(3)}}) + } + buildI8 := func() (a, b *vector.Vector) { + return newArrayVec(t, mp, types.T_array_int8, [][]int8{{1, 2}}), + newArrayVec(t, mp, types.T_array_int8, [][]int8{{1, 3}}) + } + buildU8 := func() (a, b *vector.Vector) { + return newArrayVec(t, mp, types.T_array_uint8, [][]uint8{{1, 2}}), + newArrayVec(t, mp, types.T_array_uint8, [][]uint8{{1, 3}}) + } + + for _, tc := range []struct { + name string + build func() (a, b *vector.Vector) + }{ + {"bf16", buildBF16}, + {"f16", buildF16}, + {"int8", buildI8}, + {"uint8", buildU8}, + } { + t.Run(tc.name, func(t *testing.T) { + lo, hi := tc.build() + defer lo.Free(mp) + defer hi.Free(mp) + + // Ascending: lo < hi -> negative; hi > lo -> positive; equal -> 0. + asc := New(*lo.GetType(), false, false) + asc.Set(0, lo) + asc.Set(1, hi) + require.Negative(t, asc.Compare(0, 1, 0, 0), "asc lolo") + require.Zero(t, asc.Compare(0, 0, 0, 0), "asc equal") + + // Descending flips the sign of the strict comparisons. + desc := New(*lo.GetType(), true, false) + desc.Set(0, lo) + desc.Set(1, hi) + require.Positive(t, desc.Compare(0, 1, 0, 0), "desc lo []float32{1.0,2.0,3.0} --> []bytes{11,33...} -func StringToArrayToBytes[T RealNumbers](input string) ([]byte, error) { +func StringToArrayToBytes[T ArrayElement](input string) ([]byte, error) { // Convert "[1,2,3]" --> []float32{1.0, 2.0, 3.0} a, err := StringToArray[T](input) if err != nil { @@ -123,7 +131,7 @@ func StringToArrayToBytes[T RealNumbers](input string) ([]byte, error) { return ArrayToBytes(a), nil } -func BytesToArrayToString[T RealNumbers](input []byte) string { +func BytesToArrayToString[T ArrayElement](input []byte) string { // Convert []byte{11, 33, 45, 56,.....} --> []float32{1.0, 2.0, 3.0} a := BytesToArray[T](input) diff --git a/pkg/container/types/array_str.go b/pkg/container/types/array_str.go index c95f2a1cf0279..160657943abc8 100644 --- a/pkg/container/types/array_str.go +++ b/pkg/container/types/array_str.go @@ -261,7 +261,7 @@ func indexFrom(str, substr string, start int) int { } // stringToT convert str to T -func stringToT[T RealNumbers](str string) (t T, err error) { +func stringToT[T ArrayElement](str string) (t T, err error) { switch any(t).(type) { case float32: num, err := strconv.ParseFloat(str, 32) @@ -277,6 +277,42 @@ func stringToT[T RealNumbers](str string) (t T, err error) { return t, moerr.NewInternalErrorNoCtxf("error while casting %s to %s", str, T_float64.String()) } return *(*T)(unsafe.Pointer(&num)), nil + case BF16: + num, err := strconv.ParseFloat(str, 32) + if err != nil { + return t, moerr.NewInternalErrorNoCtxf("error while casting %s to %s", str, T_array_bf16.String()) + } + bf := BF16FromFloat32(float32(num)) + return *(*T)(unsafe.Pointer(&bf)), nil + case Float16: + num, err := strconv.ParseFloat(str, 32) + if err != nil { + return t, moerr.NewInternalErrorNoCtxf("error while casting %s to %s", str, T_array_float16.String()) + } + h := Float16FromFloat32(float32(num)) + return *(*T)(unsafe.Pointer(&h)), nil + case int8: + // Strict: a vecint8 string literal must be an integer in [-128,127]. + // Non-integer ("1.4") or out-of-range ("200") values error rather than + // silently rounding/clamping. (The vecf32 -> vecint8 CAST path does + // round+clamp; only direct string parsing is strict.) + num, err := strconv.ParseInt(str, 10, 8) + if err != nil { + return t, moerr.NewInternalErrorNoCtxf("error while casting %s to %s", str, T_array_int8.String()) + } + i8 := int8(num) + return *(*T)(unsafe.Pointer(&i8)), nil + case uint8: + // Strict: a vecuint8 string literal must be an integer in [0,255]. + // Non-integer ("1.4") or out-of-range ("300") values error rather than + // silently rounding/clamping. (The vecf32 -> vecuint8 CAST path does + // round+clamp; only direct string parsing is strict.) + num, err := strconv.ParseUint(str, 10, 8) + if err != nil { + return t, moerr.NewInternalErrorNoCtxf("error while casting %s to %s", str, T_array_uint8.String()) + } + u8 := uint8(num) + return *(*T)(unsafe.Pointer(&u8)), nil default: panic(moerr.NewInternalErrorNoCtx("not implemented")) } diff --git a/pkg/container/types/bytes.go b/pkg/container/types/bytes.go index da3766afeb686..12b34d410fc97 100644 --- a/pkg/container/types/bytes.go +++ b/pkg/container/types/bytes.go @@ -110,7 +110,7 @@ func (v *Varlena) GetByteSlice(area []byte) []byte { // GetArray Returns []T from Varlena. If the Varlena size is less than Inline size, // it returns the value from the Varlena header. // Else, it returns the value from the area. -func GetArray[T RealNumbers](v *Varlena, area []byte) []T { +func GetArray[T ArrayElement](v *Varlena, area []byte) []T { svlen := (*v)[0] if svlen <= VarlenaInlineSize { return BytesToArray[T](v.ByteSlice()) diff --git a/pkg/container/types/compare.go b/pkg/container/types/compare.go index bbf2861d39685..5d9150e51fb61 100644 --- a/pkg/container/types/compare.go +++ b/pkg/container/types/compare.go @@ -107,6 +107,14 @@ func GenericDescCompare[T OrderedT](x, y T) int { // Compare returns an integer comparing two arrays/vectors lexicographically. // TODO: this function might not be correct. we need to compare using tolerance for float values. // TODO: need to check if we need len(v1)==len(v2) check. +// ArrayElementCompare orders two narrow-typed vectors by upcasting to float32. +// Direct uint16/int8 comparison would be wrong for bf16/f16 (the sign bit makes +// bit order disagree with value order), so all element types route through the +// float32 bridge. Exact for int8/float32; lossless for the stored bf16/f16 values. +func ArrayElementCompare[T ArrayElement](v1, v2 []T) int { + return ArrayCompare[float32](ToFloat32Array(v1), ToFloat32Array(v2)) +} + func ArrayCompare[T RealNumbers](v1, v2 []T) int { minLen := len(v1) if len(v2) < minLen { @@ -138,3 +146,15 @@ func CompareArrayFromBytes[T RealNumbers](_x, _y []byte, desc bool) int { } return ArrayCompare[T](x, y) } + +// CompareArrayElementFromBytes is the narrow-type (bf16/f16/int8) counterpart of +// CompareArrayFromBytes; it orders through the float32 bridge. +func CompareArrayElementFromBytes[T ArrayElement](_x, _y []byte, desc bool) int { + x := BytesToArray[T](_x) + y := BytesToArray[T](_y) + + if desc { + return ArrayElementCompare[T](y, x) + } + return ArrayElementCompare[T](x, y) +} diff --git a/pkg/container/types/compare_narrow_test.go b/pkg/container/types/compare_narrow_test.go new file mode 100644 index 0000000000000..5c83cd0f5d307 --- /dev/null +++ b/pkg/container/types/compare_narrow_test.go @@ -0,0 +1,55 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestArrayElementCompareLength covers the length-tiebreak branches of +// ArrayElementCompare (unequal-length arrays) that the value-ordering test in +// float16_test.go does not exercise. +func TestArrayElementCompareLength(t *testing.T) { + require.Equal(t, 0, ArrayElementCompare([]int8{1, 2}, []int8{1, 2})) + require.Equal(t, -1, ArrayElementCompare([]int8{1}, []int8{1, 2})) + require.Equal(t, 1, ArrayElementCompare([]int8{1, 2}, []int8{1})) + require.Equal(t, -1, ArrayElementCompare([]uint8{1, 2}, []uint8{1, 3})) + f16a := Float32ToFloat16Slice([]float32{1, 2}) + f16b := Float32ToFloat16Slice([]float32{1, 2, 3}) + require.Equal(t, -1, ArrayElementCompare(f16a, f16b)) +} + +// TestCompareArrayElementFromBytes covers the bytes-level narrow-vector comparator +// (bf16/f16/int8/uint8), including the desc (descending) flip. +func TestCompareArrayElementFromBytes(t *testing.T) { + x := ArrayToBytes[int8]([]int8{1, 2, 3}) + y := ArrayToBytes[int8]([]int8{1, 2, 4}) + require.Equal(t, -1, CompareArrayElementFromBytes[int8](x, y, false)) + require.Equal(t, 1, CompareArrayElementFromBytes[int8](x, y, true)) // desc flips the order + + fx := ArrayToBytes(Float32ToFloat16Slice([]float32{1, 2})) + fy := ArrayToBytes(Float32ToFloat16Slice([]float32{1, 2})) + require.Equal(t, 0, CompareArrayElementFromBytes[Float16](fx, fy, false)) + + bx := ArrayToBytes(Float32ToBF16Slice([]float32{1})) + by := ArrayToBytes(Float32ToBF16Slice([]float32{2})) + require.Equal(t, -1, CompareArrayElementFromBytes[BF16](bx, by, false)) + + ux := ArrayToBytes[uint8]([]uint8{9}) + uy := ArrayToBytes[uint8]([]uint8{1}) + require.Equal(t, 1, CompareArrayElementFromBytes[uint8](ux, uy, false)) +} diff --git a/pkg/container/types/encoding.go b/pkg/container/types/encoding.go index 813d143150334..d61272b6bf4b5 100644 --- a/pkg/container/types/encoding.go +++ b/pkg/container/types/encoding.go @@ -381,7 +381,7 @@ func DecodeValue(val []byte, t T) any { return DecodeFixed[TS](val) case T_Rowid: return DecodeFixed[Rowid](val) - case T_char, T_varchar, T_blob, T_json, T_text, T_binary, T_varbinary, T_array_float32, T_array_float64, T_datalink, T_geometry, T_geometry32: + case T_char, T_varchar, T_blob, T_json, T_text, T_binary, T_varbinary, T_array_float32, T_array_float64, T_array_bf16, T_array_float16, T_array_int8, T_array_uint8, T_datalink, T_geometry, T_geometry32: return val case T_enum: return DecodeFixed[Enum](val) @@ -548,7 +548,7 @@ func EncodeValue(val any, t T) []byte { case T_Rowid: return EncodeFixed(val.(Rowid)) case T_char, T_varchar, T_blob, T_json, T_text, T_binary, T_varbinary, - T_array_float32, T_array_float64, T_datalink, T_geometry, T_geometry32: + T_array_float32, T_array_float64, T_array_bf16, T_array_float16, T_array_int8, T_array_uint8, T_datalink, T_geometry, T_geometry32: // Mainly used by Zonemap, which receives val input from DN batch/vector. // This val is mostly []bytes and not []float32 or []float64 return val.([]byte) diff --git a/pkg/container/types/float16.go b/pkg/container/types/float16.go new file mode 100644 index 0000000000000..9a61fb9166995 --- /dev/null +++ b/pkg/container/types/float16.go @@ -0,0 +1,325 @@ +// Copyright 2021 - 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "math" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" +) + +// This file defines the narrow element types used by the vecbf16 / vecf16 / +// veci8 vector column types. They participate ONLY in the storage / +// serialization / accessor / display / cast-plumbing layer (the ArrayElement +// constraint). All arithmetic (distance, normalize, ...) is performed by +// upcasting to []float32, running the existing float32 kernels, and (for +// vector-returning ops) converting back. uint16/int8 arithmetic kernels are +// never written. + +// BF16 is the bfloat16 floating-point format: the top 16 bits of an IEEE +// float32 (1 sign bit, 8 exponent bits, 7 mantissa bits). Conversion to +// float32 is a left shift; conversion from float32 truncates the low 16 bits +// with round-to-nearest-even. +type BF16 uint16 + +// Float16 is the IEEE 754 binary16 (half precision) format: 1 sign bit, 5 +// exponent bits, 10 mantissa bits. +type Float16 uint16 + +// ---------------------------------------------------------------------------- +// BF16 +// ---------------------------------------------------------------------------- + +// ToFloat32 widens a bfloat16 to float32 by placing its bits in the high half +// of the float32 representation. +func (b BF16) ToFloat32() float32 { + return math.Float32frombits(uint32(b) << 16) +} + +// BF16FromFloat32 narrows a float32 to bfloat16 using round-to-nearest-even. +// NaN inputs are preserved as a (quiet) NaN. +func BF16FromFloat32(f float32) BF16 { + x := math.Float32bits(f) + if (x>>23)&0xff == 0xff && x&0x7fffff != 0 { + // NaN: truncating the low 16 bits could zero the mantissa and turn it + // into an Inf, so force a non-zero mantissa bit. + return BF16(x>>16 | 0x0040) + } + // Round to nearest even: add 0x7fff plus the LSB of the surviving mantissa. + rounding := uint32(0x7fff) + ((x >> 16) & 1) + return BF16((x + rounding) >> 16) +} + +// ---------------------------------------------------------------------------- +// Float16 (IEEE binary16) +// +// The conversion routines below follow the vetted scalar algorithm used by +// github.com/x448/float16 (Apache-2.0), with full subnormal / Inf / NaN +// handling and round-to-nearest-even. +// ---------------------------------------------------------------------------- + +// ToFloat32 widens an IEEE half to float32. +func (h Float16) ToFloat32() float32 { + return math.Float32frombits(f16bitsToF32bits(uint16(h))) +} + +// Float16FromFloat32 narrows a float32 to an IEEE half using +// round-to-nearest-even, with overflow to Inf and subnormal handling. +func Float16FromFloat32(f float32) Float16 { + return Float16(f32bitsToF16bits(math.Float32bits(f))) +} + +func f16bitsToF32bits(in uint16) uint32 { + sign := uint32(in&0x8000) << 16 // sign bit, shifted to float32 position + exp := uint32(in&0x7c00) >> 10 // 5-bit exponent + coef := uint32(in&0x03ff) << 13 // 10-bit mantissa, shifted to float32 position + + if exp == 0x1f { + if coef == 0 { + // Infinity + return sign | 0x7f800000 + } + // NaN + return sign | 0x7fc00000 | coef + } + + if exp == 0 { + if coef == 0 { + // signed zero + return sign + } + // normalize the subnormal + exp++ + for coef&0x7f800000 == 0 { + coef <<= 1 + exp-- + } + coef &= 0x007fffff + } + + return sign | ((exp + (0x7f - 0xf)) << 23) | coef +} + +func f32bitsToF16bits(u32 uint32) uint16 { + sign := u32 & 0x80000000 + exp := u32 & 0x7f800000 + coef := u32 & 0x007fffff + + if exp == 0x7f800000 { + // NaN or Infinity + nanBit := uint32(0) + if coef != 0 { + nanBit = uint32(0x0200) + } + return uint16((sign >> 16) | uint32(0x7c00) | nanBit | (coef >> 13)) + } + + halfSign := sign >> 16 + + unbiasedExp := int32(exp>>23) - 127 + halfExp := unbiasedExp + 15 + + if halfExp >= 0x1f { + // overflow -> Inf + return uint16(halfSign | uint32(0x7c00)) + } + + if halfExp <= 0 { + if 14-halfExp > 24 { + // too small -> signed zero + return uint16(halfSign) + } + coef := coef | uint32(0x00800000) + halfCoef := coef >> uint32(14-halfExp) + roundBit := uint32(1) << uint32(13-halfExp) + if (coef&roundBit) != 0 && (coef&(3*roundBit-1)) != 0 { + halfCoef++ + } + return uint16(halfSign | halfCoef) + } + + halfExp2 := uint32(halfExp) << 10 + halfCoef := coef >> 13 + roundBit := uint32(0x00001000) + if (coef&roundBit) != 0 && (coef&(3*roundBit-1)) != 0 { + return uint16((halfSign | halfExp2 | halfCoef) + 1) + } + return uint16(halfSign | halfExp2 | halfCoef) +} + +// ---------------------------------------------------------------------------- +// Batch converters (hot path). These power the float32 bridge used by the +// distance / cast wrappers; keep them allocation-light. +// ---------------------------------------------------------------------------- + +func BF16ToFloat32Slice(src []BF16) []float32 { + dst := make([]float32, len(src)) + for i, v := range src { + dst[i] = v.ToFloat32() + } + return dst +} + +func Float16ToFloat32Slice(src []Float16) []float32 { + dst := make([]float32, len(src)) + for i, v := range src { + dst[i] = v.ToFloat32() + } + return dst +} + +func Int8ToFloat32Slice(src []int8) []float32 { + dst := make([]float32, len(src)) + for i, v := range src { + dst[i] = float32(v) + } + return dst +} + +func Uint8ToFloat32Slice(src []uint8) []float32 { + dst := make([]float32, len(src)) + for i, v := range src { + dst[i] = float32(v) + } + return dst +} + +func Float32ToBF16Slice(src []float32) []BF16 { + dst := make([]BF16, len(src)) + for i, v := range src { + dst[i] = BF16FromFloat32(v) + } + return dst +} + +func Float32ToFloat16Slice(src []float32) []Float16 { + dst := make([]Float16, len(src)) + for i, v := range src { + dst[i] = Float16FromFloat32(v) + } + return dst +} + +// Float32ToInt8Slice rounds to nearest and clamps to the int8 range +// [-128, 127]. NaN maps to 0. +func Float32ToInt8Slice(src []float32) []int8 { + dst := make([]int8, len(src)) + for i, v := range src { + dst[i] = Float32ToInt8(v) + } + return dst +} + +// Float32ToInt8 rounds to nearest (ties away from zero, via math.Round) and +// clamps to [-128, 127]. NaN maps to 0. +func Float32ToInt8(v float32) int8 { + if v != v { // NaN + return 0 + } + r := math.Round(float64(v)) + if r > 127 { + return 127 + } + if r < -128 { + return -128 + } + return int8(r) +} + +// Float32ToUint8Slice rounds to nearest and clamps to the uint8 range +// [0, 255]. NaN maps to 0. +func Float32ToUint8Slice(src []float32) []uint8 { + dst := make([]uint8, len(src)) + for i, v := range src { + dst[i] = Float32ToUint8(v) + } + return dst +} + +// Float32ToUint8 rounds to nearest (ties away from zero, via math.Round) and +// clamps to [0, 255]. NaN maps to 0. +func Float32ToUint8(v float32) uint8 { + if v != v { // NaN + return 0 + } + r := math.Round(float64(v)) + if r > 255 { + return 255 + } + if r < 0 { + return 0 + } + return uint8(r) +} + +// ---------------------------------------------------------------------------- +// Generic float32 bridge. This is THE boundary between the storage tier +// (ArrayElement) and the compute tier (RealNumbers). Any math on a narrow type +// upcasts here, runs the float32 kernel, and (for vector results) converts back. +// ---------------------------------------------------------------------------- + +// ToFloat32Array upcasts any ArrayElement slice to []float32. For []float32 it +// returns the input unchanged (no copy); callers must not mutate the result in +// place when T is float32 unless they own the input. +func ToFloat32Array[T ArrayElement](in []T) []float32 { + switch v := any(in).(type) { + case []float32: + return v + case []float64: + out := make([]float32, len(v)) + for i, x := range v { + out[i] = float32(x) + } + return out + case []BF16: + return BF16ToFloat32Slice(v) + case []Float16: + return Float16ToFloat32Slice(v) + case []int8: + return Int8ToFloat32Slice(v) + case []uint8: + return Uint8ToFloat32Slice(v) + default: + panic(moerr.NewInternalErrorNoCtx("ToFloat32Array: unsupported element type")) + } +} + +// FromFloat32Array narrows a []float32 back to the target ArrayElement type. +// int8 rounds-to-nearest and clamps to [-128,127]; bf16/f16 round-to-nearest-even. +func FromFloat32Array[T ArrayElement](in []float32) []T { + var zero T + switch any(zero).(type) { + case float32: + out := make([]float32, len(in)) + copy(out, in) + return any(out).([]T) + case float64: + out := make([]float64, len(in)) + for i, x := range in { + out[i] = float64(x) + } + return any(out).([]T) + case BF16: + return any(Float32ToBF16Slice(in)).([]T) + case Float16: + return any(Float32ToFloat16Slice(in)).([]T) + case int8: + return any(Float32ToInt8Slice(in)).([]T) + case uint8: + return any(Float32ToUint8Slice(in)).([]T) + default: + panic(moerr.NewInternalErrorNoCtx("FromFloat32Array: unsupported element type")) + } +} diff --git a/pkg/container/types/float16_test.go b/pkg/container/types/float16_test.go new file mode 100644 index 0000000000000..53e586da7cca2 --- /dev/null +++ b/pkg/container/types/float16_test.go @@ -0,0 +1,250 @@ +// Copyright 2021 - 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "math" + "testing" +) + +func TestFloat16ReferenceValues(t *testing.T) { + // (float32 input, expected IEEE half bits, expected float32 after round-trip) + cases := []struct { + in float32 + bits uint16 + out float32 + }{ + {0.0, 0x0000, 0.0}, + {1.0, 0x3c00, 1.0}, + {-1.0, 0xbc00, -1.0}, + {2.0, 0x4000, 2.0}, + {0.5, 0x3800, 0.5}, + {-0.5, 0xb800, -0.5}, + {65504.0, 0x7bff, 65504.0}, // largest normal half + {0.00006103515625, 0x0400, 0.00006103515625}, // smallest normal half (2^-14) + {0.00006097555, 0x03ff, 0.000060975552}, // largest subnormal half + {5.9604645e-08, 0x0001, 5.9604645e-08}, // smallest positive subnormal (2^-24) + } + for _, c := range cases { + got := Float16FromFloat32(c.in) + if uint16(got) != c.bits { + t.Errorf("Float16FromFloat32(%v) bits = 0x%04x, want 0x%04x", c.in, uint16(got), c.bits) + } + back := Float16(c.bits).ToFloat32() + if math.Abs(float64(back-c.out)) > 1e-9 { + t.Errorf("Float16(0x%04x).ToFloat32() = %v, want %v", c.bits, back, c.out) + } + } +} + +func TestFloat16InfNaN(t *testing.T) { + // +Inf + if got := Float16FromFloat32(float32(math.Inf(1))); uint16(got) != 0x7c00 { + t.Errorf("+Inf -> 0x%04x, want 0x7c00", uint16(got)) + } + if got := Float16FromFloat32(float32(math.Inf(-1))); uint16(got) != 0xfc00 { + t.Errorf("-Inf -> 0x%04x, want 0xfc00", uint16(got)) + } + // overflow to +Inf + if got := Float16FromFloat32(70000.0); uint16(got) != 0x7c00 { + t.Errorf("70000 -> 0x%04x, want 0x7c00 (overflow to Inf)", uint16(got)) + } + // NaN stays NaN + nan := Float16FromFloat32(float32(math.NaN())) + if !math.IsNaN(float64(nan.ToFloat32())) { + t.Errorf("NaN did not survive round-trip: got %v", nan.ToFloat32()) + } + // Inf round-trip + if v := Float16(0x7c00).ToFloat32(); !math.IsInf(float64(v), 1) { + t.Errorf("0x7c00 -> %v, want +Inf", v) + } +} + +func TestBF16ReferenceValues(t *testing.T) { + cases := []struct { + in float32 + bits uint16 + }{ + {0.0, 0x0000}, + {1.0, 0x3f80}, + {-1.0, 0xbf80}, + {2.0, 0x4000}, + {0.5, 0x3f00}, + {3.14159265, 0x4049}, // pi truncated/rounded to bf16 + } + for _, c := range cases { + got := BF16FromFloat32(c.in) + if uint16(got) != c.bits { + t.Errorf("BF16FromFloat32(%v) = 0x%04x, want 0x%04x", c.in, uint16(got), c.bits) + } + } + // bf16 keeps full float32 exponent range: round-trip is close + for _, v := range []float32{1.0, -2.5, 100.0, 0.001, 12345.0} { + back := BF16FromFloat32(v).ToFloat32() + rel := math.Abs(float64((back - v) / v)) + if rel > 0.01 { // bf16 has ~7 mantissa bits -> ~0.4% worst case + t.Errorf("BF16 round-trip %v -> %v, rel err %v too high", v, back, rel) + } + } + // NaN survives + if !math.IsNaN(float64(BF16FromFloat32(float32(math.NaN())).ToFloat32())) { + t.Errorf("BF16 NaN did not survive round-trip") + } + // Inf survives + if !math.IsInf(float64(BF16FromFloat32(float32(math.Inf(1))).ToFloat32()), 1) { + t.Errorf("BF16 +Inf did not survive round-trip") + } +} + +func TestInt8Clamp(t *testing.T) { + cases := []struct { + in float32 + out int8 + }{ + {0.0, 0}, + {1.4, 1}, + {1.6, 2}, + {-1.6, -2}, + {127.0, 127}, + {128.0, 127}, // clamp high + {200.0, 127}, // clamp high + {-128.0, -128}, + {-129.0, -128}, // clamp low + {-500.0, -128}, // clamp low + } + for _, c := range cases { + if got := Float32ToInt8(c.in); got != c.out { + t.Errorf("Float32ToInt8(%v) = %d, want %d", c.in, got, c.out) + } + } + if Float32ToInt8(float32(math.NaN())) != 0 { + t.Errorf("NaN -> int8 should be 0") + } +} + +func TestFloat32Bridge(t *testing.T) { + // ToFloat32Array for each element type + if got := ToFloat32Array([]float32{1, 2, 3}); got[0] != 1 || got[2] != 3 { + t.Errorf("f32 bridge = %v", got) + } + if got := ToFloat32Array([]float64{1, 2, 3}); got[1] != 2 { + t.Errorf("f64 bridge = %v", got) + } + if got := ToFloat32Array([]BF16{BF16FromFloat32(1.5)}); got[0] != 1.5 { + t.Errorf("bf16 bridge = %v", got) + } + if got := ToFloat32Array([]Float16{Float16FromFloat32(2.5)}); got[0] != 2.5 { + t.Errorf("f16 bridge = %v", got) + } + if got := ToFloat32Array([]int8{-5, 7}); got[0] != -5 || got[1] != 7 { + t.Errorf("int8 bridge = %v", got) + } + // FromFloat32Array narrows correctly + src := []float32{1.0, 2.0, -3.0} + if out := FromFloat32Array[float32](src); out[2] != -3.0 { + t.Errorf("from f32 = %v", out) + } + if out := FromFloat32Array[float64](src); out[0] != 1.0 { + t.Errorf("from f64 = %v", out) + } + if out := FromFloat32Array[BF16](src); out[0].ToFloat32() != 1.0 { + t.Errorf("from bf16 = %v", out) + } + if out := FromFloat32Array[Float16](src); out[1].ToFloat32() != 2.0 { + t.Errorf("from f16 = %v", out) + } + if out := FromFloat32Array[int8]([]float32{1.4, 130, -200}); out[0] != 1 || out[1] != 127 || out[2] != -128 { + t.Errorf("from int8 = %v", out) + } +} + +func TestArrayElementCompare(t *testing.T) { + // bf16/f16 must order by value, not raw bits (negative has high bit set) + neg := []BF16{BF16FromFloat32(-1.0)} + pos := []BF16{BF16FromFloat32(1.0)} + if ArrayElementCompare(neg, pos) >= 0 { + t.Errorf("bf16 compare: -1 should be < 1") + } + negh := []Float16{Float16FromFloat32(-2.0)} + posh := []Float16{Float16FromFloat32(0.5)} + if ArrayElementCompare(negh, posh) >= 0 { + t.Errorf("f16 compare: -2 should be < 0.5") + } + if ArrayElementCompare([]int8{-5}, []int8{3}) >= 0 { + t.Errorf("int8 compare: -5 should be < 3") + } +} + +func TestStringToArrayNarrow(t *testing.T) { + // int8: strict integer parse. Valid integers in range round-trip exactly. + i8, err := StringToArray[int8]("[1, -2, 127, -128, 0]") + if err != nil { + t.Fatalf("int8 parse: %v", err) + } + want := []int8{1, -2, 127, -128, 0} + for i := range want { + if i8[i] != want[i] { + t.Errorf("int8[%d] = %d, want %d", i, i8[i], want[i]) + } + } + // int8: non-integer and out-of-range literals error (no silent round/clamp). + if _, err := StringToArray[int8]("[1.4]"); err == nil { + t.Errorf("int8 parse of non-integer should error") + } + if _, err := StringToArray[int8]("[200]"); err == nil { + t.Errorf("int8 parse of out-of-range should error") + } + if _, err := StringToArray[int8]("[-129]"); err == nil { + t.Errorf("int8 parse of out-of-range (low) should error") + } + // bf16 / f16: small integers round-trip exactly. + bf, err := StringToArray[BF16]("[1, 2, 3]") + if err != nil || bf[0].ToFloat32() != 1 || bf[2].ToFloat32() != 3 { + t.Errorf("bf16 parse: %v %v", bf, err) + } + h, err := StringToArray[Float16]("[0.5, -2, 4]") + if err != nil || h[0].ToFloat32() != 0.5 || h[1].ToFloat32() != -2 { + t.Errorf("f16 parse: %v %v", h, err) + } + // ArrayToString round-trips the narrow types. + if s := ArrayToString[int8]([]int8{1, -2, 127}); s != "[1, -2, 127]" { + t.Errorf("int8 ArrayToString = %q", s) + } + if s := ArrayToString[BF16](Float32ToBF16Slice([]float32{1, 2, 3})); s != "[1, 2, 3]" { + t.Errorf("bf16 ArrayToString = %q", s) + } +} + +func TestFloat16SliceRoundTrip(t *testing.T) { + src := []float32{1.0, 2.0, 0.5, -3.0, 0.0} + f16 := Float32ToFloat16Slice(src) + back := Float16ToFloat32Slice(f16) + for i := range src { + if back[i] != src[i] { + t.Errorf("f16 slice round-trip[%d]: %v != %v", i, back[i], src[i]) + } + } + bf := Float32ToBF16Slice(src) + bback := BF16ToFloat32Slice(bf) + for i := range src { + if math.Abs(float64(bback[i]-src[i])) > math.Abs(float64(src[i]))*0.01+1e-6 { + t.Errorf("bf16 slice round-trip[%d]: %v vs %v", i, bback[i], src[i]) + } + } + i8 := Float32ToInt8Slice([]float32{1.2, 130.0, -200.0}) + if i8[0] != 1 || i8[1] != 127 || i8[2] != -128 { + t.Errorf("int8 slice = %v", i8) + } +} diff --git a/pkg/container/types/types.go b/pkg/container/types/types.go index e31cb15ef0b3e..750952167033b 100644 --- a/pkg/container/types/types.go +++ b/pkg/container/types/types.go @@ -98,10 +98,47 @@ const ( // Array/Vector family T_array_float32 T = 224 // In SQL , it is vecf32 T_array_float64 T = 225 // In SQL , it is vecf64 + T_array_bf16 T = 226 // In SQL , it is vecbf16 (bfloat16) + T_array_float16 T = 227 // In SQL , it is vecf16 (IEEE fp16/half) + T_array_int8 T = 228 // In SQL , it is vecint8 (int8) + T_array_uint8 T = 229 // In SQL , it is vecuint8 (uint8) //note: max value of uint8 is 255 ) +// Canonical lowercase SQL type names for the array/vector types — the spelling +// used in DDL and CAST (`col vecf32(4)`, `cast(x as vecint8(4))`) and recognized +// by the parser keyword table. T.String() returns the uppercase display form; +// these are the single source of truth for the lowercase SQL spelling. +const ( + ArrayFloat32SQLName = "vecf32" + ArrayFloat64SQLName = "vecf64" + ArrayBF16SQLName = "vecbf16" + ArrayFloat16SQLName = "vecf16" + ArrayInt8SQLName = "vecint8" + ArrayUint8SQLName = "vecuint8" +) + +// ArraySQLName returns the lowercase SQL type name for an array element type +// (e.g. T_array_float32 -> "vecf32"), or "" if t is not an array/vector type. +func (t T) ArraySQLName() string { + switch t { + case T_array_float32: + return ArrayFloat32SQLName + case T_array_float64: + return ArrayFloat64SQLName + case T_array_bf16: + return ArrayBF16SQLName + case T_array_float16: + return ArrayFloat16SQLName + case T_array_int8: + return ArrayInt8SQLName + case T_array_uint8: + return ArrayUint8SQLName + } + return "" +} + const ( TxnTsSize = 12 SegmentidSize = 16 @@ -365,6 +402,16 @@ type RealNumbers interface { constraints.Float } +// ArrayElement is the set of element types that can back a vector column. +// It is used ONLY by the storage / serialization / accessor / display / +// cast-plumbing layer (pure byte reinterpretation + formatting). All math +// kernels stay on RealNumbers; narrow types reach them via a float32 bridge. +// Do NOT widen RealNumbers to include these — int8 is not a float and +// BF16/Float16 have no native arithmetic. +type ArrayElement interface { + ~float32 | ~float64 | BF16 | Float16 | int8 | uint8 +} + type FixedSizeTExceptStrType interface { bool | OrderedT | Decimal | TS | Rowid | Uuid | Blockid } @@ -426,6 +473,10 @@ var Types = map[string]T{ "array float32": T_array_float32, "array float64": T_array_float64, + "array bf16": T_array_bf16, + "array float16": T_array_float16, + "array int8": T_array_int8, + "array uint8": T_array_uint8, } func New(oid T, width, scale int32) Type { @@ -571,6 +622,14 @@ func (t Type) DescString() string { return fmt.Sprintf("VECF32(%d)", t.Width) case T_array_float64: return fmt.Sprintf("VECF64(%d)", t.Width) + case T_array_bf16: + return fmt.Sprintf("VECBF16(%d)", t.Width) + case T_array_float16: + return fmt.Sprintf("VECF16(%d)", t.Width) + case T_array_int8: + return fmt.Sprintf("VECINT8(%d)", t.Width) + case T_array_uint8: + return fmt.Sprintf("VECUINT8(%d)", t.Width) } return t.Oid.String() } @@ -581,6 +640,14 @@ func (t Type) GetArrayElementSize() int { return 4 case T_array_float64: return 8 + case T_array_bf16: + return 2 + case T_array_float16: + return 2 + case T_array_int8: + return 1 + case T_array_uint8: + return 1 } panic(moerr.NewInternalErrorNoCtx(fmt.Sprintf("unknown array type %d", t))) } @@ -653,7 +720,7 @@ func (t T) ToType() Type { case T_varchar: typ.Size = VarlenaSize typ.Width = MaxVarcharLen - case T_array_float32, T_array_float64: + case T_array_float32, T_array_float64, T_array_bf16, T_array_float16, T_array_int8, T_array_uint8: typ.Size = VarlenaSize typ.Width = MaxArrayDimension case T_binary: @@ -761,6 +828,14 @@ func (t T) String() string { return "VECF32" case T_array_float64: return "VECF64" + case T_array_bf16: + return "VECBF16" + case T_array_float16: + return "VECF16" + case T_array_int8: + return "VECINT8" + case T_array_uint8: + return "VECUINT8" case T_enum: return "ENUM" } @@ -846,6 +921,14 @@ func (t T) OidString() string { return "T_array_float32" case T_array_float64: return "T_array_float64" + case T_array_bf16: + return "T_array_bf16" + case T_array_float16: + return "T_array_float16" + case T_array_int8: + return "T_array_int8" + case T_array_uint8: + return "T_array_uint8" } return "unknown_type" } @@ -879,7 +962,7 @@ func (t T) TypeLen() int { return 4 case T_float64: return 8 - case T_char, T_varchar, T_json, T_blob, T_text, T_binary, T_varbinary, T_array_float32, T_array_float64, T_datalink, T_geometry, T_geometry32: + case T_char, T_varchar, T_json, T_blob, T_text, T_binary, T_varbinary, T_array_float32, T_array_float64, T_array_bf16, T_array_float16, T_array_int8, T_array_uint8, T_datalink, T_geometry, T_geometry32: return VarlenaSize case T_decimal64: return 8 @@ -934,7 +1017,7 @@ func (t T) FixedLength() int { return RowidSize case T_Blockid: return BlockidSize - case T_char, T_varchar, T_blob, T_json, T_text, T_binary, T_varbinary, T_array_float32, T_array_float64, T_datalink, T_geometry, T_geometry32: + case T_char, T_varchar, T_blob, T_json, T_text, T_binary, T_varbinary, T_array_float32, T_array_float64, T_array_bf16, T_array_float16, T_array_int8, T_array_uint8, T_datalink, T_geometry, T_geometry32: return -24 case T_enum: return 2 @@ -1015,7 +1098,8 @@ func (t T) IsDateRelate() bool { } func (t T) IsArrayRelate() bool { - if t == T_array_float32 || t == T_array_float64 { + if t == T_array_float32 || t == T_array_float64 || + t == T_array_bf16 || t == T_array_float16 || t == T_array_int8 || t == T_array_uint8 { return true } return false diff --git a/pkg/container/types/types_test.go b/pkg/container/types/types_test.go index 5f3b4650738e9..b1d0d31095507 100644 --- a/pkg/container/types/types_test.go +++ b/pkg/container/types/types_test.go @@ -423,3 +423,21 @@ func BenchmarkTypesCompare(b *testing.B) { } }) } + +func TestArraySQLName(t *testing.T) { + // every array/vector type maps to its lowercase SQL name. + arrayTypes := []T{T_array_float32, T_array_float64, T_array_bf16, T_array_float16, T_array_int8, T_array_uint8} + wantNames := []string{"vecf32", "vecf64", "vecbf16", "vecf16", "vecint8", "vecuint8"} + for i, at := range arrayTypes { + require.Equal(t, wantNames[i], at.ArraySQLName()) + } + + // constants stay in sync with the method (and with the literal spellings). + require.Equal(t, ArrayFloat32SQLName, T_array_float32.ArraySQLName()) + require.Equal(t, ArrayInt8SQLName, T_array_int8.ArraySQLName()) + require.Equal(t, "vecbf16", ArrayBF16SQLName) + + // non-array types -> "". + require.Equal(t, "", T_int32.ArraySQLName()) + require.Equal(t, "", T_varchar.ArraySQLName()) +} diff --git a/pkg/container/vector/tools.go b/pkg/container/vector/tools.go index 4bb9254a90c97..8920131cf729e 100644 --- a/pkg/container/vector/tools.go +++ b/pkg/container/vector/tools.go @@ -122,7 +122,7 @@ func InefficientMustStrCol(v *Vector) []string { } // MustArrayCol Converts Vector<[]T> to [][]T -func MustArrayCol[T types.RealNumbers](v *Vector) [][]T { +func MustArrayCol[T types.ArrayElement](v *Vector) [][]T { if v.GetType().Oid == types.T_any || len(v.data) == 0 { return nil } diff --git a/pkg/container/vector/utils.go b/pkg/container/vector/utils.go index d553e5070b7a6..2d6f45420f4f1 100644 --- a/pkg/container/vector/utils.go +++ b/pkg/container/vector/utils.go @@ -331,6 +331,33 @@ func ArrayGetMinMax[T types.RealNumbers](vec *Vector) (minv, maxv []T) { return } +// ArrayElementGetMinMax mirrors ArrayGetMinMax for the narrow vector element +// types (bf16/f16/int8). Ordering goes through the float32 bridge via +// ArrayElementCompare so that bf16/f16 sign bits don't corrupt the comparison. +// The returned min/max are original stored values (no reconversion). +func ArrayElementGetMinMax[T types.ArrayElement](vec *Vector) (minv, maxv []T) { + col, area := MustVarlenaRawData(vec) + first := true + for i, j := 0, vec.Length(); i < j; i++ { + if vec.HasNull() && vec.IsNull(uint64(i)) { + continue + } + val := types.GetArray[T](&col[i], area) + if first { + minv, maxv = val, val + first = false + continue + } + if types.ArrayElementCompare[T](minv, val) > 0 { + minv = val + } + if types.ArrayElementCompare[T](maxv, val) < 0 { + maxv = val + } + } + return +} + func typeCompatible[T any](typ types.Type) bool { var t T switch (any)(t).(type) { diff --git a/pkg/container/vector/vector.go b/pkg/container/vector/vector.go index a1013b626debd..ea119573ae912 100644 --- a/pkg/container/vector/vector.go +++ b/pkg/container/vector/vector.go @@ -19,6 +19,7 @@ import ( "fmt" "io" "slices" + "sort" "time" "unsafe" @@ -378,7 +379,7 @@ func (v *Vector) GetStringAt(i int) string { } // GetArrayAt Returns []T at the specific index of the vector -func GetArrayAt[T types.RealNumbers](v *Vector, i int) []T { +func GetArrayAt[T types.ArrayElement](v *Vector, i int) []T { if v.IsConst() { i = 0 } @@ -387,7 +388,7 @@ func GetArrayAt[T types.RealNumbers](v *Vector, i int) []T { return types.GetArray[T](&bs[i], v.area) } -func GetArrayAt2[T types.RealNumbers](v *Vector, bs []types.Varlena, i int) []T { +func GetArrayAt2[T types.ArrayElement](v *Vector, bs []types.Varlena, i int) []T { if v.IsConst() { i = 0 } @@ -450,7 +451,7 @@ func GetAny(vec *Vector, i int, deepCopy bool) any { case types.T_Blockid: return GetFixedAtNoTypeCheck[types.Blockid](vec, i) case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_json, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink, types.T_geometry, types.T_geometry32: + types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32: ret := vec.GetBytesAt(i) if deepCopy { copied := make([]byte, len(ret)) @@ -544,7 +545,7 @@ func NewConstBytes(typ types.Type, val []byte, length int, mp *mpool.MPool) (vec } // NewConstArray Creates a Const_Array Vector -func NewConstArray[T types.RealNumbers](typ types.Type, val []T, length int, mp *mpool.MPool) (vec *Vector, err error) { +func NewConstArray[T types.ArrayElement](typ types.Type, val []T, length int, mp *mpool.MPool) (vec *Vector, err error) { vec = NewVecFromReuse() vec.typ = typ vec.class = CONSTANT @@ -1049,7 +1050,7 @@ func (v *Vector) Shrink(sels []int64, negate bool) { case types.T_float64: shrinkFixed[float64](v, sels, negate) case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_json, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink, types.T_geometry, types.T_geometry32: + types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32: // XXX shrink varlena, but did not shrink area. For our vector, this // may well be the right thing. If want to shrink area as well, we // have to copy each varlena value and swizzle pointer. @@ -1121,7 +1122,7 @@ func (v *Vector) ShrinkByMask(sels *bitmap.Bitmap, negate bool, offset uint64) { case types.T_float64: shrinkFixedByMask[float64](v, sels, negate, offset) case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_json, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink, types.T_geometry, types.T_geometry32: + types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32: // XXX shrink varlena, but did not shrink area. For our vector, this // may well be the right thing. If want to shrink area as well, we // have to copy each varlena value and swizzle pointer. @@ -1189,7 +1190,7 @@ func (v *Vector) Shuffle(sels []int64, mp *mpool.MPool) (err error) { case types.T_float64: err = shuffleFixedNoTypeCheck[float64](v, sels, mp) case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_json, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink, types.T_geometry, types.T_geometry32: + types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32: err = shuffleFixedNoTypeCheck[types.Varlena](v, sels, mp) case types.T_date: err = shuffleFixedNoTypeCheck[types.Date](v, sels, mp) @@ -1263,7 +1264,7 @@ func (v *Vector) ShuffleWithBuf(sels []int64, mp *mpool.MPool, buf *[]byte) (err case types.T_float64: err = shuffleFixedNoTypeCheckWithBuf[float64](v, sels, buf) case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_json, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink, types.T_geometry, types.T_geometry32: + types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32: err = shuffleFixedNoTypeCheckWithBuf[types.Varlena](v, sels, buf) case types.T_date: err = shuffleFixedNoTypeCheckWithBuf[types.Date](v, sels, buf) @@ -2066,7 +2067,7 @@ func GetUnionAllFunction(typ types.Type, mp *mpool.MPool) func(v, w *Vector) err } case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_json, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink, types.T_geometry, types.T_geometry32: + types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32: return func(v, w *Vector) error { if w.IsConstNull() { if err := appendMultiFixed(v, 0, true, w.length, mp); err != nil { @@ -2425,7 +2426,7 @@ func GetConstSetFunction(typ types.Type, mp *mpool.MPool) func(v, w *Vector, sel return SetConstFixed(v, ws[sel], length, mp) } case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, - types.T_json, types.T_blob, types.T_text, types.T_array_float32, types.T_array_float64, types.T_datalink, types.T_geometry, types.T_geometry32: + types.T_json, types.T_blob, types.T_text, types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32: return func(v, w *Vector, sel int64, length int) error { if w.IsConstNull() || w.nsp.Contains(uint64(sel)) { return SetConstNull(v, length, mp) @@ -3132,6 +3133,46 @@ func (v *Vector) String() string { return fmt.Sprintf("%v-%s", str, v.nsp.GetBitmap().String()) } return fmt.Sprintf("%v-%s", str, v.nsp.GetBitmap().String()) + case types.T_array_bf16: + col := MustArrayCol[types.BF16](v) + if len(col) == 1 { + if nulls.Contains(&v.nsp, 0) { + return "null" + } + return types.ArrayToString[types.BF16](col[0]) + } + str := types.ArraysToString[types.BF16](col, types.DefaultArraysToStringSep) + return fmt.Sprintf("%v-%s", str, v.nsp.GetBitmap().String()) + case types.T_array_float16: + col := MustArrayCol[types.Float16](v) + if len(col) == 1 { + if nulls.Contains(&v.nsp, 0) { + return "null" + } + return types.ArrayToString[types.Float16](col[0]) + } + str := types.ArraysToString[types.Float16](col, types.DefaultArraysToStringSep) + return fmt.Sprintf("%v-%s", str, v.nsp.GetBitmap().String()) + case types.T_array_uint8: + col := MustArrayCol[uint8](v) + if len(col) == 1 { + if nulls.Contains(&v.nsp, 0) { + return "null" + } + return types.ArrayToString[uint8](col[0]) + } + str := types.ArraysToString[uint8](col, types.DefaultArraysToStringSep) + return fmt.Sprintf("%v-%s", str, v.nsp.GetBitmap().String()) + case types.T_array_int8: + col := MustArrayCol[int8](v) + if len(col) == 1 { + if nulls.Contains(&v.nsp, 0) { + return "null" + } + return types.ArrayToString[int8](col[0]) + } + str := types.ArraysToString[int8](col, types.DefaultArraysToStringSep) + return fmt.Sprintf("%v-%s", str, v.nsp.GetBitmap().String()) default: panic("vec to string unknown types.") } @@ -3218,7 +3259,7 @@ func implDecimalRowToString[T types.DecimalWithFormat](v *Vector, idx int) strin } } -func implArrayRowToString[T types.RealNumbers](v *Vector, idx int) string { +func implArrayRowToString[T types.ArrayElement](v *Vector, idx int) string { if v.IsConstNull() { return "null" } @@ -3308,6 +3349,14 @@ func (v *Vector) RowToString(idx int) string { return implArrayRowToString[float32](v, idx) case types.T_array_float64: return implArrayRowToString[float64](v, idx) + case types.T_array_bf16: + return implArrayRowToString[types.BF16](v, idx) + case types.T_array_float16: + return implArrayRowToString[types.Float16](v, idx) + case types.T_array_int8: + return implArrayRowToString[int8](v, idx) + case types.T_array_uint8: + return implArrayRowToString[uint8](v, idx) default: panic("vec to string unknown types.") } @@ -3361,7 +3410,7 @@ func SetConstByteJson(vec *Vector, bj bytejson.ByteJson, length int, mp *mpool.M } // SetConstArray set current vector as Constant_Array vector of given length. -func SetConstArray[T types.RealNumbers](vec *Vector, val []T, length int, mp *mpool.MPool) error { +func SetConstArray[T types.ArrayElement](vec *Vector, val []T, length int, mp *mpool.MPool) error { var err error if err := extend(vec, 1, mp); err != nil { @@ -3444,7 +3493,7 @@ func AppendAny(vec *Vector, val any, isNull bool, mp *mpool.MPool) error { case types.T_Blockid: return appendOneFixed(vec, val.(types.Blockid), false, mp) case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_json, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink, types.T_geometry, types.T_geometry32: + types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32: return appendOneBytes(vec, val.([]byte), false, mp) } return nil @@ -3485,7 +3534,7 @@ func AppendByteJson(vec *Vector, bj bytejson.ByteJson, isNull bool, mp *mpool.MP } // AppendArray mainly used in tests -func AppendArray[T types.RealNumbers](vec *Vector, val []T, isNull bool, mp *mpool.MPool) error { +func AppendArray[T types.ArrayElement](vec *Vector, val []T, isNull bool, mp *mpool.MPool) error { if vec.IsConst() { panic(moerr.NewInternalErrorNoCtx("append to const vector")) } @@ -3555,7 +3604,7 @@ func AppendStringList(vec *Vector, ws []string, isNulls []bool, mp *mpool.MPool) } // AppendArrayList mainly used in unit tests -func AppendArrayList[T types.RealNumbers](vec *Vector, ws [][]T, isNulls []bool, mp *mpool.MPool) error { +func AppendArrayList[T types.ArrayElement](vec *Vector, ws [][]T, isNulls []bool, mp *mpool.MPool) error { if vec.IsConst() { panic(moerr.NewInternalErrorNoCtx("append to const vector")) } @@ -3619,7 +3668,7 @@ func appendOneByteJson(vec *Vector, bj bytejson.ByteJson, isNull bool, mp *mpool } // appendOneArray mainly used for unit tests -func appendOneArray[T types.RealNumbers](vec *Vector, val []T, isNull bool, mp *mpool.MPool) error { +func appendOneArray[T types.ArrayElement](vec *Vector, val []T, isNull bool, mp *mpool.MPool) error { var err error var va types.Varlena @@ -3737,7 +3786,7 @@ func appendStringList(vec *Vector, vals []string, isNulls []bool, mp *mpool.MPoo } // appendArrayList mainly used for unit tests -func appendArrayList[T types.RealNumbers](vec *Vector, vals [][]T, isNulls []bool, mp *mpool.MPool) error { +func appendArrayList[T types.ArrayElement](vec *Vector, vals [][]T, isNulls []bool, mp *mpool.MPool) error { var err error if err = extend(vec, len(vals), mp); err != nil { @@ -4436,6 +4485,22 @@ func (v *Vector) GetMinMaxValue() (ok bool, minv, maxv []byte) { _minv, _maxv := ArrayGetMinMax[float64](v) minv = types.ArrayToBytes[float64](_minv) maxv = types.ArrayToBytes[float64](_maxv) + case types.T_array_bf16: + _minv, _maxv := ArrayElementGetMinMax[types.BF16](v) + minv = types.ArrayToBytes[types.BF16](_minv) + maxv = types.ArrayToBytes[types.BF16](_maxv) + case types.T_array_float16: + _minv, _maxv := ArrayElementGetMinMax[types.Float16](v) + minv = types.ArrayToBytes[types.Float16](_minv) + maxv = types.ArrayToBytes[types.Float16](_maxv) + case types.T_array_int8: + _minv, _maxv := ArrayElementGetMinMax[int8](v) + minv = types.ArrayToBytes[int8](_minv) + maxv = types.ArrayToBytes[int8](_maxv) + case types.T_array_uint8: + _minv, _maxv := ArrayElementGetMinMax[uint8](v) + minv = types.ArrayToBytes[uint8](_minv) + maxv = types.ArrayToBytes[uint8](_maxv) default: panic(fmt.Sprintf("unsupported type %s", v.GetType().String())) } @@ -4805,6 +4870,14 @@ func (v *Vector) InplaceSortAndCompact() { cleanDataNotResetArea() appendList(v, newCol, nil, nil) } + case types.T_array_bf16: + inplaceSortAndCompactArrayElement[types.BF16](v, cleanDataNotResetArea) + case types.T_array_float16: + inplaceSortAndCompactArrayElement[types.Float16](v, cleanDataNotResetArea) + case types.T_array_int8: + inplaceSortAndCompactArrayElement[int8](v, cleanDataNotResetArea) + case types.T_array_uint8: + inplaceSortAndCompactArrayElement[uint8](v, cleanDataNotResetArea) default: return } @@ -4813,6 +4886,28 @@ func (v *Vector) InplaceSortAndCompact() { v.SetSorted(true) } +// inplaceSortAndCompactArrayElement sorts+dedups a narrow-typed vector using the +// float32-bridged comparator (so bf16/f16 order by value, not by raw bits). +func inplaceSortAndCompactArrayElement[T types.ArrayElement](v *Vector, cleanDataNotResetArea func()) { + col, area := MustVarlenaRawData(v) + sort.Slice(col, func(i, j int) bool { + return types.ArrayElementCompare[T]( + types.GetArray[T](&col[i], area), + types.GetArray[T](&col[j], area), + ) < 0 + }) + newCol := slices.CompactFunc(col, func(a, b types.Varlena) bool { + return types.ArrayElementCompare[T]( + types.GetArray[T](&a, area), + types.GetArray[T](&b, area), + ) == 0 + }) + if len(newCol) != len(col) { + cleanDataNotResetArea() + appendList(v, newCol, nil, nil) + } +} + func (v *Vector) InplaceSort() { switch v.GetType().Oid { case types.T_bool: @@ -4983,9 +5078,29 @@ func (v *Vector) InplaceSort() { types.GetArray[float64](&b, area), ) }) + case types.T_array_bf16: + sortArrayElement[types.BF16](v) + case types.T_array_float16: + sortArrayElement[types.Float16](v) + case types.T_array_int8: + sortArrayElement[int8](v) + case types.T_array_uint8: + sortArrayElement[uint8](v) } } +// sortArrayElement sorts a narrow-typed vector in place using the +// float32-bridged comparator. +func sortArrayElement[T types.ArrayElement](v *Vector) { + col, area := MustVarlenaRawData(v) + sort.Slice(col, func(i, j int) bool { + return types.ArrayElementCompare[T]( + types.GetArray[T](&col[i], area), + types.GetArray[T](&col[j], area), + ) < 0 + }) +} + func BuildVarlenaInline(v1, v2 *types.Varlena) { // use three dword operation to improve performance p1 := v1.UnsafePtr() @@ -5082,7 +5197,7 @@ func BuildVarlenaFromByteJson(vec *Vector, v *types.Varlena, bj bytejson.ByteJso } // BuildVarlenaFromArray convert array to Varlena so that it can be stored in the vector -func BuildVarlenaFromArray[T types.RealNumbers](vec *Vector, v *types.Varlena, array *[]T, m *mpool.MPool) error { +func BuildVarlenaFromArray[T types.ArrayElement](vec *Vector, v *types.Varlena, array *[]T, m *mpool.MPool) error { _bs := types.ArrayToBytes[T](*array) bs := &_bs vlen := len(*bs) diff --git a/pkg/container/vector/vector_narrow_test.go b/pkg/container/vector/vector_narrow_test.go new file mode 100644 index 0000000000000..927041e7dc392 --- /dev/null +++ b/pkg/container/vector/vector_narrow_test.go @@ -0,0 +1,99 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vector + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// exerciseNarrowArray drives the generic narrow-vector (bf16/f16/int8/uint8) array +// functions added for quantized vector columns: Append*, GetArrayAt, NewConstArray, +// SetConstArray, and the String()/RowToString() narrow branches. +func exerciseNarrowArray[T types.ArrayElement](t *testing.T, oid types.T, a, b []T) { + mp := mpool.MustNewZero() + dim := int32(len(a)) + + // AppendArray (value + null) then AppendArrayList + vec := NewVec(types.New(oid, dim, 0)) + require.NoError(t, AppendArray[T](vec, a, false, mp)) + require.NoError(t, AppendArray[T](vec, nil, true, mp)) // null row + require.NoError(t, AppendArrayList[T](vec, [][]T{a, b}, nil, mp)) + require.Equal(t, a, GetArrayAt[T](vec, 0)) + require.Equal(t, b, GetArrayAt[T](vec, 3)) + _ = vec.String() // String() narrow branch (multi-row + null bitmap) + _ = vec.RowToString(0) // RowToString -> implArrayRowToString narrow branch + _ = vec.RowToString(2) + vec.Free(mp) + + // NewConstArray + GetArrayAt + single-row String() + cv, err := NewConstArray[T](types.New(oid, dim, 0), a, 2, mp) + require.NoError(t, err) + require.Equal(t, a, GetArrayAt[T](cv, 0)) + _ = cv.String() + _ = cv.RowToString(0) + cv.Free(mp) + + // SetConstArray + sv := NewVec(types.New(oid, dim, 0)) + require.NoError(t, SetConstArray[T](sv, b, 3, mp)) + require.Equal(t, b, GetArrayAt[T](sv, 0)) + sv.Free(mp) + + // single-row String()/RowToString — the len(col)==1 narrow branch + one := NewVec(types.New(oid, dim, 0)) + require.NoError(t, AppendArray[T](one, a, false, mp)) + _ = one.String() + _ = one.RowToString(0) + one.Free(mp) + + // InplaceSort narrow branch (unsorted input with a duplicate) + srt := NewVec(types.New(oid, dim, 0)) + require.NoError(t, AppendArrayList[T](srt, [][]T{b, a, a}, nil, mp)) + srt.InplaceSort() + srt.Free(mp) + + // InplaceSortAndCompact narrow branch (sort + dedup) + cmp := NewVec(types.New(oid, dim, 0)) + require.NoError(t, AppendArrayList[T](cmp, [][]T{b, a, a}, nil, mp)) + cmp.InplaceSortAndCompact() + cmp.Free(mp) + + // GetMinMaxValue narrow branch + mm := NewVec(types.New(oid, dim, 0)) + require.NoError(t, AppendArrayList[T](mm, [][]T{a, b}, nil, mp)) + _, _, _ = mm.GetMinMaxValue() + mm.Free(mp) +} + +func TestNarrowArrayVectorOps(t *testing.T) { + t.Run("bf16", func(t *testing.T) { + exerciseNarrowArray(t, types.T_array_bf16, + types.Float32ToBF16Slice([]float32{1, 2, 3}), types.Float32ToBF16Slice([]float32{4, 5, 6})) + }) + t.Run("f16", func(t *testing.T) { + exerciseNarrowArray(t, types.T_array_float16, + types.Float32ToFloat16Slice([]float32{1, 2, 3}), types.Float32ToFloat16Slice([]float32{4, 5, 6})) + }) + t.Run("int8", func(t *testing.T) { + exerciseNarrowArray(t, types.T_array_int8, []int8{1, 2, 3}, []int8{4, 5, 6}) + }) + t.Run("uint8", func(t *testing.T) { + exerciseNarrowArray(t, types.T_array_uint8, []uint8{1, 2, 3}, []uint8{4, 5, 6}) + }) +} diff --git a/pkg/cuvs/brute_force.go b/pkg/cuvs/brute_force.go index 2b1c13a2bba97..3d35db8c39d39 100644 --- a/pkg/cuvs/brute_force.go +++ b/pkg/cuvs/brute_force.go @@ -28,18 +28,24 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" ) -// GpuBruteForce represents the C++ gpu_brute_force_t object -type GpuBruteForce[T VectorType] struct { +// GpuBruteForce represents the C++ gpu_brute_force_t object. +// B is the base/query element type, Q is the storage element type. The native +// dataset/chunks are storage-typed ([]Q); the quantize search entry points take +// base-typed ([]B) queries and quantize B->Q inside cuVS. cuVS brute force only +// supports (B,Q) combos (float,float),(float,half),(half,half); int8/uint8 +// storage is not supported and throws at runtime. +type GpuBruteForce[B, Q VectorType] struct { cIndex C.gpu_brute_force_c } // NewGpuBruteForce creates a new GpuBruteForce instance -func NewGpuBruteForce[T VectorType](dataset []T, countVectors uint64, dimension uint32, metric DistanceType, nthread uint32, deviceID int) (*GpuBruteForce[T], error) { +func NewGpuBruteForce[B, Q VectorType](dataset []Q, countVectors uint64, dimension uint32, metric DistanceType, nthread uint32, deviceID int) (*GpuBruteForce[B, Q], error) { if len(dataset) == 0 || countVectors == 0 || dimension == 0 { return nil, moerr.NewInternalErrorNoCtx("dataset, count_vectors, and dimension cannot be zero") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cIndex := C.gpu_brute_force_new( unsafe.Pointer(&dataset[0]), @@ -48,6 +54,7 @@ func NewGpuBruteForce[T VectorType](dataset []T, countVectors uint64, dimension C.distance_type_t(metric), C.uint32_t(nthread), C.int(deviceID), + C.quantization_t(btype), C.quantization_t(qtype), nil, unsafe.Pointer(&errmsg), @@ -63,14 +70,15 @@ func NewGpuBruteForce[T VectorType](dataset []T, countVectors uint64, dimension if cIndex == nil { return nil, moerr.NewInternalErrorNoCtx("failed to create GpuBruteForce") } - return &GpuBruteForce[T]{cIndex: cIndex}, nil + return &GpuBruteForce[B, Q]{cIndex: cIndex}, nil } // NewGpuBruteForceEmpty creates a new GpuBruteForce instance with pre-allocated buffer but no data yet. -func NewGpuBruteForceEmpty[T VectorType](totalCount uint64, dimension uint32, metric DistanceType, - nthread uint32, deviceID int) (*GpuBruteForce[T], error) { +func NewGpuBruteForceEmpty[B, Q VectorType](totalCount uint64, dimension uint32, metric DistanceType, + nthread uint32, deviceID int) (*GpuBruteForce[B, Q], error) { - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cBruteForce := C.gpu_brute_force_new_empty( @@ -79,6 +87,7 @@ func NewGpuBruteForceEmpty[T VectorType](totalCount uint64, dimension uint32, me C.distance_type_t(metric), C.uint32_t(nthread), C.int(deviceID), + C.quantization_t(btype), C.quantization_t(qtype), nil, unsafe.Pointer(&errmsg), @@ -94,11 +103,11 @@ func NewGpuBruteForceEmpty[T VectorType](totalCount uint64, dimension uint32, me return nil, moerr.NewInternalErrorNoCtx("failed to create GpuBruteForce") } - return &GpuBruteForce[T]{cIndex: cBruteForce}, nil + return &GpuBruteForce[B, Q]{cIndex: cBruteForce}, nil } // Start initializes the worker and resources -func (gb *GpuBruteForce[T]) Start() error { +func (gb *GpuBruteForce[B, Q]) Start() error { if gb.cIndex == nil { return moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -113,7 +122,7 @@ func (gb *GpuBruteForce[T]) Start() error { } // Build triggers the dataset loading to GPU -func (gb *GpuBruteForce[T]) Build() error { +func (gb *GpuBruteForce[B, Q]) Build() error { if gb.cIndex == nil { return moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -131,7 +140,7 @@ func (gb *GpuBruteForce[T]) Build() error { // If ids is non-nil it must have length chunkCount and supplies external int64 // ids (e.g. pkids) that the brute-force search will return in `neighbors` // instead of the internal 0..N-1 row index. -func (gb *GpuBruteForce[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) error { +func (gb *GpuBruteForce[B, Q]) AddChunk(chunk []Q, chunkCount uint64, ids []int64) error { if gb.cIndex == nil { return moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -165,9 +174,10 @@ func (gb *GpuBruteForce[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) return nil } -// AddChunkFloat adds a chunk of float32 data, performing on-the-fly conversion if needed. -// See AddChunk for the meaning of ids. -func (gb *GpuBruteForce[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []int64) error { +// AddChunkQuantize adds a chunk of base-typed (B) vectors, converting them to the +// storage type Q on the C++ side (native store when B==Q, f32->f16 cast, or learned +// SQ for 1-byte Q) — the add counterpart of SearchQuantize. See AddChunk for ids. +func (gb *GpuBruteForce[B, Q]) AddChunkQuantize(chunk []B, chunkCount uint64, ids []int64) error { if gb.cIndex == nil { return moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -183,9 +193,9 @@ func (gb *GpuBruteForce[T]) AddChunkFloat(chunk []float32, chunkCount uint64, id if ids != nil { idsPtr = (*C.int64_t)(&ids[0]) } - C.gpu_brute_force_add_chunk_float( + C.gpu_brute_force_add_chunk_quantize( gb.cIndex, - (*C.float)(&chunk[0]), + unsafe.Pointer(&chunk[0]), C.uint64_t(chunkCount), idsPtr, unsafe.Pointer(&errmsg), @@ -203,7 +213,7 @@ func (gb *GpuBruteForce[T]) AddChunkFloat(chunk []float32, chunkCount uint64, id // SearchInto performs a search and writes results into caller-provided slices (no internal allocation). // neighbors and distances must be pre-allocated to at least numQueries*limit elements. -func (gb *GpuBruteForce[T]) SearchInto(queries []T, numQueries uint64, queryDimension uint32, limit uint32, neighbors []int64, distances []float32) error { +func (gb *GpuBruteForce[B, Q]) SearchInto(queries []Q, numQueries uint64, queryDimension uint32, limit uint32, neighbors []int64, distances []float32) error { if gb.cIndex == nil { return moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -239,7 +249,7 @@ func (gb *GpuBruteForce[T]) SearchInto(queries []T, numQueries uint64, queryDime } // Search performs a search operation -func (gb *GpuBruteForce[T]) Search(queries []T, numQueries uint64, queryDimension uint32, limit uint32) ([]int64, []float32, error) { +func (gb *GpuBruteForce[B, Q]) Search(queries []Q, numQueries uint64, queryDimension uint32, limit uint32) ([]int64, []float32, error) { neighbors := make([]int64, numQueries*uint64(limit)) distances := make([]float32, numQueries*uint64(limit)) if err := gb.SearchInto(queries, numQueries, queryDimension, limit, neighbors, distances); err != nil { @@ -248,9 +258,10 @@ func (gb *GpuBruteForce[T]) Search(queries []T, numQueries uint64, queryDimensio return neighbors, distances, nil } -// SearchFloatInto performs a search with float32 queries and writes results into caller-provided slices. +// SearchQuantizeInto performs a search with base-typed (B) queries and writes +// results into caller-provided slices. cuVS quantizes B -> storage Q internally. // neighbors and distances must be pre-allocated to at least numQueries*limit elements. -func (gb *GpuBruteForce[T]) SearchFloatInto(queries []float32, numQueries uint64, queryDimension uint32, limit uint32, neighbors []int64, distances []float32) error { +func (gb *GpuBruteForce[B, Q]) SearchQuantizeInto(queries []B, numQueries uint64, queryDimension uint32, limit uint32, neighbors []int64, distances []float32) error { if gb.cIndex == nil { return moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -259,9 +270,9 @@ func (gb *GpuBruteForce[T]) SearchFloatInto(queries []float32, numQueries uint64 } var errmsg *C.char - cResult := C.gpu_brute_force_search_float( + cResult := C.gpu_brute_force_search_quantize( gb.cIndex, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(queryDimension), C.uint32_t(limit), @@ -285,18 +296,19 @@ func (gb *GpuBruteForce[T]) SearchFloatInto(queries []float32, numQueries uint64 return nil } -// SearchFloat performs a search operation with float32 queries -func (gb *GpuBruteForce[T]) SearchFloat(queries []float32, numQueries uint64, queryDimension uint32, limit uint32) ([]int64, []float32, error) { +// SearchQuantize performs a search operation with base-typed (B) queries; +// cuVS quantizes B -> storage Q internally. +func (gb *GpuBruteForce[B, Q]) SearchQuantize(queries []B, numQueries uint64, queryDimension uint32, limit uint32) ([]int64, []float32, error) { neighbors := make([]int64, numQueries*uint64(limit)) distances := make([]float32, numQueries*uint64(limit)) - if err := gb.SearchFloatInto(queries, numQueries, queryDimension, limit, neighbors, distances); err != nil { + if err := gb.SearchQuantizeInto(queries, numQueries, queryDimension, limit, neighbors, distances); err != nil { return nil, nil, err } return neighbors, distances, nil } // SearchAsync performs a K-Nearest Neighbor search asynchronously. -func (gb *GpuBruteForce[T]) SearchAsync(queries []T, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { +func (gb *GpuBruteForce[B, Q]) SearchAsync(queries []Q, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { if gb.cIndex == nil { return 0, moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -324,8 +336,9 @@ func (gb *GpuBruteForce[T]) SearchAsync(queries []T, numQueries uint64, dimensio return uint64(jobID), nil } -// SearchFloat32Async performs a K-Nearest Neighbor search with float32 queries asynchronously. -func (gb *GpuBruteForce[T]) SearchFloat32Async(queries []float32, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { +// SearchQuantizeAsync performs a K-Nearest Neighbor search with base-typed (B) +// queries asynchronously; cuVS quantizes B -> storage Q internally. +func (gb *GpuBruteForce[B, Q]) SearchQuantizeAsync(queries []B, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { if gb.cIndex == nil { return 0, moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -334,9 +347,9 @@ func (gb *GpuBruteForce[T]) SearchFloat32Async(queries []float32, numQueries uin } var errmsg *C.char - jobID := C.gpu_brute_force_search_float_async( + jobID := C.gpu_brute_force_search_quantize_async( gb.cIndex, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -354,7 +367,7 @@ func (gb *GpuBruteForce[T]) SearchFloat32Async(queries []float32, numQueries uin } // SearchWait waits for an asynchronous search to complete and returns the results. -func (gb *GpuBruteForce[T]) SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) { +func (gb *GpuBruteForce[B, Q]) SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) { if gb.cIndex == nil { return nil, nil, moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -386,7 +399,7 @@ func (gb *GpuBruteForce[T]) SearchWait(jobID uint64, numQueries uint64, limit ui } // Cap returns the capacity of the index buffer -func (gb *GpuBruteForce[T]) Cap() uint64 { +func (gb *GpuBruteForce[B, Q]) Cap() uint64 { if gb.cIndex == nil { return 0 } @@ -394,7 +407,7 @@ func (gb *GpuBruteForce[T]) Cap() uint64 { } // Len returns current number of vectors in index -func (gb *GpuBruteForce[T]) Len() uint64 { +func (gb *GpuBruteForce[B, Q]) Len() uint64 { if gb.cIndex == nil { return 0 } @@ -402,7 +415,7 @@ func (gb *GpuBruteForce[T]) Len() uint64 { } // Info returns detailed information about the index as a JSON string. -func (gb *GpuBruteForce[T]) Info() (string, error) { +func (gb *GpuBruteForce[B, Q]) Info() (string, error) { if gb.cIndex == nil { return "", moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -425,7 +438,7 @@ func (gb *GpuBruteForce[T]) Info() (string, error) { } // Destroy frees the C++ GpuBruteForce instance -func (gb *GpuBruteForce[T]) Destroy() error { +func (gb *GpuBruteForce[B, Q]) Destroy() error { if gb.cIndex == nil { return nil } @@ -441,7 +454,7 @@ func (gb *GpuBruteForce[T]) Destroy() error { } // SetFilterColumns registers filter-column metadata. See GpuCagra.SetFilterColumns. -func (gb *GpuBruteForce[T]) SetFilterColumns(colMetaJSON string, totalCount uint64) error { +func (gb *GpuBruteForce[B, Q]) SetFilterColumns(colMetaJSON string, totalCount uint64) error { if gb.cIndex == nil { return moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -458,7 +471,7 @@ func (gb *GpuBruteForce[T]) SetFilterColumns(colMetaJSON string, totalCount uint } // AddFilterChunk appends raw filter-column bytes. See GpuCagra.AddFilterChunk. -func (gb *GpuBruteForce[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { +func (gb *GpuBruteForce[B, Q]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { if gb.cIndex == nil { return moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -489,7 +502,7 @@ func (gb *GpuBruteForce[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitma } // SearchWithFilter runs a filtered K-NN search. predsJSON="" = unfiltered. -func (gb *GpuBruteForce[T]) SearchWithFilter(queries []T, numQueries uint64, dimension uint32, limit uint32, predsJSON string) ([]int64, []float32, error) { +func (gb *GpuBruteForce[B, Q]) SearchWithFilter(queries []Q, numQueries uint64, dimension uint32, limit uint32, predsJSON string) ([]int64, []float32, error) { if gb.cIndex == nil { return nil, nil, moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -531,8 +544,9 @@ func (gb *GpuBruteForce[T]) SearchWithFilter(queries []T, numQueries uint64, dim return neighbors, distances, nil } -// SearchFloatWithFilter runs a filtered K-NN search with float32 queries. -func (gb *GpuBruteForce[T]) SearchFloatWithFilter(queries []float32, numQueries uint64, dimension uint32, limit uint32, predsJSON string) ([]int64, []float32, error) { +// SearchQuantizeWithFilter runs a filtered K-NN search with base-typed (B) queries; +// cuVS quantizes B -> storage Q internally. +func (gb *GpuBruteForce[B, Q]) SearchQuantizeWithFilter(queries []B, numQueries uint64, dimension uint32, limit uint32, predsJSON string) ([]int64, []float32, error) { if gb.cIndex == nil { return nil, nil, moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -544,9 +558,9 @@ func (gb *GpuBruteForce[T]) SearchFloatWithFilter(queries []float32, numQueries cPreds := C.CString(predsJSON) defer C.free(unsafe.Pointer(cPreds)) - cResult := C.gpu_brute_force_search_float_with_filter( + cResult := C.gpu_brute_force_search_quantize_with_filter( gb.cIndex, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -574,12 +588,47 @@ func (gb *GpuBruteForce[T]) SearchFloatWithFilter(queries []float32, numQueries return neighbors, distances, nil } -// SearchFloatWithFilterAsync submits a filtered float32 K-NN search and +// SearchQuantizeWithFilterAsync submits a filtered base-typed (B) K-NN search and // returns a job_id; collect the result with SearchWait. Mirrors -// SearchFloat32Async + the predicate-eval semantics of SearchFloatWithFilter. -// Used by the multi-index brute-force fallback so it runs in parallel with -// the primary IVF/CAGRA shards. -func (gb *GpuBruteForce[T]) SearchFloatWithFilterAsync(queries []float32, numQueries uint64, dimension uint32, limit uint32, predsJSON string) (uint64, error) { +// SearchQuantizeAsync + the predicate-eval semantics of SearchQuantizeWithFilter. +// cuVS quantizes B -> storage Q internally. Used by the multi-index brute-force +// fallback so it runs in parallel with the primary IVF/CAGRA shards. +func (gb *GpuBruteForce[B, Q]) SearchQuantizeWithFilterAsync(queries []B, numQueries uint64, dimension uint32, limit uint32, predsJSON string) (uint64, error) { + if gb.cIndex == nil { + return 0, moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") + } + if len(queries) == 0 || numQueries == 0 { + return 0, nil + } + + var errmsg *C.char + cPreds := C.CString(predsJSON) + defer C.free(unsafe.Pointer(cPreds)) + + jobID := C.gpu_brute_force_search_quantize_with_filter_async( + gb.cIndex, + unsafe.Pointer(&queries[0]), + C.uint64_t(numQueries), + C.uint32_t(dimension), + C.uint32_t(limit), + cPreds, + unsafe.Pointer(&errmsg), + ) + runtime.KeepAlive(queries) + + if errmsg != nil { + errStr := C.GoString(errmsg) + C.free(unsafe.Pointer(errmsg)) + return 0, moerr.NewInternalErrorNoCtx(errStr) + } + return uint64(jobID), nil +} + +// SearchWithFilterAsync submits a filtered K-NN search with native-typed (T) +// queries and returns a job_id; collect the result with SearchWait. Native +// counterpart of SearchFloatWithFilterAsync (no widening) — lets the filtered +// overflow stay in the base element type T (e.g. half). +func (gb *GpuBruteForce[B, Q]) SearchWithFilterAsync(queries []Q, numQueries uint64, dimension uint32, limit uint32, predsJSON string) (uint64, error) { if gb.cIndex == nil { return 0, moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -591,9 +640,9 @@ func (gb *GpuBruteForce[T]) SearchFloatWithFilterAsync(queries []float32, numQue cPreds := C.CString(predsJSON) defer C.free(unsafe.Pointer(cPreds)) - jobID := C.gpu_brute_force_search_float_with_filter_async( + jobID := C.gpu_brute_force_search_with_filter_async( gb.cIndex, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), diff --git a/pkg/cuvs/brute_force_test.go b/pkg/cuvs/brute_force_test.go index 91c5086bc548f..83ece6e810a8e 100644 --- a/pkg/cuvs/brute_force_test.go +++ b/pkg/cuvs/brute_force_test.go @@ -30,7 +30,7 @@ func TestGpuBruteForce(t *testing.T) { dataset[i*uint64(dimension)+1] = float32(i) } - index, err := NewGpuBruteForce[float32](dataset, n_vectors, dimension, L2Expanded, 1, 0) + index, err := NewGpuBruteForce[float32, float32](dataset, n_vectors, dimension, L2Expanded, 1, 0) if err != nil { t.Fatalf("Failed to create GpuBruteForce: %v", err) } @@ -62,7 +62,7 @@ func TestGpuBruteForceChunked(t *testing.T) { totalCount := uint64(100) // Create empty index (target type half) - index, err := NewGpuBruteForceEmpty[Float16](totalCount, dimension, L2Expanded, 1, 0) + index, err := NewGpuBruteForceEmpty[float32, Float16](totalCount, dimension, L2Expanded, 1, 0) if err != nil { t.Fatalf("Failed to create GpuBruteForceEmpty: %v", err) } @@ -88,7 +88,7 @@ func TestGpuBruteForceChunked(t *testing.T) { for j := range chunk { chunk[j] = val } - err = index.AddChunkFloat(chunk, chunkSize, nil) + err = index.AddChunkQuantize(chunk, chunkSize, nil) if err != nil { t.Fatalf("AddChunkFloat failed at offset %d: %v", i, err) } @@ -132,7 +132,7 @@ func TestGpuBruteForceFloat16(t *testing.T) { t.Fatalf("Failed to convert dataset to F16: %v", err) } - index, err := NewGpuBruteForce(hDataset, count, dimension, L2Expanded, 1, 0) + index, err := NewGpuBruteForce[Float16, Float16](hDataset, count, dimension, L2Expanded, 1, 0) if err != nil { t.Fatalf("Failed to create F16 GpuBruteForce: %v", err) } @@ -179,7 +179,7 @@ func TestGpuBruteForceFilter(t *testing.T) { pkids[i] = int64(1000 + i) } - idx, err := NewGpuBruteForceEmpty[float32](nVectors, dimension, L2Expanded, 1, 0) + idx, err := NewGpuBruteForceEmpty[float32, float32](nVectors, dimension, L2Expanded, 1, 0) if err != nil { t.Fatalf("NewGpuBruteForceEmpty: %v", err) } @@ -192,7 +192,7 @@ func TestGpuBruteForceFilter(t *testing.T) { if err = idx.SetFilterColumns(colMetaJSON, nVectors); err != nil { t.Fatalf("SetFilterColumns: %v", err) } - if err = idx.AddChunkFloat(dataset, nVectors, pkids); err != nil { + if err = idx.AddChunkQuantize(dataset, nVectors, pkids); err != nil { t.Fatalf("AddChunkFloat: %v", err) } // One column of int64; row i value = i. No nulls. @@ -214,7 +214,7 @@ func TestGpuBruteForceFilter(t *testing.T) { // Query closest to row 0; without filter NN would be pkid 1000 (row 0). queries := []float32{0.0, 0.0} predsJSON := `[{"col":0,"op":">","val":50}]` - jobID, err := idx.SearchFloatWithFilterAsync(queries, 1, dimension, 1, predsJSON) + jobID, err := idx.SearchQuantizeWithFilterAsync(queries, 1, dimension, 1, predsJSON) if err != nil { t.Fatalf("SearchFloatWithFilterAsync: %v", err) } @@ -231,7 +231,7 @@ func TestGpuBruteForceFilter(t *testing.T) { } // Sanity: empty preds JSON falls through to unfiltered NN (pkid 1000). - jobID2, err := idx.SearchFloatWithFilterAsync(queries, 1, dimension, 1, "") + jobID2, err := idx.SearchQuantizeWithFilterAsync(queries, 1, dimension, 1, "") if err != nil { t.Fatalf("SearchFloatWithFilterAsync (no preds): %v", err) } @@ -254,8 +254,8 @@ func BenchmarkGpuAddChunkAndSearchBruteForceF16(b *testing.B) { dataset[i] = rand.Float32() } - // Use Float16 as internal type - index, err := NewGpuBruteForceEmpty[Float16](uint64(totalCount), dimension, L2Expanded, 8, 0) + // Use Float16 storage with float32 base/query (quantize f32 -> half). + index, err := NewGpuBruteForceEmpty[float32, Float16](uint64(totalCount), dimension, L2Expanded, 8, 0) if err != nil { b.Fatalf("Failed to create index: %v", err) } @@ -268,7 +268,7 @@ func BenchmarkGpuAddChunkAndSearchBruteForceF16(b *testing.B) { // Add data in chunks using AddChunkFloat for i := 0; i < totalCount; i += chunkSize { chunk := dataset[i*dimension : (i+chunkSize)*dimension] - if err := index.AddChunkFloat(chunk, uint64(chunkSize), nil); err != nil { + if err := index.AddChunkQuantize(chunk, uint64(chunkSize), nil); err != nil { b.Fatalf("AddChunkFloat failed at %d: %v", i, err) } } @@ -286,7 +286,7 @@ func BenchmarkGpuAddChunkAndSearchBruteForceF16(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, _, err := index.SearchFloat(queries, 1, dimension, 10) + _, _, err := index.SearchQuantize(queries, 1, dimension, 10) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -294,7 +294,7 @@ func BenchmarkGpuAddChunkAndSearchBruteForceF16(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(totalCount), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - neighbors, _, err := index.SearchFloat(queries, numQueries, dimension, limit) + neighbors, _, err := index.SearchQuantize(queries, numQueries, dimension, limit) if err != nil { return nil, err } @@ -311,7 +311,7 @@ func BenchmarkGpuBruteForceF32(b *testing.B) { dataset[i] = rand.Float32() } - index, err := NewGpuBruteForce[float32](dataset, uint64(totalCount), dimension, L2Expanded, 8, 0) + index, err := NewGpuBruteForce[float32, float32](dataset, uint64(totalCount), dimension, L2Expanded, 8, 0) if err != nil { b.Fatalf("Failed to create index: %v", err) } @@ -331,7 +331,7 @@ func BenchmarkGpuBruteForceF32(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, _, err := index.SearchFloat(queries, 1, dimension, 10) + _, _, err := index.SearchQuantize(queries, 1, dimension, 10) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -339,7 +339,7 @@ func BenchmarkGpuBruteForceF32(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(totalCount), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - neighbors, _, err := index.SearchFloat(queries, numQueries, dimension, limit) + neighbors, _, err := index.SearchQuantize(queries, numQueries, dimension, limit) if err != nil { return nil, err } diff --git a/pkg/cuvs/cagra.go b/pkg/cuvs/cagra.go index e14a53871a769..8ca474c3a8000 100644 --- a/pkg/cuvs/cagra.go +++ b/pkg/cuvs/cagra.go @@ -32,7 +32,7 @@ import ( ) // GpuCagra represents the C++ gpu_cagra_t object. -type GpuCagra[T VectorType] struct { +type GpuCagra[B, Q VectorType] struct { cCagra C.gpu_cagra_c dimension uint32 nthread uint32 @@ -43,7 +43,7 @@ type GpuCagra[T VectorType] struct { // SetBatchWindow sets the batching window in microseconds for search operations. // A window of 0 disables batching; any positive value enables batching with that delay. -func (gi *GpuCagra[T]) SetBatchWindow(windowUs int64) error { +func (gi *GpuCagra[B, Q]) SetBatchWindow(windowUs int64) error { gi.batchWindowUs = windowUs if gi.cCagra != nil { var errmsg *C.char @@ -61,7 +61,7 @@ func (gi *GpuCagra[T]) SetBatchWindow(windowUs int64) error { // flag. false (default): dispatch eagerly at the full batch size. true: wait for // the batch to fill or the window to elapse, then dispatch at the real size. // Has no effect unless the batch window is > 0. -func (gi *GpuCagra[T]) SetDynbConservativeDispatch(enable bool) error { +func (gi *GpuCagra[B, Q]) SetDynbConservativeDispatch(enable bool) error { gi.dynbConservativeDispatch = enable if gi.cCagra != nil { var errmsg *C.char @@ -77,13 +77,14 @@ func (gi *GpuCagra[T]) SetDynbConservativeDispatch(enable bool) error { // NewGpuCagra creates a new GpuCagra instance from a dataset. // ids may be nil to use internal sequential IDs (0..count-1). -func NewGpuCagra[T VectorType](dataset []T, count uint64, dimension uint32, metric DistanceType, - bp CagraBuildParams, devices []int, nthread uint32, mode DistributionMode, ids []int64) (*GpuCagra[T], error) { +func NewGpuCagra[B, Q VectorType](dataset []Q, count uint64, dimension uint32, metric DistanceType, + bp CagraBuildParams, devices []int, nthread uint32, mode DistributionMode, ids []int64) (*GpuCagra[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cDevices := make([]C.int, len(devices)) for i, d := range devices { @@ -111,6 +112,7 @@ func NewGpuCagra[T VectorType](dataset []T, count uint64, dimension uint32, metr C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), cIds, unsafe.Pointer(&errmsg), @@ -129,7 +131,7 @@ func NewGpuCagra[T VectorType](dataset []T, count uint64, dimension uint32, metr return nil, moerr.NewInternalErrorNoCtx("failed to create GpuCagra") } - return &GpuCagra[T]{ + return &GpuCagra[B, Q]{ cCagra: cCagra, dimension: dimension, nthread: nthread, @@ -138,13 +140,14 @@ func NewGpuCagra[T VectorType](dataset []T, count uint64, dimension uint32, metr } // NewGpuCagraFromFile creates a new GpuCagra instance by loading from a file. -func NewGpuCagraFromFile[T VectorType](filename string, dimension uint32, metric DistanceType, - bp CagraBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuCagra[T], error) { +func NewGpuCagraFromFile[B, Q VectorType](filename string, dimension uint32, metric DistanceType, + bp CagraBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuCagra[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cFilename := C.CString(filename) defer C.free(unsafe.Pointer(cFilename)) @@ -169,6 +172,7 @@ func NewGpuCagraFromFile[T VectorType](filename string, dimension uint32, metric C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), unsafe.Pointer(&errmsg), ) @@ -184,7 +188,7 @@ func NewGpuCagraFromFile[T VectorType](filename string, dimension uint32, metric return nil, moerr.NewInternalErrorNoCtx("failed to load GpuCagra from file") } - return &GpuCagra[T]{ + return &GpuCagra[B, Q]{ cCagra: cCagra, dimension: dimension, nthread: nthread, @@ -196,8 +200,8 @@ func NewGpuCagraFromFile[T VectorType](filename string, dimension uint32, metric // For Sharded loads we peek manifest.json to learn the saved shard count and // truncate `devices` to that count, so the C++ worker only spawns threads / // RMM pools on devices that will actually host a shard. -func NewGpuCagraFromDataDirectory[T VectorType](dir string, dimension uint32, metric DistanceType, - bp CagraBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuCagra[T], error) { +func NewGpuCagraFromDataDirectory[B, Q VectorType](dir string, dimension uint32, metric DistanceType, + bp CagraBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuCagra[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } @@ -208,7 +212,8 @@ func NewGpuCagraFromDataDirectory[T VectorType](dir string, dimension uint32, me return nil, err } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() cDevices := make([]C.int, len(devices)) for i, d := range devices { cDevices[i] = C.int(d) @@ -230,6 +235,7 @@ func NewGpuCagraFromDataDirectory[T VectorType](dir string, dimension uint32, me C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), nil, unsafe.Pointer(&errmsg), @@ -264,7 +270,7 @@ func NewGpuCagraFromDataDirectory[T VectorType](dir string, dimension uint32, me return nil, moerr.NewInternalErrorNoCtx(errStr) } - return &GpuCagra[T]{ + return &GpuCagra[B, Q]{ cCagra: cCagra, dimension: dimension, nthread: nthread, @@ -273,7 +279,7 @@ func NewGpuCagraFromDataDirectory[T VectorType](dir string, dimension uint32, me } // Destroy frees the C++ gpu_cagra_t instance -func (gi *GpuCagra[T]) Destroy() error { +func (gi *GpuCagra[B, Q]) Destroy() error { if gi.cCagra == nil { return nil } @@ -289,7 +295,7 @@ func (gi *GpuCagra[T]) Destroy() error { } // Start initializes the worker and resources -func (gi *GpuCagra[T]) Start() error { +func (gi *GpuCagra[B, Q]) Start() error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -317,7 +323,7 @@ func (gi *GpuCagra[T]) Start() error { } // Build triggers the build or file loading process -func (gi *GpuCagra[T]) Build() error { +func (gi *GpuCagra[B, Q]) Build() error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -332,13 +338,14 @@ func (gi *GpuCagra[T]) Build() error { } // NewGpuCagraEmpty creates a new GpuCagra instance with pre-allocated buffer but no data yet. -func NewGpuCagraEmpty[T VectorType](totalCount uint64, dimension uint32, metric DistanceType, - bp CagraBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuCagra[T], error) { +func NewGpuCagraEmpty[B, Q VectorType](totalCount uint64, dimension uint32, metric DistanceType, + bp CagraBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuCagra[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cDevices := make([]C.int, len(devices)) for i, d := range devices { @@ -360,6 +367,7 @@ func NewGpuCagraEmpty[T VectorType](totalCount uint64, dimension uint32, metric C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), nil, unsafe.Pointer(&errmsg), @@ -376,7 +384,7 @@ func NewGpuCagraEmpty[T VectorType](totalCount uint64, dimension uint32, metric return nil, moerr.NewInternalErrorNoCtx("failed to create empty GpuCagra") } - return &GpuCagra[T]{ + return &GpuCagra[B, Q]{ cCagra: cCagra, dimension: dimension, nthread: nthread, @@ -385,7 +393,7 @@ func NewGpuCagraEmpty[T VectorType](totalCount uint64, dimension uint32, metric } // AddChunk adds a chunk of data to the pre-allocated buffer. -func (gi *GpuCagra[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) error { +func (gi *GpuCagra[B, Q]) AddChunk(chunk []Q, chunkCount uint64, ids []int64) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -416,8 +424,10 @@ func (gi *GpuCagra[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) error return nil } -// AddChunkFloat adds a chunk of float32 data, performing on-the-fly quantization if needed. -func (gi *GpuCagra[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []int64) error { +// AddChunkQuantize adds a chunk of base-typed (B) data, quantizing natively to +// the storage type Q (int8/uint8) via the B-source quantizer. base_data is the +// raw bytes of chunkCount*dim B-typed elements. No f32 detour. +func (gi *GpuCagra[B, Q]) AddChunkQuantize(chunk []B, chunkCount uint64, ids []int64) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -430,9 +440,9 @@ func (gi *GpuCagra[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []i if len(ids) > 0 { cIds = (*C.int64_t)(unsafe.Pointer(&ids[0])) } - C.gpu_cagra_add_chunk_float( + C.gpu_cagra_add_chunk_quantize( gi.cCagra, - (*C.float)(&chunk[0]), + unsafe.Pointer(&chunk[0]), C.uint64_t(chunkCount), cIds, unsafe.Pointer(&errmsg), @@ -448,8 +458,9 @@ func (gi *GpuCagra[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []i return nil } -// TrainQuantizer trains the scalar quantizer (if T is 1-byte) -func (gi *GpuCagra[T]) TrainQuantizer(trainData []float32, nSamples uint64) error { +// TrainQuantizer trains the scalar quantizer (if Q is 1-byte) from base-typed +// (B) training data. +func (gi *GpuCagra[B, Q]) TrainQuantizer(trainData []B, nSamples uint64) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -460,7 +471,7 @@ func (gi *GpuCagra[T]) TrainQuantizer(trainData []float32, nSamples uint64) erro var errmsg *C.char C.gpu_cagra_train_quantizer( gi.cCagra, - (*C.float)(&trainData[0]), + unsafe.Pointer(&trainData[0]), C.uint64_t(nSamples), unsafe.Pointer(&errmsg), ) @@ -475,7 +486,7 @@ func (gi *GpuCagra[T]) TrainQuantizer(trainData []float32, nSamples uint64) erro } // SetQuantizer sets the scalar quantizer parameters (if T is 1-byte) -func (gi *GpuCagra[T]) SetQuantizer(min, max float32) error { +func (gi *GpuCagra[B, Q]) SetQuantizer(min, max float32) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -497,7 +508,7 @@ func (gi *GpuCagra[T]) SetQuantizer(min, max float32) error { } // GetQuantizer gets the scalar quantizer parameters (if T is 1-byte) -func (gi *GpuCagra[T]) GetQuantizer() (float32, float32, error) { +func (gi *GpuCagra[B, Q]) GetQuantizer() (float32, float32, error) { if gi.cCagra == nil { return 0, 0, moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -520,7 +531,7 @@ func (gi *GpuCagra[T]) GetQuantizer() (float32, float32, error) { } // Save serializes the index to a file -func (gi *GpuCagra[T]) Save(filename string) error { +func (gi *GpuCagra[B, Q]) Save(filename string) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -538,7 +549,7 @@ func (gi *GpuCagra[T]) Save(filename string) error { } // Pack saves the index to a .tar or .tar.gz file using save_dir. -func (gi *GpuCagra[T]) Pack(filename string) error { +func (gi *GpuCagra[B, Q]) Pack(filename string) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -567,7 +578,7 @@ func (gi *GpuCagra[T]) Pack(filename string) error { // mode overrides the distribution mode at load time — pass Replicated to broadcast // a SINGLE_GPU .tar to all GPUs without rebuilding. // The index must already be initialized and started before calling Unpack. -func (gi *GpuCagra[T]) Unpack(filename string, mode DistributionMode) error { +func (gi *GpuCagra[B, Q]) Unpack(filename string, mode DistributionMode) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -596,7 +607,7 @@ func (gi *GpuCagra[T]) Unpack(filename string, mode DistributionMode) error { } // DeleteId removes an ID from the index (soft delete). -func (gi *GpuCagra[T]) DeleteId(id int64) error { +func (gi *GpuCagra[B, Q]) DeleteId(id int64) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -614,7 +625,7 @@ func (gi *GpuCagra[T]) DeleteId(id int64) error { // path; if profiling shows the cgo crossing dominates we can swap to a // single batched cgo entry (the C++ side already does the host-side // id_to_index_ lookup; the loop is per-id). -func (gi *GpuCagra[T]) DeleteIds(ids []int64) error { +func (gi *GpuCagra[B, Q]) DeleteIds(ids []int64) error { for _, id := range ids { if err := gi.DeleteId(id); err != nil { return err @@ -623,8 +634,8 @@ func (gi *GpuCagra[T]) DeleteIds(ids []int64) error { return nil } -func (gi *GpuCagra[T]) adjustSearchParams(sp CagraSearchParams, limit uint32) CagraSearchParams { - qtype := GetQuantization[T]() +func (gi *GpuCagra[B, Q]) adjustSearchParams(sp CagraSearchParams, limit uint32) CagraSearchParams { + qtype := GetQuantization[Q]() isByteType := (qtype == INT8 || qtype == UINT8) if isByteType { @@ -641,7 +652,7 @@ func (gi *GpuCagra[T]) adjustSearchParams(sp CagraSearchParams, limit uint32) Ca } // Search performs a K-Nearest Neighbor search -func (gi *GpuCagra[T]) Search(queries []T, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) (SearchResult, error) { +func (gi *GpuCagra[B, Q]) Search(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) (SearchResult, error) { if gi.cCagra == nil { return SearchResult{}, moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -696,7 +707,7 @@ func (gi *GpuCagra[T]) Search(queries []T, numQueries uint64, dimension uint32, } // SearchFloat performs a K-Nearest Neighbor search with float32 queries -func (gi *GpuCagra[T]) SearchFloat(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) (SearchResult, error) { +func (gi *GpuCagra[B, Q]) SearchQuantize(queries []B, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) (SearchResult, error) { if gi.cCagra == nil { return SearchResult{}, moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -712,9 +723,9 @@ func (gi *GpuCagra[T]) SearchFloat(queries []float32, numQueries uint64, dimensi search_width: C.size_t(sp.SearchWidth), } - res := C.gpu_cagra_search_float( + res := C.gpu_cagra_search_quantize( gi.cCagra, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -751,12 +762,12 @@ func (gi *GpuCagra[T]) SearchFloat(queries []float32, numQueries uint64, dimensi } // SearchAsync performs a K-Nearest Neighbor search asynchronously. -func (gi *GpuCagra[T]) SearchAsync(queries []T, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { +func (gi *GpuCagra[B, Q]) SearchAsync(queries []Q, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { return gi.SearchAsyncWithParams(queries, numQueries, dimension, limit, DefaultCagraSearchParams()) } // SearchAsyncWithParams performs a K-Nearest Neighbor search asynchronously with custom parameters. -func (gi *GpuCagra[T]) SearchAsyncWithParams(queries []T, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) (uint64, error) { +func (gi *GpuCagra[B, Q]) SearchAsyncWithParams(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) (uint64, error) { if gi.cCagra == nil { return 0, moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -792,13 +803,10 @@ func (gi *GpuCagra[T]) SearchAsyncWithParams(queries []T, numQueries uint64, dim return uint64(jobID), nil } -// SearchFloat32Async performs a K-Nearest Neighbor search with float32 queries asynchronously. -func (gi *GpuCagra[T]) SearchFloat32Async(queries []float32, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { - return gi.SearchFloat32AsyncWithParams(queries, numQueries, dimension, limit, DefaultCagraSearchParams()) -} - -// SearchFloat32AsyncWithParams performs a K-Nearest Neighbor search with float32 queries asynchronously with custom parameters. -func (gi *GpuCagra[T]) SearchFloat32AsyncWithParams(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) (uint64, error) { +// SearchQuantizeAsyncWithParams submits an async KNN search with a base-typed (B) +// query; the index converts B to its storage type Q on device (B==Q copy, or the +// learned/cast quantizer for narrower Q). Unifies the former float32 and half query paths. +func (gi *GpuCagra[B, Q]) SearchQuantizeAsyncWithParams(queries []B, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) (uint64, error) { if gi.cCagra == nil { return 0, moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -814,9 +822,9 @@ func (gi *GpuCagra[T]) SearchFloat32AsyncWithParams(queries []float32, numQuerie search_width: C.size_t(sp.SearchWidth), } - jobID := C.gpu_cagra_search_float_async( + jobID := C.gpu_cagra_search_quantize_async( gi.cCagra, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -835,7 +843,7 @@ func (gi *GpuCagra[T]) SearchFloat32AsyncWithParams(queries []float32, numQuerie } // SearchWait waits for an asynchronous search to complete and returns the results. -func (gi *GpuCagra[T]) SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) { +func (gi *GpuCagra[B, Q]) SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) { if gi.cCagra == nil { return nil, nil, moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -868,7 +876,7 @@ func (gi *GpuCagra[T]) SearchWait(jobID uint64, numQueries uint64, limit uint32) } // Cap returns the capacity of the index buffer -func (gi *GpuCagra[T]) Cap() uint64 { +func (gi *GpuCagra[B, Q]) Cap() uint64 { if gi.cCagra == nil { return 0 } @@ -876,7 +884,7 @@ func (gi *GpuCagra[T]) Cap() uint64 { } // Len returns current number of vectors in index -func (gi *GpuCagra[T]) Len() uint64 { +func (gi *GpuCagra[B, Q]) Len() uint64 { if gi.cCagra == nil { return 0 } @@ -886,7 +894,7 @@ func (gi *GpuCagra[T]) Len() uint64 { // GetFilterColMetaJSON returns the INCLUDE-column metadata of the loaded // index as a JSON string ready to be re-fed into SetFilterColumns. Returns // "" for indexes that were built without INCLUDE columns. -func (gi *GpuCagra[T]) GetFilterColMetaJSON() string { +func (gi *GpuCagra[B, Q]) GetFilterColMetaJSON() string { if gi.cCagra == nil { return "" } @@ -904,7 +912,7 @@ func (gi *GpuCagra[T]) GetFilterColMetaJSON() string { } // Info returns detailed information about the index as a JSON string. -func (gi *GpuCagra[T]) Info() (string, error) { +func (gi *GpuCagra[B, Q]) Info() (string, error) { if gi.cCagra == nil { return "", moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -928,7 +936,7 @@ func (gi *GpuCagra[T]) Info() (string, error) { // Extend adds more vectors to the index (single-GPU only). // newIDs may be nil to auto-assign sequential IDs starting from the current index size. -func (gi *GpuCagra[T]) Extend(additionalData []T, numVectors uint64, newIDs []int64) error { +func (gi *GpuCagra[B, Q]) Extend(additionalData []Q, numVectors uint64, newIDs []int64) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -961,7 +969,7 @@ func (gi *GpuCagra[T]) Extend(additionalData []T, numVectors uint64, newIDs []in } // MergeGpuCagra combines multiple single-GPU GpuCagra indices into a new one. -func MergeGpuCagra[T VectorType](indices []*GpuCagra[T], nthread uint32, devices []int) (*GpuCagra[T], error) { +func MergeGpuCagra[B, Q VectorType](indices []*GpuCagra[B, Q], nthread uint32, devices []int) (*GpuCagra[B, Q], error) { if len(indices) == 0 { return nil, moerr.NewInternalErrorNoCtx("no indices to merge") } @@ -1001,7 +1009,7 @@ func MergeGpuCagra[T VectorType](indices []*GpuCagra[T], nthread uint32, devices return nil, moerr.NewInternalErrorNoCtx("failed to merge GpuCagra indices") } - return &GpuCagra[T]{ + return &GpuCagra[B, Q]{ cCagra: cCagra, dimension: indices[0].dimension, nthread: nthread, @@ -1017,7 +1025,7 @@ type SearchResult struct { // SaveToDir saves the index files to a directory using gpu_cagra_save_dir. // This is used by CagraModel to save to a directory before packing to tar. -func (gi *GpuCagra[T]) SaveToDir(dirPath string) error { +func (gi *GpuCagra[B, Q]) SaveToDir(dirPath string) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -1036,7 +1044,7 @@ func (gi *GpuCagra[T]) SaveToDir(dirPath string) error { // LoadFromDir loads index components from a directory using gpu_cagra_load_dir. // mode overrides the distribution mode at load time. // The index must already be initialized and started before calling LoadFromDir. -func (gi *GpuCagra[T]) LoadFromDir(dirPath string, mode DistributionMode) error { +func (gi *GpuCagra[B, Q]) LoadFromDir(dirPath string, mode DistributionMode) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -1056,7 +1064,7 @@ func (gi *GpuCagra[T]) LoadFromDir(dirPath string, mode DistributionMode) error // colMetaJSON is a JSON array of {"name":"...","type":N} entries, where N is // 0=int32, 1=int64, 2=float32, 3=float64, 4=uint64 (VARCHAR hash). // Must be called after Start() and before Build(). -func (gi *GpuCagra[T]) SetFilterColumns(colMetaJSON string, totalCount uint64) error { +func (gi *GpuCagra[B, Q]) SetFilterColumns(colMetaJSON string, totalCount uint64) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -1078,7 +1086,7 @@ func (gi *GpuCagra[T]) SetFilterColumns(colMetaJSON string, totalCount uint64) e // matching MO's null-mask convention) of ceil(nrows/32) entries, or nil when // the chunk has no nulls. // Ownership transfers to C++ at call return — the Go slice can be freed. -func (gi *GpuCagra[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { +func (gi *GpuCagra[B, Q]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -1110,7 +1118,7 @@ func (gi *GpuCagra[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []u // SearchWithFilter runs a filtered K-NN search. predsJSON is a JSON predicate // array; passing "" yields unfiltered behavior identical to Search(). -func (gi *GpuCagra[T]) SearchWithFilter(queries []T, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams, predsJSON string) (SearchResult, error) { +func (gi *GpuCagra[B, Q]) SearchWithFilter(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams, predsJSON string) (SearchResult, error) { if gi.cCagra == nil { return SearchResult{}, moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -1161,8 +1169,9 @@ func (gi *GpuCagra[T]) SearchWithFilter(queries []T, numQueries uint64, dimensio return SearchResult{Neighbors: neighbors, Distances: distances}, nil } -// SearchFloatWithFilter runs a filtered K-NN search with float32 queries. -func (gi *GpuCagra[T]) SearchFloatWithFilter(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams, predsJSON string) (SearchResult, error) { +// SearchQuantizeWithFilter runs a filtered K-NN search with base-typed (B) +// queries; the index converts B to storage T (copy / quantize / f32->f16 cast). +func (gi *GpuCagra[B, Q]) SearchQuantizeWithFilter(queries []B, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams, predsJSON string) (SearchResult, error) { if gi.cCagra == nil { return SearchResult{}, moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -1180,9 +1189,9 @@ func (gi *GpuCagra[T]) SearchFloatWithFilter(queries []float32, numQueries uint6 cPreds := C.CString(predsJSON) defer C.free(unsafe.Pointer(cPreds)) - res := C.gpu_cagra_search_float_with_filter( + res := C.gpu_cagra_search_quantize_with_filter( gi.cCagra, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -1213,12 +1222,12 @@ func (gi *GpuCagra[T]) SearchFloatWithFilter(queries []float32, numQueries uint6 return SearchResult{Neighbors: neighbors, Distances: distances}, nil } -// SearchFloatWithFilterAsync submits a filtered float32 K-NN search and -// returns a job_id; collect the result with SearchWait. Mirrors -// SearchFloat32AsyncWithParams + the predicate-eval semantics of -// SearchFloatWithFilter. Used by MultiGpuCagra to dispatch per-shard -// filtered searches in parallel. -func (gi *GpuCagra[T]) SearchFloatWithFilterAsync(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams, predsJSON string) (uint64, error) { +// SearchQuantizeWithFilterAsync submits a filtered K-NN search with base-typed +// (B) queries and returns a job_id; collect the result with SearchWait. Mirrors +// SearchQuantizeAsyncWithParams + the predicate-eval semantics of +// SearchQuantizeWithFilter. Used by MultiGpuCagra to dispatch per-shard +// filtered searches in parallel. The index converts B to storage T. +func (gi *GpuCagra[B, Q]) SearchQuantizeWithFilterAsync(queries []B, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams, predsJSON string) (uint64, error) { if gi.cCagra == nil { return 0, moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -1236,9 +1245,9 @@ func (gi *GpuCagra[T]) SearchFloatWithFilterAsync(queries []float32, numQueries cPreds := C.CString(predsJSON) defer C.free(unsafe.Pointer(cPreds)) - jobID := C.gpu_cagra_search_float_with_filter_async( + jobID := C.gpu_cagra_search_quantize_with_filter_async( gi.cCagra, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), diff --git a/pkg/cuvs/cagra_test.go b/pkg/cuvs/cagra_test.go index 7095aabd3ccc4..704ed0b71af4d 100644 --- a/pkg/cuvs/cagra_test.go +++ b/pkg/cuvs/cagra_test.go @@ -36,7 +36,7 @@ func TestGpuCagra(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuCagra: %v", err) } @@ -80,7 +80,7 @@ func TestGpuCagraSaveLoad(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuCagra: %v", err) } @@ -97,7 +97,7 @@ func TestGpuCagraSaveLoad(t *testing.T) { defer os.Remove(filename) index.Destroy() - index2, err := NewGpuCagraFromFile[float32](filename, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index2, err := NewGpuCagraFromFile[float32, float32](filename, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("Failed to create GpuCagra from file: %v", err) } @@ -129,7 +129,7 @@ func TestGpuCagraPackUnpack(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuCagra: %v", err) } @@ -147,7 +147,7 @@ func TestGpuCagraPackUnpack(t *testing.T) { } defer os.Remove(filename) - index2, err := NewGpuCagraEmpty[float32](0, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index2, err := NewGpuCagraEmpty[float32, float32](0, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("NewGpuCagraEmpty failed: %v", err) } @@ -180,7 +180,7 @@ func TestGpuCagraFromDataDirectory(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuCagra: %v", err) } @@ -223,7 +223,7 @@ func TestGpuCagraFromDataDirectory(t *testing.T) { t.Fatalf("Unpack to dir failed: %v", err) } - index2, err := NewGpuCagraFromDataDirectory[float32](tmpDir, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index2, err := NewGpuCagraFromDataDirectory[float32, float32](tmpDir, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("NewGpuCagraFromDataDirectory failed: %v", err) } @@ -271,7 +271,7 @@ func TestGpuShardedCagra(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Sharded, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Sharded, nil) if err != nil { t.Fatalf("Failed to create sharded CAGRA: %v", err) } @@ -306,7 +306,7 @@ func TestGpuCagraChunked(t *testing.T) { bp.GraphDegree = 128 // Create empty index (target type int8) - index, err := NewGpuCagraEmpty[int8](totalCount, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index, err := NewGpuCagraEmpty[float32, int8](totalCount, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("Failed to create GpuCagraEmpty: %v", err) } @@ -325,7 +325,7 @@ func TestGpuCagraChunked(t *testing.T) { for j := range chunk { chunk[j] = val } - err = index.AddChunkFloat(chunk, chunkSize, nil) + err = index.AddChunkQuantize(chunk, chunkSize, nil) if err != nil { t.Fatalf("AddChunkFloat failed at offset %d: %v", i, err) } @@ -380,7 +380,7 @@ func TestGpuCagraExtend(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuCagra[float32, float32](dataset, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuCagra: %v", err) } @@ -436,11 +436,11 @@ func TestGpuCagraMerge(t *testing.T) { bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - idx1, err := NewGpuCagra[float32](ds1, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + idx1, err := NewGpuCagra[float32, float32](ds1, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create idx1: %v", err) } - idx2, err := NewGpuCagra[float32](ds2, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + idx2, err := NewGpuCagra[float32, float32](ds2, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create idx2: %v", err) } @@ -455,7 +455,7 @@ func TestGpuCagraMerge(t *testing.T) { defer idx1.Destroy() defer idx2.Destroy() - merged, err := MergeGpuCagra([]*GpuCagra[float32]{idx1, idx2}, 1, devices) + merged, err := MergeGpuCagra([]*GpuCagra[float32, float32]{idx1, idx2}, 1, devices) if err != nil { t.Fatalf("Merge failed: %v", err) } @@ -514,14 +514,14 @@ func TestGpuCagraMergeWithIds(t *testing.T) { } bp := DefaultCagraBuildParams() - idx1, err := NewGpuCagra[float32](ds1, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, ids1) + idx1, err := NewGpuCagra[float32, float32](ds1, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, ids1) if err != nil { t.Fatalf("Failed to create idx1: %v", err) } idx1.Start() idx1.Build() - idx2, err := NewGpuCagra[float32](ds2, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, ids2) + idx2, err := NewGpuCagra[float32, float32](ds2, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, ids2) if err != nil { t.Fatalf("Failed to create idx2: %v", err) } @@ -531,7 +531,7 @@ func TestGpuCagraMergeWithIds(t *testing.T) { defer idx1.Destroy() defer idx2.Destroy() - merged, err := MergeGpuCagra([]*GpuCagra[float32]{idx1, idx2}, 1, devices) + merged, err := MergeGpuCagra([]*GpuCagra[float32, float32]{idx1, idx2}, 1, devices) if err != nil { t.Fatalf("Merge failed: %v", err) } @@ -583,7 +583,7 @@ func TestGpuCagraDeleteId(t *testing.T) { } bp := DefaultCagraBuildParams() - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuCagra: %v", err) } @@ -643,7 +643,7 @@ func TestGpuReplicatedCagra(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Replicated, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Replicated, nil) if err != nil { t.Fatalf("Failed to create replicated CAGRA: %v", err) } @@ -684,7 +684,7 @@ func BenchmarkGpuShardedCagra(b *testing.B) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 16, Sharded, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 16, Sharded, nil) if err != nil { b.Fatalf("Failed to create sharded CAGRA: %v", err) } @@ -720,7 +720,7 @@ func BenchmarkGpuShardedCagra(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(n_vectors), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -743,7 +743,7 @@ func BenchmarkGpuSingleCagra(b *testing.B) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, SingleGpu, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, SingleGpu, nil) if err != nil { b.Fatalf("Failed to create single CAGRA: %v", err) } @@ -779,7 +779,7 @@ func BenchmarkGpuSingleCagra(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(n_vectors), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -805,7 +805,7 @@ func BenchmarkGpuReplicatedCagra(b *testing.B) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Replicated, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Replicated, nil) if err != nil { b.Fatalf("Failed to create replicated CAGRA: %v", err) } @@ -841,7 +841,7 @@ func BenchmarkGpuReplicatedCagra(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(n_vectors), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -866,7 +866,7 @@ func BenchmarkGpuAddChunkAndSearchCagraF16(b *testing.B) { bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 // Use Float16 as internal type - index, err := NewGpuCagraEmpty[Float16](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) + index, err := NewGpuCagraEmpty[float32, Float16](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) if err != nil { b.Fatalf("Failed to create index: %v", err) } @@ -879,7 +879,7 @@ func BenchmarkGpuAddChunkAndSearchCagraF16(b *testing.B) { // Add data in chunks using AddChunkFloat for i := 0; i < totalCount; i += chunkSize { chunk := dataset[i*dimension : (i+chunkSize)*dimension] - if err := index.AddChunkFloat(chunk, uint64(chunkSize), nil); err != nil { + if err := index.AddChunkQuantize(chunk, uint64(chunkSize), nil); err != nil { b.Fatalf("AddChunkFloat failed at %d: %v", i, err) } } @@ -899,7 +899,7 @@ func BenchmarkGpuAddChunkAndSearchCagraF16(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -907,7 +907,7 @@ func BenchmarkGpuAddChunkAndSearchCagraF16(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(totalCount), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -930,7 +930,7 @@ func BenchmarkGpuAddChunkAndSearchCagraInt8(b *testing.B) { bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 // Use int8 as internal type - index, err := NewGpuCagraEmpty[int8](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) + index, err := NewGpuCagraEmpty[float32, int8](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) if err != nil { b.Fatalf("Failed to create index: %v", err) } @@ -943,7 +943,7 @@ func BenchmarkGpuAddChunkAndSearchCagraInt8(b *testing.B) { // Add data in chunks using AddChunkFloat for i := 0; i < totalCount; i += chunkSize { chunk := dataset[i*dimension : (i+chunkSize)*dimension] - if err := index.AddChunkFloat(chunk, uint64(chunkSize), nil); err != nil { + if err := index.AddChunkQuantize(chunk, uint64(chunkSize), nil); err != nil { b.Fatalf("AddChunkFloat failed at %d: %v", i, err) } } @@ -963,7 +963,7 @@ func BenchmarkGpuAddChunkAndSearchCagraInt8(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -971,7 +971,7 @@ func BenchmarkGpuAddChunkAndSearchCagraInt8(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(totalCount), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -990,7 +990,7 @@ func TestGpuCagraLargeTopK(t *testing.T) { devices := []int{0} bp := DefaultCagraBuildParams() - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuCagra: %v", err) } diff --git a/pkg/cuvs/consolidate_test.go b/pkg/cuvs/consolidate_test.go index 1be296620b5fc..2a89a4db8477b 100644 --- a/pkg/cuvs/consolidate_test.go +++ b/pkg/cuvs/consolidate_test.go @@ -147,7 +147,7 @@ func TestShardedLoadWithFewerSavedShards(t *testing.T) { // --- Save phase: 2 shards over devs[:2] --- saveDevs := devs[:2] - src, err := NewGpuIvfFlat[float32](dataset, nVectors, dimension, L2Expanded, + src, err := NewGpuIvfFlat[float32, float32](dataset, nVectors, dimension, L2Expanded, bp, saveDevs, uint32(len(saveDevs)), Sharded, nil) if err != nil { t.Fatalf("save-side build: %v", err) @@ -182,7 +182,7 @@ func TestShardedLoadWithFewerSavedShards(t *testing.T) { // --- Load phase: caller supplies ALL available devs; wrapper should // truncate to the saved 2. --- - dst, err := NewGpuIvfFlatFromDataDirectory[float32](extractDir, dimension, L2Expanded, + dst, err := NewGpuIvfFlatFromDataDirectory[float32, float32](extractDir, dimension, L2Expanded, bp, devs, uint32(len(devs)), Sharded) if err != nil { t.Fatalf("load with extra devices: %v", err) diff --git a/pkg/cuvs/get_centers_test.go b/pkg/cuvs/get_centers_test.go index 8e02bfb67f0be..b484812b918f5 100644 --- a/pkg/cuvs/get_centers_test.go +++ b/pkg/cuvs/get_centers_test.go @@ -33,7 +33,7 @@ func testIvfFlatGetCenters[T VectorType](t *testing.T, name string) { devices := []int{0} bp := DefaultIvfFlatBuildParams() bp.NLists = 16 - index, err := NewGpuIvfFlat[T](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, T](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfFlat: %v", err) } diff --git a/pkg/cuvs/helper.go b/pkg/cuvs/helper.go index 8406cfaba036d..9d0bb4b36d153 100644 --- a/pkg/cuvs/helper.go +++ b/pkg/cuvs/helper.go @@ -186,7 +186,6 @@ type GpuIndexBase interface { // GpuIndex is a generic interface for all GPU-accelerated indexes that support async search. type GpuIndex[T VectorType] interface { SearchAsync(queries []T, numQueries uint64, dimension uint32, limit uint32) (uint64, error) - SearchFloat32Async(queries []float32, numQueries uint64, dimension uint32, limit uint32) (uint64, error) SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) Destroy() error Cap() uint64 diff --git a/pkg/cuvs/info_test.go b/pkg/cuvs/info_test.go index 595e77e7777ee..386fd3e58a6e4 100644 --- a/pkg/cuvs/info_test.go +++ b/pkg/cuvs/info_test.go @@ -105,16 +105,16 @@ func TestIndexInfoComprehensive(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err = NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) case "IVF-Flat": bp := DefaultIvfFlatBuildParams() bp.NLists = 1000 - index, err = NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) case "IVF-PQ": bp := DefaultIvfPqBuildParams() bp.NLists = 1000 bp.M = 16 - index, err = NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) } case "Float16": dataset := make([]Float16, n_vectors*uint64(dimension)) @@ -127,16 +127,16 @@ func TestIndexInfoComprehensive(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err = NewGpuCagra[Float16](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuCagra[Float16, Float16](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) case "IVF-Flat": bp := DefaultIvfFlatBuildParams() bp.NLists = 1000 - index, err = NewGpuIvfFlat[Float16](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuIvfFlat[float32, Float16](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) case "IVF-PQ": bp := DefaultIvfPqBuildParams() bp.NLists = 1000 bp.M = 16 - index, err = NewGpuIvfPq[Float16](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuIvfPq[Float16, Float16](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) } case "int8": dataset := make([]int8, n_vectors*uint64(dimension)) @@ -149,16 +149,20 @@ func TestIndexInfoComprehensive(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err = NewGpuCagra[int8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + // int8 is a STORAGE (quantization) type, not a base type: the [B,Q] + // model only supports a float base (f32/f16) quantized to int8/uint8. + // Base f32, storage int8 (the wired f32xint8 combo); dataset is the + // storage type Q (see NewGpuCagra signature). + index, err = NewGpuCagra[float32, int8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) case "IVF-Flat": bp := DefaultIvfFlatBuildParams() bp.NLists = 1000 - index, err = NewGpuIvfFlat[int8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuIvfFlat[float32, int8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) case "IVF-PQ": bp := DefaultIvfPqBuildParams() bp.NLists = 1000 bp.M = 16 - index, err = NewGpuIvfPq[int8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuIvfPq[float32, int8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) } case "uint8": dataset := make([]uint8, n_vectors*uint64(dimension)) @@ -171,16 +175,17 @@ func TestIndexInfoComprehensive(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err = NewGpuCagra[uint8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + // uint8 storage from a float base (wired f32xuint8 combo); see int8 note above. + index, err = NewGpuCagra[float32, uint8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) case "IVF-Flat": bp := DefaultIvfFlatBuildParams() bp.NLists = 1000 - index, err = NewGpuIvfFlat[uint8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuIvfFlat[float32, uint8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) case "IVF-PQ": bp := DefaultIvfPqBuildParams() bp.NLists = 1000 bp.M = 16 - index, err = NewGpuIvfPq[uint8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuIvfPq[float32, uint8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) } } diff --git a/pkg/cuvs/ivf_flat.go b/pkg/cuvs/ivf_flat.go index cdcd1afdbdf5a..5be75b7db9a32 100644 --- a/pkg/cuvs/ivf_flat.go +++ b/pkg/cuvs/ivf_flat.go @@ -32,7 +32,7 @@ import ( ) // GpuIvfFlat represents the C++ gpu_ivf_flat_t object. -type GpuIvfFlat[T VectorType] struct { +type GpuIvfFlat[B, Q VectorType] struct { cIvfFlat C.gpu_ivf_flat_c dimension uint32 nthread uint32 @@ -43,7 +43,7 @@ type GpuIvfFlat[T VectorType] struct { // SetBatchWindow sets the batching window in microseconds for search operations. // A window of 0 disables batching; any positive value enables batching with that delay. -func (gi *GpuIvfFlat[T]) SetBatchWindow(windowUs int64) error { +func (gi *GpuIvfFlat[B, Q]) SetBatchWindow(windowUs int64) error { gi.batchWindowUs = windowUs if gi.cIvfFlat != nil { var errmsg *C.char @@ -61,7 +61,7 @@ func (gi *GpuIvfFlat[T]) SetBatchWindow(windowUs int64) error { // flag. false (default): dispatch eagerly at the full batch size. true: wait for // the batch to fill or the window to elapse, then dispatch at the real size. // Has no effect unless the batch window is > 0. -func (gi *GpuIvfFlat[T]) SetDynbConservativeDispatch(enable bool) error { +func (gi *GpuIvfFlat[B, Q]) SetDynbConservativeDispatch(enable bool) error { gi.dynbConservativeDispatch = enable if gi.cIvfFlat != nil { var errmsg *C.char @@ -80,13 +80,14 @@ func (gi *GpuIvfFlat[T]) SetDynbConservativeDispatch(enable bool) error { // For Sharded mode the shard count is len(devices) (one shard per GPU); // to use fewer shards than the GPUs you have available, just pass a // shorter `devices` slice. -func NewGpuIvfFlat[T VectorType](dataset []T, count uint64, dimension uint32, metric DistanceType, - bp IvfFlatBuildParams, devices []int, nthread uint32, mode DistributionMode, ids []int64) (*GpuIvfFlat[T], error) { +func NewGpuIvfFlat[B, Q VectorType](dataset []Q, count uint64, dimension uint32, metric DistanceType, + bp IvfFlatBuildParams, devices []int, nthread uint32, mode DistributionMode, ids []int64) (*GpuIvfFlat[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cDevices := make([]C.int, len(devices)) for i, d := range devices { @@ -114,6 +115,7 @@ func NewGpuIvfFlat[T VectorType](dataset []T, count uint64, dimension uint32, me C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), cIds, unsafe.Pointer(&errmsg), @@ -132,7 +134,7 @@ func NewGpuIvfFlat[T VectorType](dataset []T, count uint64, dimension uint32, me return nil, moerr.NewInternalErrorNoCtx("failed to create GpuIvfFlat") } - return &GpuIvfFlat[T]{ + return &GpuIvfFlat[B, Q]{ cIvfFlat: cIvfFlat, dimension: dimension, nthread: nthread, @@ -141,13 +143,14 @@ func NewGpuIvfFlat[T VectorType](dataset []T, count uint64, dimension uint32, me } // NewGpuIvfFlatFromFile creates a new GpuIvfFlat instance by loading from a file. -func NewGpuIvfFlatFromFile[T VectorType](filename string, dimension uint32, metric DistanceType, - bp IvfFlatBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfFlat[T], error) { +func NewGpuIvfFlatFromFile[B, Q VectorType](filename string, dimension uint32, metric DistanceType, + bp IvfFlatBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfFlat[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cFilename := C.CString(filename) defer C.free(unsafe.Pointer(cFilename)) @@ -172,6 +175,7 @@ func NewGpuIvfFlatFromFile[T VectorType](filename string, dimension uint32, metr C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), unsafe.Pointer(&errmsg), ) @@ -187,7 +191,7 @@ func NewGpuIvfFlatFromFile[T VectorType](filename string, dimension uint32, metr return nil, moerr.NewInternalErrorNoCtx("failed to load GpuIvfFlat from file") } - return &GpuIvfFlat[T]{ + return &GpuIvfFlat[B, Q]{ cIvfFlat: cIvfFlat, dimension: dimension, nthread: nthread, @@ -199,8 +203,8 @@ func NewGpuIvfFlatFromFile[T VectorType](filename string, dimension uint32, metr // For Sharded loads we peek manifest.json to learn the saved shard count and // truncate `devices` to that count, so the C++ worker only spawns threads / // RMM pools on devices that will actually host a shard. -func NewGpuIvfFlatFromDataDirectory[T VectorType](dir string, dimension uint32, metric DistanceType, - bp IvfFlatBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfFlat[T], error) { +func NewGpuIvfFlatFromDataDirectory[B, Q VectorType](dir string, dimension uint32, metric DistanceType, + bp IvfFlatBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfFlat[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } @@ -211,7 +215,8 @@ func NewGpuIvfFlatFromDataDirectory[T VectorType](dir string, dimension uint32, return nil, err } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() cDevices := make([]C.int, len(devices)) for i, d := range devices { cDevices[i] = C.int(d) @@ -233,6 +238,7 @@ func NewGpuIvfFlatFromDataDirectory[T VectorType](dir string, dimension uint32, C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), nil, unsafe.Pointer(&errmsg), @@ -267,7 +273,7 @@ func NewGpuIvfFlatFromDataDirectory[T VectorType](dir string, dimension uint32, return nil, moerr.NewInternalErrorNoCtx(errStr) } - return &GpuIvfFlat[T]{ + return &GpuIvfFlat[B, Q]{ cIvfFlat: cIvfFlat, dimension: dimension, nthread: nthread, @@ -276,7 +282,7 @@ func NewGpuIvfFlatFromDataDirectory[T VectorType](dir string, dimension uint32, } // Destroy frees the C++ gpu_ivf_flat_t instance -func (gi *GpuIvfFlat[T]) Destroy() error { +func (gi *GpuIvfFlat[B, Q]) Destroy() error { if gi.cIvfFlat == nil { return nil } @@ -292,7 +298,7 @@ func (gi *GpuIvfFlat[T]) Destroy() error { } // Start initializes the worker and resources -func (gi *GpuIvfFlat[T]) Start() error { +func (gi *GpuIvfFlat[B, Q]) Start() error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -320,7 +326,7 @@ func (gi *GpuIvfFlat[T]) Start() error { } // Build triggers the build or file loading process -func (gi *GpuIvfFlat[T]) Build() error { +func (gi *GpuIvfFlat[B, Q]) Build() error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -335,13 +341,14 @@ func (gi *GpuIvfFlat[T]) Build() error { } // NewGpuIvfFlatEmpty creates a new GpuIvfFlat instance with pre-allocated buffer but no data yet. -func NewGpuIvfFlatEmpty[T VectorType](totalCount uint64, dimension uint32, metric DistanceType, - bp IvfFlatBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfFlat[T], error) { +func NewGpuIvfFlatEmpty[B, Q VectorType](totalCount uint64, dimension uint32, metric DistanceType, + bp IvfFlatBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfFlat[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cDevices := make([]C.int, len(devices)) for i, d := range devices { @@ -363,6 +370,7 @@ func NewGpuIvfFlatEmpty[T VectorType](totalCount uint64, dimension uint32, metri C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), nil, unsafe.Pointer(&errmsg), @@ -379,7 +387,7 @@ func NewGpuIvfFlatEmpty[T VectorType](totalCount uint64, dimension uint32, metri return nil, moerr.NewInternalErrorNoCtx("failed to create empty GpuIvfFlat") } - return &GpuIvfFlat[T]{ + return &GpuIvfFlat[B, Q]{ cIvfFlat: cIvfFlat, dimension: dimension, nthread: nthread, @@ -388,7 +396,7 @@ func NewGpuIvfFlatEmpty[T VectorType](totalCount uint64, dimension uint32, metri } // AddChunk adds a chunk of data to the pre-allocated buffer. -func (gi *GpuIvfFlat[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) error { +func (gi *GpuIvfFlat[B, Q]) AddChunk(chunk []Q, chunkCount uint64, ids []int64) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -419,8 +427,10 @@ func (gi *GpuIvfFlat[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) err return nil } -// AddChunkFloat adds a chunk of float32 data, performing on-the-fly quantization if needed. -func (gi *GpuIvfFlat[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []int64) error { +// AddChunkQuantize adds a chunk of base-typed (B) data, converting B -> the +// storage type Q on device (B==Q copy, or the learned/cast quantizer for a +// narrower Q). Mirrors GpuCagra/GpuIvfPq.AddChunkQuantize. +func (gi *GpuIvfFlat[B, Q]) AddChunkQuantize(chunk []B, chunkCount uint64, ids []int64) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -433,9 +443,9 @@ func (gi *GpuIvfFlat[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids [ if len(ids) > 0 { cIds = (*C.int64_t)(unsafe.Pointer(&ids[0])) } - C.gpu_ivf_flat_add_chunk_float( + C.gpu_ivf_flat_add_chunk_quantize( gi.cIvfFlat, - (*C.float)(&chunk[0]), + unsafe.Pointer(&chunk[0]), C.uint64_t(chunkCount), cIds, unsafe.Pointer(&errmsg), @@ -451,8 +461,8 @@ func (gi *GpuIvfFlat[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids [ return nil } -// TrainQuantizer trains the scalar quantizer (if T is 1-byte) -func (gi *GpuIvfFlat[T]) TrainQuantizer(trainData []float32, nSamples uint64) error { +// TrainQuantizer trains the scalar quantizer (if Q is 1-byte) +func (gi *GpuIvfFlat[B, Q]) TrainQuantizer(trainData []float32, nSamples uint64) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -477,8 +487,8 @@ func (gi *GpuIvfFlat[T]) TrainQuantizer(trainData []float32, nSamples uint64) er return nil } -// SetQuantizer sets the scalar quantizer parameters (if T is 1-byte) -func (gi *GpuIvfFlat[T]) SetQuantizer(min, max float32) error { +// SetQuantizer sets the scalar quantizer parameters (if Q is 1-byte) +func (gi *GpuIvfFlat[B, Q]) SetQuantizer(min, max float32) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -499,8 +509,8 @@ func (gi *GpuIvfFlat[T]) SetQuantizer(min, max float32) error { return nil } -// GetQuantizer gets the scalar quantizer parameters (if T is 1-byte) -func (gi *GpuIvfFlat[T]) GetQuantizer() (float32, float32, error) { +// GetQuantizer gets the scalar quantizer parameters (if Q is 1-byte) +func (gi *GpuIvfFlat[B, Q]) GetQuantizer() (float32, float32, error) { if gi.cIvfFlat == nil { return 0, 0, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -523,7 +533,7 @@ func (gi *GpuIvfFlat[T]) GetQuantizer() (float32, float32, error) { } // Save serializes the index to a file -func (gi *GpuIvfFlat[T]) Save(filename string) error { +func (gi *GpuIvfFlat[B, Q]) Save(filename string) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -541,7 +551,7 @@ func (gi *GpuIvfFlat[T]) Save(filename string) error { } // Pack saves the index to a .tar or .tar.gz file using save_dir. -func (gi *GpuIvfFlat[T]) Pack(filename string) error { +func (gi *GpuIvfFlat[B, Q]) Pack(filename string) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -570,7 +580,7 @@ func (gi *GpuIvfFlat[T]) Pack(filename string) error { // mode overrides the distribution mode at load time — pass Replicated to broadcast // a SINGLE_GPU .tar to all GPUs without rebuilding. // The index must already be initialized and started before calling Unpack. -func (gi *GpuIvfFlat[T]) Unpack(filename string, mode DistributionMode) error { +func (gi *GpuIvfFlat[B, Q]) Unpack(filename string, mode DistributionMode) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -599,7 +609,7 @@ func (gi *GpuIvfFlat[T]) Unpack(filename string, mode DistributionMode) error { } // DeleteId removes an ID from the index (soft delete). -func (gi *GpuIvfFlat[T]) DeleteId(id int64) error { +func (gi *GpuIvfFlat[B, Q]) DeleteId(id int64) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -614,7 +624,7 @@ func (gi *GpuIvfFlat[T]) DeleteId(id int64) error { } // Search performs a K-Nearest Neighbor search -func (gi *GpuIvfFlat[T]) Search(queries []T, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) (SearchResultIvfFlat, error) { +func (gi *GpuIvfFlat[B, Q]) Search(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) (SearchResultIvfFlat, error) { if gi.cIvfFlat == nil { return SearchResultIvfFlat{}, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -666,7 +676,7 @@ func (gi *GpuIvfFlat[T]) Search(queries []T, numQueries uint64, dimension uint32 } // SearchFloat performs a K-Nearest Neighbor search with float32 queries -func (gi *GpuIvfFlat[T]) SearchFloat(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) (SearchResultIvfFlat, error) { +func (gi *GpuIvfFlat[B, Q]) SearchQuantize(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) (SearchResultIvfFlat, error) { if gi.cIvfFlat == nil { return SearchResultIvfFlat{}, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -679,9 +689,9 @@ func (gi *GpuIvfFlat[T]) SearchFloat(queries []float32, numQueries uint64, dimen n_probes: C.uint32_t(sp.NProbes), } - res := C.gpu_ivf_flat_search_float( + res := C.gpu_ivf_flat_search_quantize( gi.cIvfFlat, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -718,12 +728,12 @@ func (gi *GpuIvfFlat[T]) SearchFloat(queries []float32, numQueries uint64, dimen } // SearchAsync performs a K-Nearest Neighbor search asynchronously. -func (gi *GpuIvfFlat[T]) SearchAsync(queries []T, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { +func (gi *GpuIvfFlat[B, Q]) SearchAsync(queries []Q, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { return gi.SearchAsyncWithParams(queries, numQueries, dimension, limit, DefaultIvfFlatSearchParams()) } // SearchAsyncWithParams performs a K-Nearest Neighbor search asynchronously with custom parameters. -func (gi *GpuIvfFlat[T]) SearchAsyncWithParams(queries []T, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) (uint64, error) { +func (gi *GpuIvfFlat[B, Q]) SearchAsyncWithParams(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) (uint64, error) { if gi.cIvfFlat == nil { return 0, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -756,13 +766,10 @@ func (gi *GpuIvfFlat[T]) SearchAsyncWithParams(queries []T, numQueries uint64, d return uint64(jobID), nil } -// SearchFloat32Async performs a K-Nearest Neighbor search with float32 queries asynchronously. -func (gi *GpuIvfFlat[T]) SearchFloat32Async(queries []float32, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { - return gi.SearchFloat32AsyncWithParams(queries, numQueries, dimension, limit, DefaultIvfFlatSearchParams()) -} - -// SearchFloat32AsyncWithParams performs a K-Nearest Neighbor search with float32 queries asynchronously with custom parameters. -func (gi *GpuIvfFlat[T]) SearchFloat32AsyncWithParams(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) (uint64, error) { +// SearchQuantizeAsyncWithParams submits an async KNN search with a base-typed (B) +// query; the index converts B to its storage type Q on device (B==Q copy, or the +// learned/cast quantizer for narrower Q). Unifies the former float32 and half query paths. +func (gi *GpuIvfFlat[B, Q]) SearchQuantizeAsyncWithParams(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) (uint64, error) { if gi.cIvfFlat == nil { return 0, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -775,9 +782,9 @@ func (gi *GpuIvfFlat[T]) SearchFloat32AsyncWithParams(queries []float32, numQuer n_probes: C.uint32_t(sp.NProbes), } - jobID := C.gpu_ivf_flat_search_float_async( + jobID := C.gpu_ivf_flat_search_quantize_async( gi.cIvfFlat, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -796,7 +803,7 @@ func (gi *GpuIvfFlat[T]) SearchFloat32AsyncWithParams(queries []float32, numQuer } // SearchWait waits for an asynchronous search to complete and returns the results. -func (gi *GpuIvfFlat[T]) SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) { +func (gi *GpuIvfFlat[B, Q]) SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) { if gi.cIvfFlat == nil { return nil, nil, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -829,7 +836,7 @@ func (gi *GpuIvfFlat[T]) SearchWait(jobID uint64, numQueries uint64, limit uint3 } // Cap returns the capacity of the index buffer -func (gi *GpuIvfFlat[T]) Cap() uint64 { +func (gi *GpuIvfFlat[B, Q]) Cap() uint64 { if gi.cIvfFlat == nil { return 0 } @@ -837,7 +844,7 @@ func (gi *GpuIvfFlat[T]) Cap() uint64 { } // Len returns current number of vectors in index -func (gi *GpuIvfFlat[T]) Len() uint64 { +func (gi *GpuIvfFlat[B, Q]) Len() uint64 { if gi.cIvfFlat == nil { return 0 } @@ -845,7 +852,7 @@ func (gi *GpuIvfFlat[T]) Len() uint64 { } // Info returns detailed information about the index as a JSON string. -func (gi *GpuIvfFlat[T]) Info() (string, error) { +func (gi *GpuIvfFlat[B, Q]) Info() (string, error) { if gi.cIvfFlat == nil { return "", moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -868,11 +875,11 @@ func (gi *GpuIvfFlat[T]) Info() (string, error) { } // GetCenters retrieves the trained centroids. -func (gi *GpuIvfFlat[T]) GetCenters(nLists uint32) ([]T, error) { +func (gi *GpuIvfFlat[B, Q]) GetCenters(nLists uint32) ([]Q, error) { if gi.cIvfFlat == nil { return nil, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } - centers := make([]T, nLists*gi.dimension) + centers := make([]Q, nLists*gi.dimension) var errmsg *C.char C.gpu_ivf_flat_get_centers(gi.cIvfFlat, unsafe.Pointer(¢ers[0]), unsafe.Pointer(&errmsg)) runtime.KeepAlive(centers) @@ -886,7 +893,7 @@ func (gi *GpuIvfFlat[T]) GetCenters(nLists uint32) ([]T, error) { } // GetNList retrieves the number of lists (centroids) in the index. -func (gi *GpuIvfFlat[T]) GetNList() uint32 { +func (gi *GpuIvfFlat[B, Q]) GetNList() uint32 { if gi.cIvfFlat == nil { return 0 } @@ -895,7 +902,7 @@ func (gi *GpuIvfFlat[T]) GetNList() uint32 { // Extend adds new vectors to an already-built index without rebuilding. // newIDs may be nil to auto-assign sequential IDs starting from the current index size. -func (gi *GpuIvfFlat[T]) Extend(newData []T, nRows uint64, newIDs []int64) error { +func (gi *GpuIvfFlat[B, Q]) Extend(newData []Q, nRows uint64, newIDs []int64) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -929,7 +936,7 @@ func (gi *GpuIvfFlat[T]) Extend(newData []T, nRows uint64, newIDs []int64) error // ExtendFloat adds new float32 vectors to an already-built index, quantizing on-the-fly if needed. // newIDs may be nil to auto-assign sequential IDs starting from the current index size. -func (gi *GpuIvfFlat[T]) ExtendFloat(newData []float32, nRows uint64, newIDs []int64) error { +func (gi *GpuIvfFlat[B, Q]) ExtendFloat(newData []float32, nRows uint64, newIDs []int64) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -968,7 +975,7 @@ type SearchResultIvfFlat struct { } // SetFilterColumns registers filter-column metadata. See GpuCagra.SetFilterColumns. -func (gi *GpuIvfFlat[T]) SetFilterColumns(colMetaJSON string, totalCount uint64) error { +func (gi *GpuIvfFlat[B, Q]) SetFilterColumns(colMetaJSON string, totalCount uint64) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -985,7 +992,7 @@ func (gi *GpuIvfFlat[T]) SetFilterColumns(colMetaJSON string, totalCount uint64) } // AddFilterChunk appends raw filter-column bytes. See GpuCagra.AddFilterChunk. -func (gi *GpuIvfFlat[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { +func (gi *GpuIvfFlat[B, Q]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -1016,7 +1023,7 @@ func (gi *GpuIvfFlat[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap [ } // SearchWithFilter runs a filtered K-NN search. predsJSON="" = unfiltered. -func (gi *GpuIvfFlat[T]) SearchWithFilter(queries []T, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams, predsJSON string) (SearchResultIvfFlat, error) { +func (gi *GpuIvfFlat[B, Q]) SearchWithFilter(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams, predsJSON string) (SearchResultIvfFlat, error) { if gi.cIvfFlat == nil { return SearchResultIvfFlat{}, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -1062,8 +1069,9 @@ func (gi *GpuIvfFlat[T]) SearchWithFilter(queries []T, numQueries uint64, dimens return SearchResultIvfFlat{Neighbors: neighbors, Distances: distances}, nil } -// SearchFloatWithFilter runs a filtered K-NN search with float32 queries. -func (gi *GpuIvfFlat[T]) SearchFloatWithFilter(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams, predsJSON string) (SearchResultIvfFlat, error) { +// SearchQuantizeWithFilter runs a filtered K-NN search with base-typed (B) +// queries; the index converts B to storage T (copy / quantize / f32->f16 cast). +func (gi *GpuIvfFlat[B, Q]) SearchQuantizeWithFilter(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams, predsJSON string) (SearchResultIvfFlat, error) { if gi.cIvfFlat == nil { return SearchResultIvfFlat{}, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -1076,9 +1084,9 @@ func (gi *GpuIvfFlat[T]) SearchFloatWithFilter(queries []float32, numQueries uin cPreds := C.CString(predsJSON) defer C.free(unsafe.Pointer(cPreds)) - res := C.gpu_ivf_flat_search_float_with_filter( + res := C.gpu_ivf_flat_search_quantize_with_filter( gi.cIvfFlat, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -1109,12 +1117,12 @@ func (gi *GpuIvfFlat[T]) SearchFloatWithFilter(queries []float32, numQueries uin return SearchResultIvfFlat{Neighbors: neighbors, Distances: distances}, nil } -// SearchFloatWithFilterAsync submits a filtered float32 K-NN search and -// returns a job_id; collect the result with SearchWait. Mirrors -// SearchFloat32AsyncWithParams + the predicate-eval semantics of -// SearchFloatWithFilter. Used by MultiGpuIvfFlat to dispatch per-shard -// filtered searches in parallel. -func (gi *GpuIvfFlat[T]) SearchFloatWithFilterAsync(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams, predsJSON string) (uint64, error) { +// SearchQuantizeWithFilterAsync submits a filtered K-NN search with base-typed +// (B) queries and returns a job_id; collect the result with SearchWait. Mirrors +// SearchQuantizeAsyncWithParams + the predicate-eval semantics of +// SearchQuantizeWithFilter. Used by MultiGpuIvfFlat to dispatch per-shard +// filtered searches in parallel. The index converts B to storage T. +func (gi *GpuIvfFlat[B, Q]) SearchQuantizeWithFilterAsync(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams, predsJSON string) (uint64, error) { if gi.cIvfFlat == nil { return 0, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -1127,9 +1135,9 @@ func (gi *GpuIvfFlat[T]) SearchFloatWithFilterAsync(queries []float32, numQuerie cPreds := C.CString(predsJSON) defer C.free(unsafe.Pointer(cPreds)) - jobID := C.gpu_ivf_flat_search_float_with_filter_async( + jobID := C.gpu_ivf_flat_search_quantize_with_filter_async( gi.cIvfFlat, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), diff --git a/pkg/cuvs/ivf_flat_test.go b/pkg/cuvs/ivf_flat_test.go index 9799e79079cee..20345cfed8212 100644 --- a/pkg/cuvs/ivf_flat_test.go +++ b/pkg/cuvs/ivf_flat_test.go @@ -35,7 +35,7 @@ func TestGpuIvfFlat(t *testing.T) { devices := []int{0} bp := DefaultIvfFlatBuildParams() bp.NLists = 10 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfFlat: %v", err) } @@ -81,7 +81,7 @@ func TestGpuIvfFlatSaveLoad(t *testing.T) { devices := []int{0} bp := DefaultIvfFlatBuildParams() bp.NLists = 2 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfFlat: %v", err) } @@ -96,7 +96,7 @@ func TestGpuIvfFlatSaveLoad(t *testing.T) { defer os.Remove(filename) index.Destroy() - index2, err := NewGpuIvfFlatFromFile[float32](filename, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index2, err := NewGpuIvfFlatFromFile[float32, float32](filename, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("Failed to create GpuIvfFlat from file: %v", err) } @@ -131,7 +131,7 @@ func TestGpuIvfFlatPackUnpack(t *testing.T) { devices := []int{0} bp := DefaultIvfFlatBuildParams() bp.NLists = 10 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfFlat: %v", err) } @@ -147,7 +147,7 @@ func TestGpuIvfFlatPackUnpack(t *testing.T) { } defer os.Remove(filename) - index2, err := NewGpuIvfFlatEmpty[float32](0, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index2, err := NewGpuIvfFlatEmpty[float32, float32](0, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("NewGpuIvfFlatEmpty failed: %v", err) } @@ -185,7 +185,7 @@ func TestGpuIvfFlatFromDataDirectory(t *testing.T) { devices := []int{0} bp := DefaultIvfFlatBuildParams() bp.NLists = 10 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfFlat: %v", err) } @@ -211,7 +211,7 @@ func TestGpuIvfFlatFromDataDirectory(t *testing.T) { t.Fatalf("Unpack to dir failed: %v", err) } - index2, err := NewGpuIvfFlatFromDataDirectory[float32](tmpDir, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index2, err := NewGpuIvfFlatFromDataDirectory[float32, float32](tmpDir, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("NewGpuIvfFlatFromDataDirectory failed: %v", err) } @@ -244,7 +244,7 @@ func TestGpuShardedIvfFlat(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 10 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Sharded, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Sharded, nil) if err != nil { t.Fatalf("Failed to create sharded IVF-Flat: %v", err) } @@ -281,7 +281,7 @@ func TestGpuReplicatedIvfFlat(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 10 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Replicated, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Replicated, nil) if err != nil { t.Fatalf("Failed to create replicated IVF-Flat: %v", err) } @@ -316,7 +316,7 @@ func TestGpuIvfFlatExtend(t *testing.T) { devices := []int{0} bp := DefaultIvfFlatBuildParams() bp.NLists = 10 - index, err := NewGpuIvfFlat[float32](dataset, nBase, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, nBase, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfFlat: %v", err) } @@ -381,7 +381,7 @@ func TestGpuIvfFlatExtendFloat(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 10 // Use Float16 so ExtendFloat exercises quantization - index, err := NewGpuIvfFlat[Float16](dataset, nBase, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, Float16](dataset, nBase, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfFlat[Float16]: %v", err) } @@ -409,7 +409,7 @@ func TestGpuIvfFlatExtendFloat(t *testing.T) { sp := DefaultIvfFlatSearchParams() sp.NProbes = 10 - r, err := index.SearchFloat([]float32{500, 500}, 1, dimension, 1, sp) + r, err := index.SearchQuantize([]float32{500, 500}, 1, dimension, 1, sp) if err != nil { t.Fatalf("Search failed: %v", err) } @@ -435,7 +435,7 @@ func TestGpuIvfFlatDeleteId(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 10 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfFlat: %v", err) } @@ -492,7 +492,7 @@ func TestGpuShardedIvfFlatDeleteId(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 10 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Sharded, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Sharded, nil) if err != nil { t.Fatalf("Failed to create sharded IvfFlat: %v", err) } @@ -572,7 +572,7 @@ func BenchmarkGpuShardedIvfFlat(b *testing.B) { bp := DefaultIvfFlatBuildParams() bp.NLists = 1000 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Sharded, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Sharded, nil) if err != nil { b.Fatalf("Failed to create sharded IVF-Flat: %v", err) } @@ -601,7 +601,7 @@ func BenchmarkGpuShardedIvfFlat(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -609,7 +609,7 @@ func BenchmarkGpuShardedIvfFlat(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(n_vectors), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -632,7 +632,7 @@ func BenchmarkGpuSingleIvfFlat(b *testing.B) { bp := DefaultIvfFlatBuildParams() bp.NLists = 1000 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, SingleGpu, nil) if err != nil { b.Fatalf("Failed to create single IVF-Flat: %v", err) } @@ -661,7 +661,7 @@ func BenchmarkGpuSingleIvfFlat(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -669,7 +669,7 @@ func BenchmarkGpuSingleIvfFlat(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(n_vectors), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -695,7 +695,7 @@ func BenchmarkGpuReplicatedIvfFlat(b *testing.B) { bp := DefaultIvfFlatBuildParams() bp.NLists = 1000 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Replicated, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Replicated, nil) if err != nil { b.Fatalf("Failed to create replicated IVF-Flat: %v", err) } @@ -724,7 +724,7 @@ func BenchmarkGpuReplicatedIvfFlat(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -732,7 +732,7 @@ func BenchmarkGpuReplicatedIvfFlat(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(n_vectors), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -757,7 +757,7 @@ func BenchmarkGpuAddChunkAndSearchIvfFlatF16(b *testing.B) { bp := DefaultIvfFlatBuildParams() bp.NLists = 1000 // Use Float16 as internal type - index, err := NewGpuIvfFlatEmpty[Float16](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) + index, err := NewGpuIvfFlatEmpty[float32, Float16](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) if err != nil { b.Fatalf("Failed to create index: %v", err) } @@ -770,7 +770,7 @@ func BenchmarkGpuAddChunkAndSearchIvfFlatF16(b *testing.B) { // Add data in chunks using AddChunkFloat for i := 0; i < totalCount; i += chunkSize { chunk := dataset[i*dimension : (i+chunkSize)*dimension] - if err := index.AddChunkFloat(chunk, uint64(chunkSize), nil); err != nil { + if err := index.AddChunkQuantize(chunk, uint64(chunkSize), nil); err != nil { b.Fatalf("AddChunkFloat failed at %d: %v", i, err) } } @@ -791,7 +791,7 @@ func BenchmarkGpuAddChunkAndSearchIvfFlatF16(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -799,7 +799,7 @@ func BenchmarkGpuAddChunkAndSearchIvfFlatF16(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(totalCount), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -822,7 +822,7 @@ func BenchmarkGpuAddChunkAndSearchIvfFlatInt8(b *testing.B) { bp := DefaultIvfFlatBuildParams() bp.NLists = 1000 // Use int8 as internal type - index, err := NewGpuIvfFlatEmpty[int8](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) + index, err := NewGpuIvfFlatEmpty[float32, int8](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) if err != nil { b.Fatalf("Failed to create index: %v", err) } @@ -835,7 +835,7 @@ func BenchmarkGpuAddChunkAndSearchIvfFlatInt8(b *testing.B) { // Add data in chunks using AddChunkFloat for i := 0; i < totalCount; i += chunkSize { chunk := dataset[i*dimension : (i+chunkSize)*dimension] - if err := index.AddChunkFloat(chunk, uint64(chunkSize), nil); err != nil { + if err := index.AddChunkQuantize(chunk, uint64(chunkSize), nil); err != nil { b.Fatalf("AddChunkFloat failed at %d: %v", i, err) } } @@ -856,7 +856,7 @@ func BenchmarkGpuAddChunkAndSearchIvfFlatInt8(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -864,7 +864,7 @@ func BenchmarkGpuAddChunkAndSearchIvfFlatInt8(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(totalCount), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -881,7 +881,7 @@ func TestGpuIvfFlatChunked(t *testing.T) { bp.NLists = 10 // Create empty index (target type int8) - index, err := NewGpuIvfFlatEmpty[int8](totalCount, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index, err := NewGpuIvfFlatEmpty[float32, int8](totalCount, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("Failed to create GpuIvfFlatEmpty: %v", err) } @@ -900,7 +900,7 @@ func TestGpuIvfFlatChunked(t *testing.T) { for j := range chunk { chunk[j] = val } - err = index.AddChunkFloat(chunk, chunkSize, nil) + err = index.AddChunkQuantize(chunk, chunkSize, nil) if err != nil { t.Fatalf("AddChunkFloat failed at offset %d: %v", i, err) } diff --git a/pkg/cuvs/ivf_pq.go b/pkg/cuvs/ivf_pq.go index e4bf899140de5..e55d5e7af8716 100644 --- a/pkg/cuvs/ivf_pq.go +++ b/pkg/cuvs/ivf_pq.go @@ -32,7 +32,7 @@ import ( ) // GpuIvfPq represents the C++ gpu_ivf_pq_t object. -type GpuIvfPq[T VectorType] struct { +type GpuIvfPq[B, Q VectorType] struct { cIvfPq C.gpu_ivf_pq_c dimension uint32 nthread uint32 @@ -43,7 +43,7 @@ type GpuIvfPq[T VectorType] struct { // SetBatchWindow sets the batching window in microseconds for search operations. // A window of 0 disables batching; any positive value enables batching with that delay. -func (gi *GpuIvfPq[T]) SetBatchWindow(windowUs int64) error { +func (gi *GpuIvfPq[B, Q]) SetBatchWindow(windowUs int64) error { gi.batchWindowUs = windowUs if gi.cIvfPq != nil { var errmsg *C.char @@ -61,7 +61,7 @@ func (gi *GpuIvfPq[T]) SetBatchWindow(windowUs int64) error { // flag. false (default): dispatch eagerly at the full batch size. true: wait for // the batch to fill or the window to elapse, then dispatch at the real size. // Has no effect unless the batch window is > 0. -func (gi *GpuIvfPq[T]) SetDynbConservativeDispatch(enable bool) error { +func (gi *GpuIvfPq[B, Q]) SetDynbConservativeDispatch(enable bool) error { gi.dynbConservativeDispatch = enable if gi.cIvfPq != nil { var errmsg *C.char @@ -77,13 +77,14 @@ func (gi *GpuIvfPq[T]) SetDynbConservativeDispatch(enable bool) error { // NewGpuIvfPq creates a new GpuIvfPq instance from a dataset. // ids may be nil to use internal sequential IDs (0..count-1). -func NewGpuIvfPq[T VectorType](dataset []T, count uint64, dimension uint32, metric DistanceType, - bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode, ids []int64) (*GpuIvfPq[T], error) { +func NewGpuIvfPq[B, Q VectorType](dataset []Q, count uint64, dimension uint32, metric DistanceType, + bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode, ids []int64) (*GpuIvfPq[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cDevices := make([]C.int, len(devices)) for i, d := range devices { @@ -113,6 +114,7 @@ func NewGpuIvfPq[T VectorType](dataset []T, count uint64, dimension uint32, metr C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), cIds, unsafe.Pointer(&errmsg), @@ -131,7 +133,7 @@ func NewGpuIvfPq[T VectorType](dataset []T, count uint64, dimension uint32, metr return nil, moerr.NewInternalErrorNoCtx("failed to create GpuIvfPq") } - return &GpuIvfPq[T]{ + return &GpuIvfPq[B, Q]{ cIvfPq: cIvfPq, dimension: dimension, nthread: nthread, @@ -140,13 +142,14 @@ func NewGpuIvfPq[T VectorType](dataset []T, count uint64, dimension uint32, metr } // NewGpuIvfPqFromDataFile creates a new GpuIvfPq instance from a MODF datafile. -func NewGpuIvfPqFromDataFile[T VectorType](datafilename string, metric DistanceType, - bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfPq[T], error) { +func NewGpuIvfPqFromDataFile[B, Q VectorType](datafilename string, metric DistanceType, + bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfPq[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cFilename := C.CString(datafilename) defer C.free(unsafe.Pointer(cFilename)) @@ -172,6 +175,7 @@ func NewGpuIvfPqFromDataFile[T VectorType](datafilename string, metric DistanceT C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), unsafe.Pointer(&errmsg), ) @@ -189,7 +193,7 @@ func NewGpuIvfPqFromDataFile[T VectorType](datafilename string, metric DistanceT // dimension will be updated when GetDim() is called, but we can set it to 0 for now // or ideally GetDim() should be used. - return &GpuIvfPq[T]{ + return &GpuIvfPq[B, Q]{ cIvfPq: cIvfPq, dimension: 0, nthread: nthread, @@ -198,13 +202,14 @@ func NewGpuIvfPqFromDataFile[T VectorType](datafilename string, metric DistanceT } // NewGpuIvfPqEmpty creates a new GpuIvfPq instance with pre-allocated buffer but no data yet. -func NewGpuIvfPqEmpty[T VectorType](totalCount uint64, dimension uint32, metric DistanceType, - bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfPq[T], error) { +func NewGpuIvfPqEmpty[B, Q VectorType](totalCount uint64, dimension uint32, metric DistanceType, + bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfPq[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cDevices := make([]C.int, len(devices)) for i, d := range devices { @@ -228,6 +233,7 @@ func NewGpuIvfPqEmpty[T VectorType](totalCount uint64, dimension uint32, metric C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), nil, unsafe.Pointer(&errmsg), @@ -244,7 +250,7 @@ func NewGpuIvfPqEmpty[T VectorType](totalCount uint64, dimension uint32, metric return nil, moerr.NewInternalErrorNoCtx("failed to create empty GpuIvfPq") } - return &GpuIvfPq[T]{ + return &GpuIvfPq[B, Q]{ cIvfPq: cIvfPq, dimension: dimension, nthread: nthread, @@ -253,7 +259,7 @@ func NewGpuIvfPqEmpty[T VectorType](totalCount uint64, dimension uint32, metric } // AddChunk adds a chunk of data to the pre-allocated buffer. -func (gi *GpuIvfPq[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) error { +func (gi *GpuIvfPq[B, Q]) AddChunk(chunk []Q, chunkCount uint64, ids []int64) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -284,8 +290,10 @@ func (gi *GpuIvfPq[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) error return nil } -// AddChunkFloat adds a chunk of float32 data, performing on-the-fly quantization if needed. -func (gi *GpuIvfPq[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []int64) error { +// AddChunkQuantize adds a chunk of base-typed (B) data, quantizing natively to +// the storage type Q (int8/uint8) via the B-source quantizer. base_data is the +// raw bytes of chunkCount*dim B-typed elements. No f32 detour. +func (gi *GpuIvfPq[B, Q]) AddChunkQuantize(chunk []B, chunkCount uint64, ids []int64) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -298,9 +306,9 @@ func (gi *GpuIvfPq[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []i if len(ids) > 0 { cIds = (*C.int64_t)(unsafe.Pointer(&ids[0])) } - C.gpu_ivf_pq_add_chunk_float( + C.gpu_ivf_pq_add_chunk_quantize( gi.cIvfPq, - (*C.float)(&chunk[0]), + unsafe.Pointer(&chunk[0]), C.uint64_t(chunkCount), cIds, unsafe.Pointer(&errmsg), @@ -316,8 +324,9 @@ func (gi *GpuIvfPq[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []i return nil } -// TrainQuantizer trains the scalar quantizer (if T is 1-byte) -func (gi *GpuIvfPq[T]) TrainQuantizer(trainData []float32, nSamples uint64) error { +// TrainQuantizer trains the scalar quantizer (if Q is 1-byte) from base-typed +// (B) training data. +func (gi *GpuIvfPq[B, Q]) TrainQuantizer(trainData []B, nSamples uint64) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -328,7 +337,7 @@ func (gi *GpuIvfPq[T]) TrainQuantizer(trainData []float32, nSamples uint64) erro var errmsg *C.char C.gpu_ivf_pq_train_quantizer( gi.cIvfPq, - (*C.float)(&trainData[0]), + unsafe.Pointer(&trainData[0]), C.uint64_t(nSamples), unsafe.Pointer(&errmsg), ) @@ -343,7 +352,7 @@ func (gi *GpuIvfPq[T]) TrainQuantizer(trainData []float32, nSamples uint64) erro } // SetQuantizer sets the scalar quantizer parameters (if T is 1-byte) -func (gi *GpuIvfPq[T]) SetQuantizer(min, max float32) error { +func (gi *GpuIvfPq[B, Q]) SetQuantizer(min, max float32) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -365,7 +374,7 @@ func (gi *GpuIvfPq[T]) SetQuantizer(min, max float32) error { } // GetQuantizer gets the scalar quantizer parameters (if T is 1-byte) -func (gi *GpuIvfPq[T]) GetQuantizer() (float32, float32, error) { +func (gi *GpuIvfPq[B, Q]) GetQuantizer() (float32, float32, error) { if gi.cIvfPq == nil { return 0, 0, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -388,13 +397,14 @@ func (gi *GpuIvfPq[T]) GetQuantizer() (float32, float32, error) { } // NewGpuIvfPqFromFile creates a new GpuIvfPq instance by loading from a file. -func NewGpuIvfPqFromFile[T VectorType](filename string, dimension uint32, metric DistanceType, - bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfPq[T], error) { +func NewGpuIvfPqFromFile[B, Q VectorType](filename string, dimension uint32, metric DistanceType, + bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfPq[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cFilename := C.CString(filename) defer C.free(unsafe.Pointer(cFilename)) @@ -421,6 +431,7 @@ func NewGpuIvfPqFromFile[T VectorType](filename string, dimension uint32, metric C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), unsafe.Pointer(&errmsg), ) @@ -436,7 +447,7 @@ func NewGpuIvfPqFromFile[T VectorType](filename string, dimension uint32, metric return nil, moerr.NewInternalErrorNoCtx("failed to load GpuIvfPq from file") } - return &GpuIvfPq[T]{ + return &GpuIvfPq[B, Q]{ cIvfPq: cIvfPq, dimension: dimension, nthread: nthread, @@ -448,8 +459,8 @@ func NewGpuIvfPqFromFile[T VectorType](filename string, dimension uint32, metric // For Sharded loads we peek manifest.json to learn the saved shard count and // truncate `devices` to that count, so the C++ worker only spawns threads / // RMM pools on devices that will actually host a shard. -func NewGpuIvfPqFromDataDirectory[T VectorType](dir string, dimension uint32, metric DistanceType, - bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfPq[T], error) { +func NewGpuIvfPqFromDataDirectory[B, Q VectorType](dir string, dimension uint32, metric DistanceType, + bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfPq[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } @@ -460,7 +471,8 @@ func NewGpuIvfPqFromDataDirectory[T VectorType](dir string, dimension uint32, me return nil, err } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() cDevices := make([]C.int, len(devices)) for i, d := range devices { cDevices[i] = C.int(d) @@ -484,6 +496,7 @@ func NewGpuIvfPqFromDataDirectory[T VectorType](dir string, dimension uint32, me C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), nil, unsafe.Pointer(&errmsg), @@ -518,7 +531,7 @@ func NewGpuIvfPqFromDataDirectory[T VectorType](dir string, dimension uint32, me return nil, moerr.NewInternalErrorNoCtx(errStr) } - return &GpuIvfPq[T]{ + return &GpuIvfPq[B, Q]{ cIvfPq: cIvfPq, dimension: dimension, nthread: nthread, @@ -527,7 +540,7 @@ func NewGpuIvfPqFromDataDirectory[T VectorType](dir string, dimension uint32, me } // Destroy frees the C++ gpu_ivf_pq_t instance -func (gi *GpuIvfPq[T]) Destroy() error { +func (gi *GpuIvfPq[B, Q]) Destroy() error { if gi.cIvfPq == nil { return nil } @@ -543,7 +556,7 @@ func (gi *GpuIvfPq[T]) Destroy() error { } // Start initializes the worker and resources -func (gi *GpuIvfPq[T]) Start() error { +func (gi *GpuIvfPq[B, Q]) Start() error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -571,7 +584,7 @@ func (gi *GpuIvfPq[T]) Start() error { } // Build triggers the build or file loading process -func (gi *GpuIvfPq[T]) Build() error { +func (gi *GpuIvfPq[B, Q]) Build() error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -586,7 +599,7 @@ func (gi *GpuIvfPq[T]) Build() error { } // Save serializes the index to a file -func (gi *GpuIvfPq[T]) Save(filename string) error { +func (gi *GpuIvfPq[B, Q]) Save(filename string) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -604,7 +617,7 @@ func (gi *GpuIvfPq[T]) Save(filename string) error { } // Pack saves the index to a .tar or .tar.gz file using save_dir. -func (gi *GpuIvfPq[T]) Pack(filename string) error { +func (gi *GpuIvfPq[B, Q]) Pack(filename string) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -633,7 +646,7 @@ func (gi *GpuIvfPq[T]) Pack(filename string) error { // mode overrides the distribution mode at load time — pass Replicated to broadcast // a SINGLE_GPU .tar to all GPUs without rebuilding. // The index must already be initialized and started before calling Unpack. -func (gi *GpuIvfPq[T]) Unpack(filename string, mode DistributionMode) error { +func (gi *GpuIvfPq[B, Q]) Unpack(filename string, mode DistributionMode) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -662,7 +675,7 @@ func (gi *GpuIvfPq[T]) Unpack(filename string, mode DistributionMode) error { } // DeleteId removes an ID from the index (soft delete). -func (gi *GpuIvfPq[T]) DeleteId(id int64) error { +func (gi *GpuIvfPq[B, Q]) DeleteId(id int64) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -678,7 +691,7 @@ func (gi *GpuIvfPq[T]) DeleteId(id int64) error { // DeleteIds applies DeleteId in a loop. See cagra.GpuCagra.DeleteIds for // the rationale. -func (gi *GpuIvfPq[T]) DeleteIds(ids []int64) error { +func (gi *GpuIvfPq[B, Q]) DeleteIds(ids []int64) error { for _, id := range ids { if err := gi.DeleteId(id); err != nil { return err @@ -688,7 +701,7 @@ func (gi *GpuIvfPq[T]) DeleteIds(ids []int64) error { } // Search performs a K-Nearest Neighbor search -func (gi *GpuIvfPq[T]) Search(queries []T, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) (SearchResultIvfPq, error) { +func (gi *GpuIvfPq[B, Q]) Search(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) (SearchResultIvfPq, error) { if gi.cIvfPq == nil { return SearchResultIvfPq{}, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -740,7 +753,7 @@ func (gi *GpuIvfPq[T]) Search(queries []T, numQueries uint64, dimension uint32, } // SearchFloat performs an IVF-PQ search operation with float32 queries -func (gi *GpuIvfPq[T]) SearchFloat(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) (SearchResultIvfPq, error) { +func (gi *GpuIvfPq[B, Q]) SearchQuantize(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) (SearchResultIvfPq, error) { if gi.cIvfPq == nil { return SearchResultIvfPq{}, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -753,9 +766,9 @@ func (gi *GpuIvfPq[T]) SearchFloat(queries []float32, numQueries uint64, dimensi n_probes: C.uint32_t(sp.NProbes), } - res := C.gpu_ivf_pq_search_float( + res := C.gpu_ivf_pq_search_quantize( gi.cIvfPq, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -792,12 +805,12 @@ func (gi *GpuIvfPq[T]) SearchFloat(queries []float32, numQueries uint64, dimensi } // SearchAsync performs a K-Nearest Neighbor search asynchronously. -func (gi *GpuIvfPq[T]) SearchAsync(queries []T, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { +func (gi *GpuIvfPq[B, Q]) SearchAsync(queries []Q, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { return gi.SearchAsyncWithParams(queries, numQueries, dimension, limit, DefaultIvfPqSearchParams()) } // SearchAsyncWithParams performs a K-Nearest Neighbor search asynchronously with custom parameters. -func (gi *GpuIvfPq[T]) SearchAsyncWithParams(queries []T, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) (uint64, error) { +func (gi *GpuIvfPq[B, Q]) SearchAsyncWithParams(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) (uint64, error) { if gi.cIvfPq == nil { return 0, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -830,13 +843,10 @@ func (gi *GpuIvfPq[T]) SearchAsyncWithParams(queries []T, numQueries uint64, dim return uint64(jobID), nil } -// SearchFloat32Async performs a K-Nearest Neighbor search with float32 queries asynchronously. -func (gi *GpuIvfPq[T]) SearchFloat32Async(queries []float32, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { - return gi.SearchFloat32AsyncWithParams(queries, numQueries, dimension, limit, DefaultIvfPqSearchParams()) -} - -// SearchFloat32AsyncWithParams performs a K-Nearest Neighbor search with float32 queries asynchronously with custom parameters. -func (gi *GpuIvfPq[T]) SearchFloat32AsyncWithParams(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) (uint64, error) { +// SearchQuantizeAsyncWithParams submits an async KNN search with a base-typed (B) +// query; the index converts B to its storage type Q on device (B==Q copy, or the +// learned/cast quantizer for narrower Q). Unifies the former float32 and half query paths. +func (gi *GpuIvfPq[B, Q]) SearchQuantizeAsyncWithParams(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) (uint64, error) { if gi.cIvfPq == nil { return 0, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -849,9 +859,9 @@ func (gi *GpuIvfPq[T]) SearchFloat32AsyncWithParams(queries []float32, numQuerie n_probes: C.uint32_t(sp.NProbes), } - jobID := C.gpu_ivf_pq_search_float_async( + jobID := C.gpu_ivf_pq_search_quantize_async( gi.cIvfPq, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -870,7 +880,7 @@ func (gi *GpuIvfPq[T]) SearchFloat32AsyncWithParams(queries []float32, numQuerie } // SearchWait waits for an asynchronous search to complete and returns the results. -func (gi *GpuIvfPq[T]) SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) { +func (gi *GpuIvfPq[B, Q]) SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) { if gi.cIvfPq == nil { return nil, nil, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -903,7 +913,7 @@ func (gi *GpuIvfPq[T]) SearchWait(jobID uint64, numQueries uint64, limit uint32) } // Cap returns the capacity of the index buffer -func (gi *GpuIvfPq[T]) Cap() uint64 { +func (gi *GpuIvfPq[B, Q]) Cap() uint64 { if gi.cIvfPq == nil { return 0 } @@ -911,7 +921,7 @@ func (gi *GpuIvfPq[T]) Cap() uint64 { } // Len returns current number of vectors in index -func (gi *GpuIvfPq[T]) Len() uint64 { +func (gi *GpuIvfPq[B, Q]) Len() uint64 { if gi.cIvfPq == nil { return 0 } @@ -921,7 +931,7 @@ func (gi *GpuIvfPq[T]) Len() uint64 { // GetFilterColMetaJSON returns the INCLUDE-column metadata of the loaded // index as a JSON string ready to be re-fed into SetFilterColumns. Returns // "" for indexes that were built without INCLUDE columns. -func (gi *GpuIvfPq[T]) GetFilterColMetaJSON() string { +func (gi *GpuIvfPq[B, Q]) GetFilterColMetaJSON() string { if gi.cIvfPq == nil { return "" } @@ -939,7 +949,7 @@ func (gi *GpuIvfPq[T]) GetFilterColMetaJSON() string { } // Info returns detailed information about the index as a JSON string. -func (gi *GpuIvfPq[T]) Info() (string, error) { +func (gi *GpuIvfPq[B, Q]) Info() (string, error) { if gi.cIvfPq == nil { return "", moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -962,13 +972,13 @@ func (gi *GpuIvfPq[T]) Info() (string, error) { } // GetCenters retrieves the trained centroids. -func (gi *GpuIvfPq[T]) GetCenters() ([]T, error) { +func (gi *GpuIvfPq[B, Q]) GetCenters() ([]Q, error) { if gi.cIvfPq == nil { return nil, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } nList := gi.GetNList() dim := gi.GetRotDim() - centers := make([]T, nList*dim) + centers := make([]Q, nList*dim) var errmsg *C.char C.gpu_ivf_pq_get_centers(gi.cIvfPq, unsafe.Pointer(¢ers[0]), C.uint64_t(len(centers)), unsafe.Pointer(&errmsg)) runtime.KeepAlive(centers) @@ -982,7 +992,7 @@ func (gi *GpuIvfPq[T]) GetCenters() ([]T, error) { } // GetNList retrieves the number of lists (centroids) in the index. -func (gi *GpuIvfPq[T]) GetNList() uint32 { +func (gi *GpuIvfPq[B, Q]) GetNList() uint32 { if gi.cIvfPq == nil { return 0 } @@ -990,7 +1000,7 @@ func (gi *GpuIvfPq[T]) GetNList() uint32 { } // GetDim retrieves the dimension of the index. -func (gi *GpuIvfPq[T]) GetDim() uint32 { +func (gi *GpuIvfPq[B, Q]) GetDim() uint32 { if gi.cIvfPq == nil { return 0 } @@ -998,7 +1008,7 @@ func (gi *GpuIvfPq[T]) GetDim() uint32 { } // GetRotDim retrieves the rotated dimension of the index. -func (gi *GpuIvfPq[T]) GetRotDim() uint32 { +func (gi *GpuIvfPq[B, Q]) GetRotDim() uint32 { if gi.cIvfPq == nil { return 0 } @@ -1006,7 +1016,7 @@ func (gi *GpuIvfPq[T]) GetRotDim() uint32 { } // GetDimExt retrieves the extended dimension of the index (including norms and padding). -func (gi *GpuIvfPq[T]) GetDimExt() uint32 { +func (gi *GpuIvfPq[B, Q]) GetDimExt() uint32 { if gi.cIvfPq == nil { return 0 } @@ -1014,18 +1024,18 @@ func (gi *GpuIvfPq[T]) GetDimExt() uint32 { } // GetDataset retrieves the flattened host dataset (for debugging). -func (gi *GpuIvfPq[T]) GetDataset(totalElements uint64) []T { +func (gi *GpuIvfPq[B, Q]) GetDataset(totalElements uint64) []Q { if gi.cIvfPq == nil { return nil } - data := make([]T, totalElements) + data := make([]Q, totalElements) C.gpu_ivf_pq_get_dataset(gi.cIvfPq, unsafe.Pointer(&data[0])) return data } // Extend adds new vectors to an already-built index without rebuilding. // newIDs may be nil to auto-assign sequential IDs starting from the current index size. -func (gi *GpuIvfPq[T]) Extend(newData []T, nRows uint64, newIDs []int64) error { +func (gi *GpuIvfPq[B, Q]) Extend(newData []Q, nRows uint64, newIDs []int64) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -1059,7 +1069,7 @@ func (gi *GpuIvfPq[T]) Extend(newData []T, nRows uint64, newIDs []int64) error { // ExtendFloat adds new float32 vectors to an already-built index, quantizing on-the-fly if needed. // newIDs may be nil to auto-assign sequential IDs starting from the current index size. -func (gi *GpuIvfPq[T]) ExtendFloat(newData []float32, nRows uint64, newIDs []int64) error { +func (gi *GpuIvfPq[B, Q]) ExtendFloat(newData []float32, nRows uint64, newIDs []int64) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -1098,7 +1108,7 @@ type SearchResultIvfPq struct { } // SetFilterColumns registers filter-column metadata. See GpuCagra.SetFilterColumns. -func (gi *GpuIvfPq[T]) SetFilterColumns(colMetaJSON string, totalCount uint64) error { +func (gi *GpuIvfPq[B, Q]) SetFilterColumns(colMetaJSON string, totalCount uint64) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -1115,7 +1125,7 @@ func (gi *GpuIvfPq[T]) SetFilterColumns(colMetaJSON string, totalCount uint64) e } // AddFilterChunk appends raw filter-column bytes. See GpuCagra.AddFilterChunk. -func (gi *GpuIvfPq[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { +func (gi *GpuIvfPq[B, Q]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -1146,7 +1156,7 @@ func (gi *GpuIvfPq[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []u } // SearchWithFilter runs a filtered K-NN search. predsJSON="" = unfiltered. -func (gi *GpuIvfPq[T]) SearchWithFilter(queries []T, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams, predsJSON string) (SearchResultIvfPq, error) { +func (gi *GpuIvfPq[B, Q]) SearchWithFilter(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams, predsJSON string) (SearchResultIvfPq, error) { if gi.cIvfPq == nil { return SearchResultIvfPq{}, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -1192,8 +1202,9 @@ func (gi *GpuIvfPq[T]) SearchWithFilter(queries []T, numQueries uint64, dimensio return SearchResultIvfPq{Neighbors: neighbors, Distances: distances}, nil } -// SearchFloatWithFilter runs a filtered K-NN search with float32 queries. -func (gi *GpuIvfPq[T]) SearchFloatWithFilter(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams, predsJSON string) (SearchResultIvfPq, error) { +// SearchQuantizeWithFilter runs a filtered K-NN search with base-typed (B) +// queries; the index converts B to storage T (copy / quantize / f32->f16 cast). +func (gi *GpuIvfPq[B, Q]) SearchQuantizeWithFilter(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams, predsJSON string) (SearchResultIvfPq, error) { if gi.cIvfPq == nil { return SearchResultIvfPq{}, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -1206,9 +1217,9 @@ func (gi *GpuIvfPq[T]) SearchFloatWithFilter(queries []float32, numQueries uint6 cPreds := C.CString(predsJSON) defer C.free(unsafe.Pointer(cPreds)) - res := C.gpu_ivf_pq_search_float_with_filter( + res := C.gpu_ivf_pq_search_quantize_with_filter( gi.cIvfPq, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -1239,12 +1250,12 @@ func (gi *GpuIvfPq[T]) SearchFloatWithFilter(queries []float32, numQueries uint6 return SearchResultIvfPq{Neighbors: neighbors, Distances: distances}, nil } -// SearchFloatWithFilterAsync submits a filtered float32 K-NN search and -// returns a job_id; collect the result with SearchWait. Mirrors -// SearchFloat32AsyncWithParams + the predicate-eval semantics of -// SearchFloatWithFilter. Used by MultiGpuIvfPq to dispatch per-shard -// filtered searches in parallel. -func (gi *GpuIvfPq[T]) SearchFloatWithFilterAsync(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams, predsJSON string) (uint64, error) { +// SearchQuantizeWithFilterAsync submits a filtered K-NN search with base-typed +// (B) queries and returns a job_id; collect the result with SearchWait. Mirrors +// SearchQuantizeAsyncWithParams + the predicate-eval semantics of +// SearchQuantizeWithFilter. Used by MultiGpuIvfPq to dispatch per-shard +// filtered searches in parallel. The index converts B to storage T. +func (gi *GpuIvfPq[B, Q]) SearchQuantizeWithFilterAsync(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams, predsJSON string) (uint64, error) { if gi.cIvfPq == nil { return 0, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -1257,9 +1268,9 @@ func (gi *GpuIvfPq[T]) SearchFloatWithFilterAsync(queries []float32, numQueries cPreds := C.CString(predsJSON) defer C.free(unsafe.Pointer(cPreds)) - jobID := C.gpu_ivf_pq_search_float_with_filter_async( + jobID := C.gpu_ivf_pq_search_quantize_with_filter_async( gi.cIvfPq, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), diff --git a/pkg/cuvs/ivf_pq_test.go b/pkg/cuvs/ivf_pq_test.go index e6d79084c3102..8cd7301aa6c91 100644 --- a/pkg/cuvs/ivf_pq_test.go +++ b/pkg/cuvs/ivf_pq_test.go @@ -38,7 +38,7 @@ func TestGpuIvfPq(t *testing.T) { bp.NLists = 10 bp.M = 8 // dimension 16 is divisible by 8 bp.KmeansTrainsetFraction = 1.0 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfPq: %v", err) } @@ -92,7 +92,7 @@ func TestGpuIvfPqSaveLoad(t *testing.T) { bp.NLists = 10 bp.M = 2 bp.KmeansTrainsetFraction = 1.0 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfPq: %v", err) } @@ -107,7 +107,7 @@ func TestGpuIvfPqSaveLoad(t *testing.T) { defer os.Remove(filename) index.Destroy() - index2, err := NewGpuIvfPqFromFile[float32](filename, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index2, err := NewGpuIvfPqFromFile[float32, float32](filename, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("Failed to create GpuIvfPq from file: %v", err) } @@ -150,7 +150,7 @@ func TestGpuIvfPqPackUnpack(t *testing.T) { bp.NLists = 10 bp.M = 2 bp.KmeansTrainsetFraction = 1.0 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfPq: %v", err) } @@ -166,7 +166,7 @@ func TestGpuIvfPqPackUnpack(t *testing.T) { } defer os.Remove(filename) - index2, err := NewGpuIvfPqEmpty[float32](0, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index2, err := NewGpuIvfPqEmpty[float32, float32](0, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("NewGpuIvfPqEmpty failed: %v", err) } @@ -208,7 +208,7 @@ func TestGpuIvfPqFromDataDirectory(t *testing.T) { bp.NLists = 10 bp.M = 2 bp.KmeansTrainsetFraction = 1.0 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfPq: %v", err) } @@ -234,7 +234,7 @@ func TestGpuIvfPqFromDataDirectory(t *testing.T) { t.Fatalf("Unpack to dir failed: %v", err) } - index2, err := NewGpuIvfPqFromDataDirectory[float32](tmpDir, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index2, err := NewGpuIvfPqFromDataDirectory[float32, float32](tmpDir, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("NewGpuIvfPqFromDataDirectory failed: %v", err) } @@ -261,7 +261,7 @@ func TestGpuIvfPqChunked(t *testing.T) { bp.M = 4 // Create empty index (target type int8) - index, err := NewGpuIvfPqEmpty[int8](totalCount, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index, err := NewGpuIvfPqEmpty[float32, int8](totalCount, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("Failed to create GpuIvfPqEmpty: %v", err) } @@ -280,7 +280,7 @@ func TestGpuIvfPqChunked(t *testing.T) { for j := range chunk { chunk[j] = val } - err = index.AddChunkFloat(chunk, chunkSize, nil) + err = index.AddChunkQuantize(chunk, chunkSize, nil) if err != nil { t.Fatalf("AddChunkFloat failed at offset %d: %v", i, err) } @@ -343,7 +343,7 @@ func TestGpuShardedIvfPq(t *testing.T) { bp := DefaultIvfPqBuildParams() bp.NLists = 10 bp.M = 2 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Sharded, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Sharded, nil) if err != nil { t.Fatalf("Failed to create sharded IVF-PQ: %v", err) } @@ -385,7 +385,7 @@ func TestGpuReplicatedIvfPq(t *testing.T) { bp := DefaultIvfPqBuildParams() bp.NLists = 10 bp.M = 2 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Replicated, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Replicated, nil) if err != nil { t.Fatalf("Failed to create replicated IVF-PQ: %v", err) } @@ -423,7 +423,7 @@ func TestGpuIvfPqExtend(t *testing.T) { bp := DefaultIvfPqBuildParams() bp.NLists = 10 bp.M = 8 - index, err := NewGpuIvfPq[float32](dataset, nBase, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, nBase, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfPq: %v", err) } @@ -498,7 +498,7 @@ func TestGpuIvfPqExtendFloat(t *testing.T) { bp.M = 8 bp.KmeansTrainsetFraction = 1.0 // Use Float16 so ExtendFloat exercises quantization - index, err := NewGpuIvfPq[Float16](dataset, nBase, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfPq[Float16, Float16](dataset, nBase, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfPq[Float16]: %v", err) } @@ -532,14 +532,20 @@ func TestGpuIvfPqExtendFloat(t *testing.T) { sp := DefaultIvfPqSearchParams() sp.NProbes = 10 - // Query exactly at extended cluster; expect ID in [3000, 3050) - qExt := make([]float32, dimension) - for j := range qExt { - qExt[j] = extVal + // Query exactly at extended cluster; expect ID in [3000, 3050). This is a + // Float16-base index, so SearchQuantize takes a []Float16 query (the old + // SearchFloat's implicit f32->half is gone — convert explicitly). + qExtF32 := make([]float32, dimension) + for j := range qExtF32 { + qExtF32[j] = extVal } - r, err := index.SearchFloat(qExt, 1, dimension, 1, sp) + qExt := make([]Float16, dimension) + if err := GpuConvertF32ToF16(qExtF32, qExt, 0); err != nil { + t.Fatalf("convert query to f16: %v", err) + } + r, err := index.SearchQuantize(qExt, 1, dimension, 1, sp) if err != nil { - t.Fatalf("SearchFloat failed: %v", err) + t.Fatalf("SearchQuantize failed: %v", err) } if r.Neighbors[0] < 3000 || r.Neighbors[0] >= 3050 { t.Errorf("expected neighbor in [3000, 3050), got %d dist=%f", r.Neighbors[0], r.Distances[0]) @@ -565,7 +571,7 @@ func TestGpuIvfPqDeleteId(t *testing.T) { bp.NLists = 10 bp.M = 8 bp.KmeansTrainsetFraction = 1.0 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfPq: %v", err) } @@ -608,7 +614,7 @@ func TestGpuIvfPqDeleteId(t *testing.T) { } // 2. Test SearchFloat (this verifies the fix in search_float_internal) - r, err = index.SearchFloat(q50, 1, dimension, 1, sp) + r, err = index.SearchQuantize(q50, 1, dimension, 1, sp) if err != nil { t.Fatalf("SearchFloat failed: %v", err) } @@ -633,7 +639,7 @@ func BenchmarkGpuShardedIvfPq(b *testing.B) { bp := DefaultIvfPqBuildParams() bp.NLists = 1000 bp.M = 128 // 1024 / 8 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Sharded, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Sharded, nil) if err != nil { b.Fatalf("Failed to create sharded IVF-PQ: %v", err) } @@ -662,7 +668,7 @@ func BenchmarkGpuShardedIvfPq(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -670,7 +676,7 @@ func BenchmarkGpuShardedIvfPq(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(n_vectors), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -694,7 +700,7 @@ func BenchmarkGpuSingleIvfPq(b *testing.B) { bp := DefaultIvfPqBuildParams() bp.NLists = 1000 bp.M = 128 // 1024 / 8 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, SingleGpu, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, SingleGpu, nil) if err != nil { b.Fatalf("Failed to create single IVF-PQ: %v", err) } @@ -723,7 +729,7 @@ func BenchmarkGpuSingleIvfPq(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -731,7 +737,7 @@ func BenchmarkGpuSingleIvfPq(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(n_vectors), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -758,7 +764,7 @@ func BenchmarkGpuReplicatedIvfPq(b *testing.B) { bp := DefaultIvfPqBuildParams() bp.NLists = 1000 bp.M = 128 // 1024 / 8 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Replicated, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Replicated, nil) if err != nil { b.Fatalf("Failed to create replicated IVF-PQ: %v", err) } @@ -787,7 +793,7 @@ func BenchmarkGpuReplicatedIvfPq(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -795,7 +801,7 @@ func BenchmarkGpuReplicatedIvfPq(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(n_vectors), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -820,7 +826,7 @@ func BenchmarkGpuAddChunkAndSearchIvfPqF16(b *testing.B) { bp := DefaultIvfPqBuildParams() bp.NLists = 1000 // Use Float16 as internal type - index, err := NewGpuIvfPqEmpty[Float16](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) + index, err := NewGpuIvfPqEmpty[float32, Float16](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) if err != nil { b.Fatalf("Failed to create index: %v", err) } @@ -833,7 +839,7 @@ func BenchmarkGpuAddChunkAndSearchIvfPqF16(b *testing.B) { // Add data in chunks using AddChunkFloat for i := 0; i < totalCount; i += chunkSize { chunk := dataset[i*dimension : (i+chunkSize)*dimension] - if err := index.AddChunkFloat(chunk, uint64(chunkSize), nil); err != nil { + if err := index.AddChunkQuantize(chunk, uint64(chunkSize), nil); err != nil { b.Fatalf("AddChunkFloat failed at %d: %v", i, err) } } @@ -854,7 +860,7 @@ func BenchmarkGpuAddChunkAndSearchIvfPqF16(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -862,7 +868,7 @@ func BenchmarkGpuAddChunkAndSearchIvfPqF16(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(totalCount), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -884,7 +890,7 @@ func BenchmarkGpuAddChunkAndSearchIvfPqInt8(b *testing.B) { bp := DefaultIvfPqBuildParams() bp.NLists = 1000 // Use int8 as internal type - index, err := NewGpuIvfPqEmpty[int8](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) + index, err := NewGpuIvfPqEmpty[float32, int8](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) if err != nil { b.Fatalf("Failed to create index: %v", err) } @@ -897,7 +903,7 @@ func BenchmarkGpuAddChunkAndSearchIvfPqInt8(b *testing.B) { // Add data in chunks using AddChunkFloat for i := 0; i < totalCount; i += chunkSize { chunk := dataset[i*dimension : (i+chunkSize)*dimension] - if err := index.AddChunkFloat(chunk, uint64(chunkSize), nil); err != nil { + if err := index.AddChunkQuantize(chunk, uint64(chunkSize), nil); err != nil { b.Fatalf("AddChunkFloat failed at %d: %v", i, err) } } @@ -918,7 +924,7 @@ func BenchmarkGpuAddChunkAndSearchIvfPqInt8(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -926,7 +932,7 @@ func BenchmarkGpuAddChunkAndSearchIvfPqInt8(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(totalCount), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } diff --git a/pkg/cuvs/metric_support_test.go b/pkg/cuvs/metric_support_test.go index e3f124f69d189..688d4025c0b2b 100644 --- a/pkg/cuvs/metric_support_test.go +++ b/pkg/cuvs/metric_support_test.go @@ -104,7 +104,7 @@ func TestCagraMetricSupport(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 16 bp.GraphDegree = 8 - idx, err := NewGpuCagra[float32](ds, count, dim, mc.metric, bp, []int{0}, 1, SingleGpu, ids) + idx, err := NewGpuCagra[float32, float32](ds, count, dim, mc.metric, bp, []int{0}, 1, SingleGpu, ids) if err != nil { t.Fatalf("build CAGRA(%s): %v", mc.name, err) } @@ -140,7 +140,7 @@ func TestIvfPqMetricSupport(t *testing.T) { bp.M = 8 bp.BitsPerCode = 8 bp.KmeansTrainsetFraction = 1.0 - idx, err := NewGpuIvfPq[float32](ds, count, dim, mc.metric, bp, []int{0}, 1, SingleGpu, ids) + idx, err := NewGpuIvfPq[float32, float32](ds, count, dim, mc.metric, bp, []int{0}, 1, SingleGpu, ids) if err != nil { t.Fatalf("build IVF-PQ(%s): %v", mc.name, err) } diff --git a/pkg/cuvs/multi_index.go b/pkg/cuvs/multi_index.go index b76bcbc4cc30e..c6586d1b1cc45 100644 --- a/pkg/cuvs/multi_index.go +++ b/pkg/cuvs/multi_index.go @@ -23,16 +23,30 @@ import ( "github.com/matrixorigin/matrixone/pkg/vectorindex" ) +// BruteForceOverflow is the type-erased CDC overflow: the storage type OB is +// hidden so one field/helper can hold *GpuBruteForce[B, OB] for any OB — the +// index storage Q when it is float/half, else the base B (for int8/uint8 storage, +// which cuVS brute force cannot store). Queries are always the base type B; the +// overflow quantizes B -> OB inside cuVS, so OB never appears in any signature. +type BruteForceOverflow[B VectorType] interface { + SearchQuantizeAsync(queries []B, numQueries uint64, dimension uint32, limit uint32) (uint64, error) + SearchQuantizeWithFilterAsync(queries []B, numQueries uint64, dimension uint32, limit uint32, predsJSON string) (uint64, error) + SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) + Cap() uint64 + Len() uint64 + Destroy() error +} + // MultiGpuIndex manages multiple GpuIndex instances and performs search across all of them using default parameters. type MultiGpuIndex[T VectorType] struct { indices []GpuIndex[T] - bruteForce *GpuBruteForce[T] + bruteForce *GpuBruteForce[T, T] dimension uint32 metric DistanceType } // NewMultiGpuIndex creates a new MultiGpuIndex instance. -func NewMultiGpuIndex[T VectorType](indices []GpuIndex[T], bruteForce *GpuBruteForce[T], dimension uint32, metric DistanceType) *MultiGpuIndex[T] { +func NewMultiGpuIndex[T VectorType](indices []GpuIndex[T], bruteForce *GpuBruteForce[T, T], dimension uint32, metric DistanceType) *MultiGpuIndex[T] { return &MultiGpuIndex[T]{ indices: indices, bruteForce: bruteForce, @@ -48,13 +62,6 @@ func (mi *MultiGpuIndex[T]) Search(queries []T, numQueries uint64, dimension uin }, nil, nil, nil) } -// SearchFloat32 performs a K-Nearest Neighbor search with float32 queries across all internal indices asynchronously. -func (mi *MultiGpuIndex[T]) SearchFloat32(queries []float32, numQueries uint64, dimension uint32, limit uint32) ([]int64, []float32, error) { - return multiGpuSearch(mi.indices, mi.bruteForce, mi.dimension, nil, queries, numQueries, dimension, limit, nil, func(idx GpuIndex[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.SearchFloat32Async(q, nQ, d, l) - }, nil, nil) -} - // Destroy destroys all internal indices. func (mi *MultiGpuIndex[T]) Destroy() error { var firstErr error @@ -73,15 +80,15 @@ func (mi *MultiGpuIndex[T]) Destroy() error { // --- MultiGpuIvfFlat --- -type MultiGpuIvfFlat[T VectorType] struct { - indices []*GpuIvfFlat[T] - bruteForce *GpuBruteForce[T] +type MultiGpuIvfFlat[B VectorType, Q VectorType] struct { + indices []*GpuIvfFlat[B, Q] + bruteForce BruteForceOverflow[B] dimension uint32 metric DistanceType } -func NewMultiGpuIvfFlat[T VectorType](indices []*GpuIvfFlat[T], bruteForce *GpuBruteForce[T], dimension uint32, metric DistanceType) *MultiGpuIvfFlat[T] { - return &MultiGpuIvfFlat[T]{indices: indices, bruteForce: bruteForce, dimension: dimension, metric: metric} +func NewMultiGpuIvfFlat[B VectorType, Q VectorType](indices []*GpuIvfFlat[B, Q], bruteForce BruteForceOverflow[B], dimension uint32, metric DistanceType) *MultiGpuIvfFlat[B, Q] { + return &MultiGpuIvfFlat[B, Q]{indices: indices, bruteForce: bruteForce, dimension: dimension, metric: metric} } // All MultiIndex paths funnel through multiGpuSearch — every inner index @@ -90,90 +97,155 @@ func NewMultiGpuIvfFlat[T VectorType](indices []*GpuIvfFlat[T], bruteForce *GpuB // search_wait() (plan: effervescent-hatching-dewdrop.md), there is no // remaining reason to keep the sync fallbacks here; they bypassed dynamic // batching and serialized through main_thread_. -func (mi *MultiGpuIvfFlat[T]) Search(queries []T, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) ([]int64, []float32, error) { - genericIndices := make([]GpuIndex[T], len(mi.indices)) +// +// Storage-typed (Q) query path. When an overflow brute force is loaded it is +// base-typed (GpuBruteForce[B]), so it needs a []B query; we only have one when +// B==Q (i.e. F32/F16 storage, where storage type == base type). For the +// quantized combos (B=float/half, Q=int8/uint8) the []Q->[]B assertion fails, +// qB stays nil, and multiGpuSearchBQ's guard returns a "B/Q dispatch mismatch" +// error rather than searching — a storage-typed (already-quantized) query +// cannot be reconstructed into the base-typed query the overflow requires. +// Production code reaches the overflow via the base-typed query path +// (SearchQuantize), which is unaffected; callers needing the overflow with a +// quantized index should use SearchQuantize, not this typed entry point. +func (mi *MultiGpuIvfFlat[B, Q]) Search(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) ([]int64, []float32, error) { + genericIndices := make([]GpuIndex[Q], len(mi.indices)) for i, idx := range mi.indices { genericIndices[i] = idx } - return multiGpuSearch(genericIndices, mi.bruteForce, mi.dimension, queries, nil, numQueries, dimension, limit, func(idx GpuIndex[T], q []T, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.(*GpuIvfFlat[T]).SearchAsyncWithParams(q, nQ, d, l, sp) - }, nil, nil, nil) + // Reinterpret the native Q query as []B for the base-typed overflow. Only + // succeeds when B==Q; nil otherwise (see the method doc above). + var qB []B + if mi.bruteForce != nil { + qB, _ = any(queries).([]B) + } + return multiGpuSearchBQ(genericIndices, mi.bruteForce, mi.dimension, queries, nil, qB, nil, numQueries, dimension, limit, + func(idx GpuIndex[Q], q []Q, nQ uint64, d uint32, l uint32) (uint64, error) { + return idx.(*GpuIvfFlat[B, Q]).SearchAsyncWithParams(q, nQ, d, l, sp) + }, nil, nil, nil) } -func (mi *MultiGpuIvfFlat[T]) SearchFloat32(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) ([]int64, []float32, error) { - genericIndices := make([]GpuIndex[T], len(mi.indices)) +// SearchQuantize — see MultiGpuIvfPq.SearchQuantize. +func (mi *MultiGpuIvfFlat[B, Q]) SearchQuantize(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) ([]int64, []float32, error) { + genericIndices := make([]GpuIndex[Q], len(mi.indices)) for i, idx := range mi.indices { genericIndices[i] = idx } - return multiGpuSearch(genericIndices, mi.bruteForce, mi.dimension, nil, queries, numQueries, dimension, limit, nil, func(idx GpuIndex[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.(*GpuIvfFlat[T]).SearchFloat32AsyncWithParams(q, nQ, d, l, sp) - }, nil, nil) + var qOv []B + if mi.bruteForce != nil { + qOv = queries + } + return multiGpuSearchBQ(genericIndices, mi.bruteForce, mi.dimension, nil, nil, qOv, nil, numQueries, dimension, limit, + nil, nil, func(bf BruteForceOverflow[B], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return bf.SearchQuantizeAsync(q, nQ, d, l) + }, nil, idxBaseQuery[Q, B]{queries: queries, fn: func(idx GpuIndex[Q], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return idx.(*GpuIvfFlat[B, Q]).SearchQuantizeAsyncWithParams(q, nQ, d, l, sp) + }}) } // --- MultiGpuIvfPq --- -type MultiGpuIvfPq[T VectorType] struct { - indices []*GpuIvfPq[T] - bruteForce *GpuBruteForce[T] +// MultiGpuIvfPq carries two element types: storage Q (the main cuVS ivf_pq +// indices) and base B (the CDC/overflow brute force). B==Q for a direct index; +// for a quantized index (e.g. vecf16 base -> int8 storage) B is the base type +// (Float16/float32) so the overflow brute force is cuVS-supported and lossless. +type MultiGpuIvfPq[B VectorType, Q VectorType] struct { + indices []*GpuIvfPq[B, Q] + bruteForce BruteForceOverflow[B] dimension uint32 metric DistanceType } -func NewMultiGpuIvfPq[T VectorType](indices []*GpuIvfPq[T], bruteForce *GpuBruteForce[T], dimension uint32, metric DistanceType) *MultiGpuIvfPq[T] { - return &MultiGpuIvfPq[T]{indices: indices, bruteForce: bruteForce, dimension: dimension, metric: metric} +func NewMultiGpuIvfPq[B VectorType, Q VectorType](indices []*GpuIvfPq[B, Q], bruteForce BruteForceOverflow[B], dimension uint32, metric DistanceType) *MultiGpuIvfPq[B, Q] { + return &MultiGpuIvfPq[B, Q]{indices: indices, bruteForce: bruteForce, dimension: dimension, metric: metric} } -func (mi *MultiGpuIvfPq[T]) Search(queries []T, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) ([]int64, []float32, error) { - genericIndices := make([]GpuIndex[T], len(mi.indices)) +func (mi *MultiGpuIvfPq[B, Q]) Search(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) ([]int64, []float32, error) { + genericIndices := make([]GpuIndex[Q], len(mi.indices)) for i, idx := range mi.indices { genericIndices[i] = idx } - return multiGpuSearch(genericIndices, mi.bruteForce, mi.dimension, queries, nil, numQueries, dimension, limit, func(idx GpuIndex[T], q []T, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.(*GpuIvfPq[T]).SearchAsyncWithParams(q, nQ, d, l, sp) - }, nil, nil, nil) + // Native Q query — the direct (B==Q) path; the overflow takes the same query + // reinterpreted as []B (B==Q here). + var qB []B + if mi.bruteForce != nil { + qB, _ = any(queries).([]B) + } + return multiGpuSearchBQ(genericIndices, mi.bruteForce, mi.dimension, queries, nil, qB, nil, numQueries, dimension, limit, + func(idx GpuIndex[Q], q []Q, nQ uint64, d uint32, l uint32) (uint64, error) { + return idx.(*GpuIvfPq[B, Q]).SearchAsyncWithParams(q, nQ, d, l, sp) + }, nil, nil, nil) } -func (mi *MultiGpuIvfPq[T]) SearchFloat32(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) ([]int64, []float32, error) { - genericIndices := make([]GpuIndex[T], len(mi.indices)) +// SearchQuantize searches with a base-typed (B) query: each main index converts +// B -> its storage type Q on device (B==Q copy for a direct index, learned/cast +// quantizer for narrower Q), and the base-typed overflow brute force takes the +// same B query. Unifies the former SearchFloat32 (B=float32) and SearchQuantizeHalf +// (B=half) paths; the non-filter twin of SearchQuantizeWithFilter. Works +// overflow-only (no main index, small data). +func (mi *MultiGpuIvfPq[B, Q]) SearchQuantize(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) ([]int64, []float32, error) { + genericIndices := make([]GpuIndex[Q], len(mi.indices)) for i, idx := range mi.indices { genericIndices[i] = idx } - return multiGpuSearch(genericIndices, mi.bruteForce, mi.dimension, nil, queries, numQueries, dimension, limit, nil, func(idx GpuIndex[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.(*GpuIvfPq[T]).SearchFloat32AsyncWithParams(q, nQ, d, l, sp) - }, nil, nil) + var qOv []B + if mi.bruteForce != nil { + qOv = queries + } + return multiGpuSearchBQ(genericIndices, mi.bruteForce, mi.dimension, nil, nil, qOv, nil, numQueries, dimension, limit, + nil, nil, func(bf BruteForceOverflow[B], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return bf.SearchQuantizeAsync(q, nQ, d, l) + }, nil, idxBaseQuery[Q, B]{queries: queries, fn: func(idx GpuIndex[Q], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return idx.(*GpuIvfPq[B, Q]).SearchQuantizeAsyncWithParams(q, nQ, d, l, sp) + }}) } // --- MultiGpuCagra --- -type MultiGpuCagra[T VectorType] struct { - indices []*GpuCagra[T] - bruteForce *GpuBruteForce[T] +// MultiGpuCagra carries base type B (overflow) and storage type Q (cagra +// indices) — see MultiGpuIvfPq. +type MultiGpuCagra[B VectorType, Q VectorType] struct { + indices []*GpuCagra[B, Q] + bruteForce BruteForceOverflow[B] dimension uint32 metric DistanceType } -func NewMultiGpuCagra[T VectorType](indices []*GpuCagra[T], bruteForce *GpuBruteForce[T], dimension uint32, metric DistanceType) *MultiGpuCagra[T] { - return &MultiGpuCagra[T]{indices: indices, bruteForce: bruteForce, dimension: dimension, metric: metric} +func NewMultiGpuCagra[B VectorType, Q VectorType](indices []*GpuCagra[B, Q], bruteForce BruteForceOverflow[B], dimension uint32, metric DistanceType) *MultiGpuCagra[B, Q] { + return &MultiGpuCagra[B, Q]{indices: indices, bruteForce: bruteForce, dimension: dimension, metric: metric} } -func (mi *MultiGpuCagra[T]) Search(queries []T, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) ([]int64, []float32, error) { - genericIndices := make([]GpuIndex[T], len(mi.indices)) +func (mi *MultiGpuCagra[B, Q]) Search(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) ([]int64, []float32, error) { + genericIndices := make([]GpuIndex[Q], len(mi.indices)) for i, idx := range mi.indices { genericIndices[i] = idx } - return multiGpuSearch(genericIndices, mi.bruteForce, mi.dimension, queries, nil, numQueries, dimension, limit, func(idx GpuIndex[T], q []T, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.(*GpuCagra[T]).SearchAsyncWithParams(q, nQ, d, l, sp) - }, nil, nil, nil) + var qB []B + if mi.bruteForce != nil { + qB, _ = any(queries).([]B) + } + return multiGpuSearchBQ(genericIndices, mi.bruteForce, mi.dimension, queries, nil, qB, nil, numQueries, dimension, limit, + func(idx GpuIndex[Q], q []Q, nQ uint64, d uint32, l uint32) (uint64, error) { + return idx.(*GpuCagra[B, Q]).SearchAsyncWithParams(q, nQ, d, l, sp) + }, nil, nil, nil) } -func (mi *MultiGpuCagra[T]) SearchFloat32(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) ([]int64, []float32, error) { - genericIndices := make([]GpuIndex[T], len(mi.indices)) +// SearchQuantize — see MultiGpuIvfPq.SearchQuantize. +func (mi *MultiGpuCagra[B, Q]) SearchQuantize(queries []B, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) ([]int64, []float32, error) { + genericIndices := make([]GpuIndex[Q], len(mi.indices)) for i, idx := range mi.indices { genericIndices[i] = idx } - return multiGpuSearch(genericIndices, mi.bruteForce, mi.dimension, nil, queries, numQueries, dimension, limit, nil, func(idx GpuIndex[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.(*GpuCagra[T]).SearchFloat32AsyncWithParams(q, nQ, d, l, sp) - }, nil, nil) + var qOv []B + if mi.bruteForce != nil { + qOv = queries + } + return multiGpuSearchBQ(genericIndices, mi.bruteForce, mi.dimension, nil, nil, qOv, nil, numQueries, dimension, limit, + nil, nil, func(bf BruteForceOverflow[B], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return bf.SearchQuantizeAsync(q, nQ, d, l) + }, nil, idxBaseQuery[Q, B]{queries: queries, fn: func(idx GpuIndex[Q], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return idx.(*GpuCagra[B, Q]).SearchQuantizeAsyncWithParams(q, nQ, d, l, sp) + }}) } // --- Helper search function --- @@ -186,7 +258,7 @@ func (mi *MultiGpuCagra[T]) SearchFloat32(queries []float32, numQueries uint64, // uses SearchFloatWithFilterAsync. func multiGpuSearch[T VectorType]( indices []GpuIndex[T], - bruteForce *GpuBruteForce[T], + bruteForce *GpuBruteForce[T, T], miDimension uint32, queries []T, queriesF32 []float32, @@ -195,8 +267,8 @@ func multiGpuSearch[T VectorType]( limit uint32, searchFn func(GpuIndex[T], []T, uint64, uint32, uint32) (uint64, error), searchF32Fn func(GpuIndex[T], []float32, uint64, uint32, uint32) (uint64, error), - bfSearchFn func(*GpuBruteForce[T], []T, uint64, uint32, uint32) (uint64, error), - bfSearchF32Fn func(*GpuBruteForce[T], []float32, uint64, uint32, uint32) (uint64, error), + bfSearchFn func(*GpuBruteForce[T, T], []T, uint64, uint32, uint32) (uint64, error), + bfSearchF32Fn func(*GpuBruteForce[T, T], []float32, uint64, uint32, uint32) (uint64, error), ) ([]int64, []float32, error) { if queryDimension != miDimension { return nil, nil, moerr.NewInternalErrorNoCtx("query dimension mismatch") @@ -212,7 +284,7 @@ func multiGpuSearch[T VectorType]( } type jobInfo struct { - index GpuIndex[T] + w searchWaiter jobID uint64 } jobs := make([]jobInfo, 0, numIndices) @@ -228,7 +300,7 @@ func multiGpuSearch[T VectorType]( if err != nil { return nil, nil, err } - jobs = append(jobs, jobInfo{index: idx, jobID: jobID}) + jobs = append(jobs, jobInfo{w: idx, jobID: jobID}) } if bruteForce != nil { @@ -238,26 +310,31 @@ func multiGpuSearch[T VectorType]( if bfSearchFn != nil { jobID, err = bfSearchFn(bruteForce, queries, numQueries, queryDimension, limit) } else { - jobID, err = bruteForce.SearchAsync(queries, numQueries, queryDimension, limit) + // Native query reinterpreted as the base type B (== T here) for the + // quantize entry; SearchQuantizeAsync quantizes B -> storage T. + qB, _ := any(queries).([]T) + jobID, err = bruteForce.SearchQuantizeAsync(qB, numQueries, queryDimension, limit) } } else { if bfSearchF32Fn != nil { jobID, err = bfSearchF32Fn(bruteForce, queriesF32, numQueries, queryDimension, limit) } else { - jobID, err = bruteForce.SearchFloat32Async(queriesF32, numQueries, queryDimension, limit) + // f32 query reinterpreted as the base type B (== T == float32 here). + qB, _ := any(queriesF32).([]T) + jobID, err = bruteForce.SearchQuantizeAsync(qB, numQueries, queryDimension, limit) } } if err != nil { return nil, nil, err } - jobs = append(jobs, jobInfo{index: bruteForce, jobID: jobID}) + jobs = append(jobs, jobInfo{w: bruteForce, jobID: jobID}) } allNeighbors := make([][]int64, len(jobs)) allDistances := make([][]float32, len(jobs)) for i, job := range jobs { - neighbors, distances, err := job.index.SearchWait(job.jobID, numQueries, limit) + neighbors, distances, err := job.w.SearchWait(job.jobID, numQueries, limit) if err != nil { return nil, nil, err } @@ -269,6 +346,137 @@ func multiGpuSearch[T VectorType]( return n, d, nil } +// searchWaiter is the post-submission contract shared by GpuIndex[Q] and +// *GpuBruteForce[B]: once a search job is submitted, collecting its result is +// type-agnostic (jobID -> []int64 neighbors, []float32 distances). This lets +// multiGpuSearchBQ merge index (storage type Q) and overflow (base type B) +// results without the two types leaking into the wait/merge. +type searchWaiter interface { + SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) +} + +// multiGpuSearchBQ is multiGpuSearch with the brute-force overflow typed by the +// BASE type B (f16/f32) independently of the index storage type Q — the [B,Q] +// design. Indices are searched with the []Q (e.g. quantized) query, the +// base-typed overflow with the []B (or f32) query; both are submitted async to +// the worker pool and the post-submission collect/merge is type-agnostic +// (searchWaiter). No extra goroutine. When B==Q this is equivalent to +// multiGpuSearch with the overflow carrying the base type. +// idxBaseQuery carries a base-typed (B) index query + its dispatch function for +// the quantize-with-filter path: the index is searched with the native base +// query (f32 or half) and converts it to storage T inside cuVS (the const-B* +// search_quantize entry). Passed as an optional variadic to multiGpuSearchBQ so +// the many unfiltered/storage-typed callers stay untouched; when present it +// takes precedence over queriesQ/queriesQF32 for the index loop. The overflow +// still uses the queriesB/queriesBF32 channels independently. +type idxBaseQuery[Q VectorType, B VectorType] struct { + queries []B + fn func(GpuIndex[Q], []B, uint64, uint32, uint32) (uint64, error) +} + +func multiGpuSearchBQ[Q VectorType, B VectorType]( + indices []GpuIndex[Q], + bruteForce BruteForceOverflow[B], + miDimension uint32, + queriesQ []Q, + queriesQF32 []float32, + queriesB []B, + queriesBF32 []float32, + numQueries uint64, + queryDimension uint32, + limit uint32, + idxFn func(GpuIndex[Q], []Q, uint64, uint32, uint32) (uint64, error), + idxF32Fn func(GpuIndex[Q], []float32, uint64, uint32, uint32) (uint64, error), + bfFn func(BruteForceOverflow[B], []B, uint64, uint32, uint32) (uint64, error), + bfF32Fn func(BruteForceOverflow[B], []float32, uint64, uint32, uint32) (uint64, error), + idxBase ...idxBaseQuery[Q, B], +) ([]int64, []float32, error) { + if queryDimension != miDimension { + return nil, nil, moerr.NewInternalErrorNoCtx("query dimension mismatch") + } + + n := len(indices) + if bruteForce != nil { + n++ + } + if n == 0 { + return nil, nil, moerr.NewInternalErrorNoCtx("no indices in MultiIndex") + } + + var ib *idxBaseQuery[Q, B] + if len(idxBase) > 0 { + ib = &idxBase[0] + } + + type jobInfo struct { + w searchWaiter + jobID uint64 + } + jobs := make([]jobInfo, 0, n) + + for _, idx := range indices { + var jobID uint64 + var err error + if ib != nil { + // Base-typed quantize query: index converts B -> storage T in cuVS. + jobID, err = ib.fn(idx, ib.queries, numQueries, queryDimension, limit) + } else if queriesQ != nil { + jobID, err = idxFn(idx, queriesQ, numQueries, queryDimension, limit) + } else { + jobID, err = idxF32Fn(idx, queriesQF32, numQueries, queryDimension, limit) + } + if err != nil { + return nil, nil, err + } + jobs = append(jobs, jobInfo{w: idx, jobID: jobID}) + } + + if bruteForce != nil { + // Guard against a dispatch mismatch: if the overflow brute force is + // live but neither a base-typed (B) nor an f32 query was supplied, the + // async search would submit an empty job (job id 0) and SearchWait(0) + // would block forever. Fail loudly instead — this means the [B,Q] + // instantiation disagrees with the decoded query type. + if len(queriesB) == 0 && len(queriesBF32) == 0 { + return nil, nil, moerr.NewInternalErrorNoCtx("multiGpuSearchBQ: brute force is loaded but no base/f32 query was provided (B/Q dispatch mismatch)") + } + // The overflow always takes the base-typed (B) query and quantizes B->OB + // inside cuVS. When only an f32 channel was supplied (the SearchFloat32 + // paths, where B==float32), reinterpret it as []B. + qB := queriesB + if qB == nil { + qB, _ = any(queriesBF32).([]B) + } + var jobID uint64 + var err error + if bfFn != nil { + jobID, err = bfFn(bruteForce, qB, numQueries, queryDimension, limit) + } else if bfF32Fn != nil { + jobID, err = bfF32Fn(bruteForce, queriesBF32, numQueries, queryDimension, limit) + } else { + jobID, err = bruteForce.SearchQuantizeAsync(qB, numQueries, queryDimension, limit) + } + if err != nil { + return nil, nil, err + } + jobs = append(jobs, jobInfo{w: bruteForce, jobID: jobID}) + } + + allNeighbors := make([][]int64, len(jobs)) + allDistances := make([][]float32, len(jobs)) + for i, job := range jobs { + neighbors, distances, err := job.w.SearchWait(job.jobID, numQueries, limit) + if err != nil { + return nil, nil, err + } + allNeighbors[i] = neighbors + allDistances[i] = distances + } + + n2, d := mergeMultiResults(allNeighbors, allDistances, numQueries, limit) + return n2, d, nil +} + // mergeMultiResults does a k-way merge of per-index top-k results into a single // top-k per query using a max-heap. Empty slots (neighbor == -1) are skipped. // Shared by both the async-dispatched multiGpuSearch and the synchronous @@ -310,48 +518,73 @@ func mergeMultiResults(allNeighbors [][]int64, allDistances [][]float32, numQuer // --- Filtered async search variants --- // -// Every per-index filtered search is dispatched via SearchFloatWithFilterAsync -// (which returns a job_id) and collected with SearchWait, matching the -// unfiltered SearchFloat32 path. Predicate evaluation, H2D, and GPU work for -// sibling indices overlap on their own worker threads, including the -// brute-force fallback when mi.bruteForce is non-nil. +// Each per-index filtered search is dispatched async (returns a job_id) and +// collected with SearchWait. cagra/ivf_pq/ivf_flat all use the base-typed +// SearchQuantizeWithFilterAsync (the const-B* quantize path); the brute-force +// overflow uses the base-typed SearchQuantizeWithFilterAsync. Predicate evaluation, H2D, +// and GPU work for sibling indices overlap on their own worker threads, +// including the brute-force fallback when mi.bruteForce is non-nil. // // SHARDED inner indices no longer get routed through main_thread_ — see the // C++ search_*_with_filter_async branches and plan // .claude/plans/effervescent-hatching-dewdrop.md. -func (mi *MultiGpuCagra[T]) SearchFloat32WithFilter(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams, predsJSON string) ([]int64, []float32, error) { - genericIndices := make([]GpuIndex[T], len(mi.indices)) +// SearchQuantizeWithFilter runs a filtered K-NN search with base-typed (B) +// queries: each index quantizes B -> storage Q inside cuVS (the const-B* +// search_quantize_with_filter entry) and the base-typed overflow takes the same +// native B query. Covers both f32 base (B==float, query was []float32) and vecf16 +// base (B==half, query was []Float16) — they differ only in the concrete query +// slice the caller asserts to []B. Both async via the worker pool; works +// overflow-only (no main index, small data). +func (mi *MultiGpuCagra[B, Q]) SearchQuantizeWithFilter(queries []B, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams, predsJSON string) ([]int64, []float32, error) { + genericIndices := make([]GpuIndex[Q], len(mi.indices)) for i, idx := range mi.indices { genericIndices[i] = idx } - return multiGpuSearch(genericIndices, mi.bruteForce, mi.dimension, nil, queries, numQueries, dimension, limit, nil, func(idx GpuIndex[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.(*GpuCagra[T]).SearchFloatWithFilterAsync(q, nQ, d, l, sp, predsJSON) - }, nil, func(bf *GpuBruteForce[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return bf.SearchFloatWithFilterAsync(q, nQ, d, l, predsJSON) - }) + var qOv []B + if mi.bruteForce != nil { + qOv = queries + } + return multiGpuSearchBQ(genericIndices, mi.bruteForce, mi.dimension, nil, nil, qOv, nil, numQueries, dimension, limit, + nil, nil, func(bf BruteForceOverflow[B], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return bf.SearchQuantizeWithFilterAsync(q, nQ, d, l, predsJSON) + }, nil, idxBaseQuery[Q, B]{queries: queries, fn: func(idx GpuIndex[Q], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return idx.(*GpuCagra[B, Q]).SearchQuantizeWithFilterAsync(q, nQ, d, l, sp, predsJSON) + }}) } -func (mi *MultiGpuIvfFlat[T]) SearchFloat32WithFilter(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams, predsJSON string) ([]int64, []float32, error) { - genericIndices := make([]GpuIndex[T], len(mi.indices)) +// SearchQuantizeWithFilter — see MultiGpuCagra.SearchQuantizeWithFilter. +func (mi *MultiGpuIvfFlat[B, Q]) SearchQuantizeWithFilter(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams, predsJSON string) ([]int64, []float32, error) { + genericIndices := make([]GpuIndex[Q], len(mi.indices)) for i, idx := range mi.indices { genericIndices[i] = idx } - return multiGpuSearch(genericIndices, mi.bruteForce, mi.dimension, nil, queries, numQueries, dimension, limit, nil, func(idx GpuIndex[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.(*GpuIvfFlat[T]).SearchFloatWithFilterAsync(q, nQ, d, l, sp, predsJSON) - }, nil, func(bf *GpuBruteForce[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return bf.SearchFloatWithFilterAsync(q, nQ, d, l, predsJSON) - }) + var qOv []B + if mi.bruteForce != nil { + qOv = queries + } + return multiGpuSearchBQ(genericIndices, mi.bruteForce, mi.dimension, nil, nil, qOv, nil, numQueries, dimension, limit, + nil, nil, func(bf BruteForceOverflow[B], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return bf.SearchQuantizeWithFilterAsync(q, nQ, d, l, predsJSON) + }, nil, idxBaseQuery[Q, B]{queries: queries, fn: func(idx GpuIndex[Q], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return idx.(*GpuIvfFlat[B, Q]).SearchQuantizeWithFilterAsync(q, nQ, d, l, sp, predsJSON) + }}) } -func (mi *MultiGpuIvfPq[T]) SearchFloat32WithFilter(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams, predsJSON string) ([]int64, []float32, error) { - genericIndices := make([]GpuIndex[T], len(mi.indices)) +// SearchQuantizeWithFilter — see MultiGpuCagra.SearchQuantizeWithFilter. +func (mi *MultiGpuIvfPq[B, Q]) SearchQuantizeWithFilter(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams, predsJSON string) ([]int64, []float32, error) { + genericIndices := make([]GpuIndex[Q], len(mi.indices)) for i, idx := range mi.indices { genericIndices[i] = idx } - return multiGpuSearch(genericIndices, mi.bruteForce, mi.dimension, nil, queries, numQueries, dimension, limit, nil, func(idx GpuIndex[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.(*GpuIvfPq[T]).SearchFloatWithFilterAsync(q, nQ, d, l, sp, predsJSON) - }, nil, func(bf *GpuBruteForce[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return bf.SearchFloatWithFilterAsync(q, nQ, d, l, predsJSON) - }) + var qOv []B + if mi.bruteForce != nil { + qOv = queries + } + return multiGpuSearchBQ(genericIndices, mi.bruteForce, mi.dimension, nil, nil, qOv, nil, numQueries, dimension, limit, + nil, nil, func(bf BruteForceOverflow[B], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return bf.SearchQuantizeWithFilterAsync(q, nQ, d, l, predsJSON) + }, nil, idxBaseQuery[Q, B]{queries: queries, fn: func(idx GpuIndex[Q], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return idx.(*GpuIvfPq[B, Q]).SearchQuantizeWithFilterAsync(q, nQ, d, l, sp, predsJSON) + }}) } diff --git a/pkg/cuvs/multi_index_test.go b/pkg/cuvs/multi_index_test.go index bd73a53f8c854..61d349589f1b3 100644 --- a/pkg/cuvs/multi_index_test.go +++ b/pkg/cuvs/multi_index_test.go @@ -53,7 +53,7 @@ func TestMultiGpuIndex(t *testing.T) { // IVF-Flat bpIvf := DefaultIvfFlatBuildParams() - idx2, err := NewGpuIvfFlat[float32](dataset2, count2, dimension, metric, bpIvf, devices, nthread, SingleGpu, nil) + idx2, err := NewGpuIvfFlat[float32, float32](dataset2, count2, dimension, metric, bpIvf, devices, nthread, SingleGpu, nil) assert.NoError(t, err) err = idx2.Start() assert.NoError(t, err) @@ -61,7 +61,7 @@ func TestMultiGpuIndex(t *testing.T) { assert.NoError(t, err) // Brute Force - bf, err := NewGpuBruteForce[float32](dataset1, count1, dimension, metric, nthread, 0) + bf, err := NewGpuBruteForce[float32, float32](dataset1, count1, dimension, metric, nthread, 0) assert.NoError(t, err) err = bf.Start() assert.NoError(t, err) @@ -93,7 +93,7 @@ func TestMultiGpuIndex(t *testing.T) { // --- Test Specialized MultiGpuIvfFlat --- t.Run("SpecializedIvfFlat", func(t *testing.T) { - mivf := NewMultiGpuIvfFlat[float32]([]*GpuIvfFlat[float32]{idx2}, bf, dimension, metric) + mivf := NewMultiGpuIvfFlat[float32, float32]([]*GpuIvfFlat[float32, float32]{idx2}, bf, dimension, metric) numQueries := uint64(5) limit := uint32(10) diff --git a/pkg/cuvs/search_async_batch_test.go b/pkg/cuvs/search_async_batch_test.go index b713d70c0c18c..198e7facbb694 100644 --- a/pkg/cuvs/search_async_batch_test.go +++ b/pkg/cuvs/search_async_batch_test.go @@ -128,7 +128,7 @@ func TestGpuCagraSearchFloat32AsyncBatched(t *testing.T) { // result demuxing through submit_batched_async's per-request setter. runConcurrentAsync(t, 16 /*nGoroutines*/, 8 /*nPerGoroutine*/, func(qid int) (int64, error) { q := []float32{float32(qid), float32(qid)} - jobID, err := index.SearchFloat32AsyncWithParams(q, 1, dimension, 1, sp) + jobID, err := index.SearchQuantizeAsyncWithParams(q, 1, dimension, 1, sp) if err != nil { return -1, err } @@ -151,7 +151,7 @@ func TestGpuIvfFlatSearchFloat32AsyncBatched(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 16 - index, err := NewGpuIvfFlat[float32](dataset, nVectors, dimension, L2Expanded, bp, []int{0}, 4, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, nVectors, dimension, L2Expanded, bp, []int{0}, 4, SingleGpu, nil) if err != nil { t.Fatalf("NewGpuIvfFlat: %v", err) } @@ -174,7 +174,7 @@ func TestGpuIvfFlatSearchFloat32AsyncBatched(t *testing.T) { runConcurrentAsync(t, 16 /*nGoroutines*/, 8 /*nPerGoroutine*/, func(qid int) (int64, error) { q := []float32{float32(qid), float32(qid)} - jobID, err := index.SearchFloat32AsyncWithParams(q, 1, dimension, 1, sp) + jobID, err := index.SearchQuantizeAsyncWithParams(q, 1, dimension, 1, sp) if err != nil { return -1, err } @@ -251,7 +251,7 @@ func ivfPqAsyncBatchedMatchesSync(t *testing.T, conservativeDispatch bool) { } want := make([][]int64, nQueries) for qid := 0; qid < nQueries; qid++ { - res, err := index.SearchFloat(queryOf(qid), 1, dimension, limit, sp) + res, err := index.SearchQuantize(queryOf(qid), 1, dimension, limit, sp) if err != nil { t.Fatalf("SearchFloat reference qid=%d: %v", qid, err) } @@ -269,7 +269,7 @@ func ivfPqAsyncBatchedMatchesSync(t *testing.T, conservativeDispatch bool) { wg.Add(1) go func(qid int) { defer wg.Done() - jobID, err := index.SearchFloat32AsyncWithParams(queryOf(qid), 1, dimension, limit, sp) + jobID, err := index.SearchQuantizeAsyncWithParams(queryOf(qid), 1, dimension, limit, sp) if err != nil { errCh <- err return @@ -353,7 +353,7 @@ func TestGpuCagraAsyncBatchedMatchesSync(t *testing.T) { want := make([]int64, nQueries) for qid := 0; qid < nQueries; qid++ { q := []float32{float32(qid * 10), float32(qid * 10)} - res, err := index.SearchFloat(q, 1, dimension, 1, sp) + res, err := index.SearchQuantize(q, 1, dimension, 1, sp) if err != nil { t.Fatalf("SearchFloat reference: %v", err) } @@ -375,7 +375,7 @@ func TestGpuCagraAsyncBatchedMatchesSync(t *testing.T) { go func(qid int) { defer wg.Done() q := []float32{float32(qid * 10), float32(qid * 10)} - jobID, err := index.SearchFloat32AsyncWithParams(q, 1, dimension, 1, sp) + jobID, err := index.SearchQuantizeAsyncWithParams(q, 1, dimension, 1, sp) if err != nil { errCh <- err return diff --git a/pkg/cuvs/search_f16quant_test.go b/pkg/cuvs/search_f16quant_test.go new file mode 100644 index 0000000000000..f567adac056e0 --- /dev/null +++ b/pkg/cuvs/search_f16quant_test.go @@ -0,0 +1,170 @@ +//go:build gpu + +// Copyright 2021 - 2022 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cuvs + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "golang.org/x/exp/rand" +) + +// f16 converts a float32 to the cuVS half (Float16) bit pattern. types.Float16 +// and cuvs.Float16 are both uint16 IEEE-754 halfs, so the bits transfer directly. +func f16(f float32) Float16 { return Float16(types.Float16FromFloat32(f)) } + +// makeF16Dataset builds a deterministic random half dataset in [0,1). +func makeF16Dataset(n uint64, dim uint32) []Float16 { + r := rand.New(rand.NewSource(1)) + ds := make([]Float16, n*uint64(dim)) + for i := range ds { + ds[i] = f16(r.Float32()) + } + return ds +} + +// TestGpuF16QuantizeAll covers the vecf16-base -> int8/uint8 quantization path +// (the native half-source quantizer) for IVF-PQ and CAGRA: build from native +// Float16 input via AddChunkQuantize, then SearchQuantize with a Float16 query. +// This is the f16->int8 / f16->uint8 combination the float32-base info_test and +// search_float_test do not exercise. +// +// Correctness is graded as self-match RECALL: each probe query is an exact copy +// of a stored row, so a working quantized search must return that row's id in the +// top-k for the large majority of probes. (Exact top-1 is not asserted — int8/ +// uint8 + product quantization are lossy by design; recall is the right metric, +// matching the C++ Int8VsUint8SignedDataHalf test.) A broken search would score +// ~0, so the 0.8 floor is a real correctness check, not a shape check. +func TestGpuF16QuantizeAll(t *testing.T) { + const ( + dimension = uint32(16) + nVectors = uint64(2000) + k = uint32(10) + minRecall = 0.8 + ) + deviceID := 0 + ds := makeF16Dataset(nVectors, dimension) + probes := []int64{0, 1, 7, 100, 500, 999, 1500, 1999} // rows to self-query + + t.Run("IVF-PQ/f16-int8", func(t *testing.T) { + runIvfPqF16Quant[int8](t, ds, probes, nVectors, dimension, k, deviceID, minRecall) + }) + t.Run("IVF-PQ/f16-uint8", func(t *testing.T) { + runIvfPqF16Quant[uint8](t, ds, probes, nVectors, dimension, k, deviceID, minRecall) + }) + t.Run("CAGRA/f16-int8", func(t *testing.T) { + runCagraF16Quant[int8](t, ds, probes, nVectors, dimension, k, deviceID, minRecall) + }) + t.Run("CAGRA/f16-uint8", func(t *testing.T) { + runCagraF16Quant[uint8](t, ds, probes, nVectors, dimension, k, deviceID, minRecall) + }) +} + +// rowVec returns a copy of row id from ds (an exact self-query). +func rowVec(ds []Float16, id int64, dim uint32) []Float16 { + off := uint64(id) * uint64(dim) + return append([]Float16(nil), ds[off:off+uint64(dim)]...) +} + +func contains(xs []int64, want int64) bool { + for _, x := range xs { + if x == want { + return true + } + } + return false +} + +// gradeSelfMatch runs each probe as a self-query and returns the fraction whose +// own id appears in the top-k. +func gradeSelfMatch(t *testing.T, probes []int64, k uint32, search func(int64) ([]int64, error)) float64 { + t.Helper() + hits := 0 + for _, id := range probes { + neighbors, err := search(id) + if err != nil { + t.Fatalf("SearchQuantize(row %d): %v", id, err) + } + if uint32(len(neighbors)) != k { + t.Fatalf("row %d: expected %d neighbors, got %d", id, k, len(neighbors)) + } + if contains(neighbors, id) { + hits++ + } + } + return float64(hits) / float64(len(probes)) +} + +func runIvfPqF16Quant[Q VectorType](t *testing.T, ds []Float16, probes []int64, n uint64, dim, k uint32, dev int, minRecall float64) { + // Start from the defaults (which set KmeansTrainsetFraction etc.) and override + // only NLists — the default 1024 lists would be near-empty for 2000 vectors. + // A struct literal would zero-default the omitted fields, e.g. + // KmeansTrainsetFraction=0 => no kmeans training => near-zero recall. + bp := DefaultIvfPqBuildParams() + bp.NLists = 50 + index, err := NewGpuIvfPqEmpty[Float16, Q](n, dim, L2Expanded, bp, []int{dev}, 1, SingleGpu) + if err != nil { + t.Fatalf("NewGpuIvfPqEmpty[Float16,Q]: %v", err) + } + defer index.Destroy() + index.Start() + if err = index.TrainQuantizer(ds, n); err != nil { + t.Fatalf("TrainQuantizer: %v", err) + } + if err = index.AddChunkQuantize(ds, n, nil); err != nil { + t.Fatalf("AddChunkQuantize: %v", err) + } + if err = index.Build(); err != nil { + t.Fatalf("Build: %v", err) + } + sp := DefaultIvfPqSearchParams() + sp.NProbes = bp.NLists // probe every list for a deterministic exhaustive search + recall := gradeSelfMatch(t, probes, k, func(id int64) ([]int64, error) { + res, err := index.SearchQuantize(rowVec(ds, id, dim), 1, dim, k, sp) + return res.Neighbors, err + }) + if recall < minRecall { + t.Errorf("IVF-PQ f16-quant self-match recall %.2f < %.2f", recall, minRecall) + } +} + +func runCagraF16Quant[Q VectorType](t *testing.T, ds []Float16, probes []int64, n uint64, dim, k uint32, dev int, minRecall float64) { + bp := DefaultCagraBuildParams() + index, err := NewGpuCagraEmpty[Float16, Q](n, dim, L2Expanded, bp, []int{dev}, 1, SingleGpu) + if err != nil { + t.Fatalf("NewGpuCagraEmpty[Float16,Q]: %v", err) + } + defer index.Destroy() + index.Start() + if err = index.TrainQuantizer(ds, n); err != nil { + t.Fatalf("TrainQuantizer: %v", err) + } + if err = index.AddChunkQuantize(ds, n, nil); err != nil { + t.Fatalf("AddChunkQuantize: %v", err) + } + if err = index.Build(); err != nil { + t.Fatalf("Build: %v", err) + } + sp := DefaultCagraSearchParams() + recall := gradeSelfMatch(t, probes, k, func(id int64) ([]int64, error) { + res, err := index.SearchQuantize(rowVec(ds, id, dim), 1, dim, k, sp) + return res.Neighbors, err + }) + if recall < minRecall { + t.Errorf("CAGRA f16-quant self-match recall %.2f < %.2f", recall, minRecall) + } +} diff --git a/pkg/cuvs/search_float_test.go b/pkg/cuvs/search_float_test.go index 7ad9106e7a0df..7b288f661b624 100644 --- a/pkg/cuvs/search_float_test.go +++ b/pkg/cuvs/search_float_test.go @@ -33,7 +33,7 @@ func TestGpuSearchFloatAll(t *testing.T) { } bp := IvfPqBuildParams{NLists: 10, M: 4, BitsPerCode: 8, AddDataOnBuild: true} // Create empty index - index, err := NewGpuIvfPqEmpty[int8](n_vectors, dimension, L2Expanded, bp, []int{deviceID}, 1, SingleGpu) + index, err := NewGpuIvfPqEmpty[float32, int8](n_vectors, dimension, L2Expanded, bp, []int{deviceID}, 1, SingleGpu) if err != nil { t.Fatalf("Failed to create IVF-PQ: %v", err) } @@ -46,7 +46,7 @@ func TestGpuSearchFloatAll(t *testing.T) { t.Fatalf("TrainQuantizer failed: %v", err) } - err = index.AddChunkFloat(dataset, n_vectors, nil) + err = index.AddChunkQuantize(dataset, n_vectors, nil) if err != nil { t.Fatalf("AddChunkFloat failed: %v", err) } @@ -56,7 +56,7 @@ func TestGpuSearchFloatAll(t *testing.T) { for i := range queries { queries[i] = float32(i % 10) } - res, err := index.SearchFloat(queries, 2, dimension, 1, IvfPqSearchParams{NProbes: 1}) + res, err := index.SearchQuantize(queries, 2, dimension, 1, IvfPqSearchParams{NProbes: 1}) if err != nil { t.Fatalf("SearchFloat failed: %v", err) } @@ -69,7 +69,7 @@ func TestGpuSearchFloatAll(t *testing.T) { t.Run("IVF-Flat", func(t *testing.T) { dataset := make([]Float16, n_vectors*uint64(dimension)) bp := IvfFlatBuildParams{NLists: 10, AddDataOnBuild: true} - index, err := NewGpuIvfFlat[Float16](dataset, n_vectors, dimension, L2Expanded, bp, []int{deviceID}, 1, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, Float16](dataset, n_vectors, dimension, L2Expanded, bp, []int{deviceID}, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create IVF-Flat: %v", err) } @@ -78,7 +78,7 @@ func TestGpuSearchFloatAll(t *testing.T) { index.Build() queries := make([]float32, uint64(dimension)) - res, err := index.SearchFloat(queries, 1, dimension, 1, IvfFlatSearchParams{NProbes: 1}) + res, err := index.SearchQuantize(queries, 1, dimension, 1, IvfFlatSearchParams{NProbes: 1}) if err != nil { t.Fatalf("SearchFloat failed: %v", err) } @@ -91,7 +91,7 @@ func TestGpuSearchFloatAll(t *testing.T) { t.Run("CAGRA", func(t *testing.T) { dataset := make([]float32, n_vectors*uint64(dimension)) bp := CagraBuildParams{IntermediateGraphDegree: 64, GraphDegree: 32, AttachDatasetOnBuild: true} - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, []int{deviceID}, 1, SingleGpu, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, []int{deviceID}, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create CAGRA: %v", err) } @@ -100,7 +100,7 @@ func TestGpuSearchFloatAll(t *testing.T) { index.Build() queries := make([]float32, uint64(dimension)) - res, err := index.SearchFloat(queries, 1, dimension, 1, CagraSearchParams{ItopkSize: 64, SearchWidth: 1}) + res, err := index.SearchQuantize(queries, 1, dimension, 1, CagraSearchParams{ItopkSize: 64, SearchWidth: 1}) if err != nil { t.Fatalf("SearchFloat failed: %v", err) } @@ -112,7 +112,7 @@ func TestGpuSearchFloatAll(t *testing.T) { // 4. Test Brute-Force SearchFloat (with half) t.Run("Brute-Force", func(t *testing.T) { dataset := make([]Float16, n_vectors*uint64(dimension)) - index, err := NewGpuBruteForce[Float16](dataset, n_vectors, dimension, L2Expanded, 1, deviceID) + index, err := NewGpuBruteForce[float32, Float16](dataset, n_vectors, dimension, L2Expanded, 1, deviceID) if err != nil { t.Fatalf("Failed to create Brute-Force: %v", err) } @@ -121,9 +121,9 @@ func TestGpuSearchFloatAll(t *testing.T) { index.Build() queries := make([]float32, uint64(dimension)) - neighbors, _, err := index.SearchFloat(queries, 1, dimension, 1) + neighbors, _, err := index.SearchQuantize(queries, 1, dimension, 1) if err != nil { - t.Fatalf("SearchFloat failed: %v", err) + t.Fatalf("SearchQuantize failed: %v", err) } if len(neighbors) != 1 { t.Errorf("Expected 1 neighbor, got %d", len(neighbors)) diff --git a/pkg/cuvs/simulation_test.go b/pkg/cuvs/simulation_test.go index 4a4f2ba05e4a7..3015bd696f848 100644 --- a/pkg/cuvs/simulation_test.go +++ b/pkg/cuvs/simulation_test.go @@ -107,7 +107,7 @@ func TestSimulatedReplicatedIvfFlat(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 4 bp.KmeansTrainsetFraction = 1.0 - idx, err := NewGpuIvfFlat[float32](ds, count, dim, L2Expanded, bp, simDevices(), simRanks, Replicated, ids) + idx, err := NewGpuIvfFlat[float32, float32](ds, count, dim, L2Expanded, bp, simDevices(), simRanks, Replicated, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -142,7 +142,7 @@ func TestSimulatedShardedIvfFlat(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 4 bp.KmeansTrainsetFraction = 1.0 - idx, err := NewGpuIvfFlat[float32](ds, count, dim, L2Expanded, bp, simDevices(), simRanks, Sharded, ids) + idx, err := NewGpuIvfFlat[float32, float32](ds, count, dim, L2Expanded, bp, simDevices(), simRanks, Sharded, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -185,7 +185,7 @@ func TestSimulatedShardedDeleteIvfFlat(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 4 bp.KmeansTrainsetFraction = 1.0 - idx, err := NewGpuIvfFlat[float32](ds, count, dim, L2Expanded, bp, simDevices(), simRanks, Sharded, ids) + idx, err := NewGpuIvfFlat[float32, float32](ds, count, dim, L2Expanded, bp, simDevices(), simRanks, Sharded, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -249,7 +249,7 @@ func TestSimulatedReplicatedExtendIvfFlat(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 4 bp.KmeansTrainsetFraction = 1.0 - idx, err := NewGpuIvfFlat[float32](ds, base, dim, L2Expanded, bp, simDevices(), simRanks, Replicated, ids) + idx, err := NewGpuIvfFlat[float32, float32](ds, base, dim, L2Expanded, bp, simDevices(), simRanks, Replicated, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -303,7 +303,7 @@ func TestSimulatedReplicatedIvfPq(t *testing.T) { count := uint64(64) ds, ids := simData(count, dim) - idx, err := NewGpuIvfPq[float32](ds, count, dim, L2Expanded, simIvfPqParams(), simDevices(), simRanks, Replicated, ids) + idx, err := NewGpuIvfPq[float32, float32](ds, count, dim, L2Expanded, simIvfPqParams(), simDevices(), simRanks, Replicated, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -335,7 +335,7 @@ func TestSimulatedShardedIvfPq(t *testing.T) { count := uint64(128) // 4 shards of 32 ds, ids := simData(count, dim) - idx, err := NewGpuIvfPq[float32](ds, count, dim, L2Expanded, simIvfPqParams(), simDevices(), simRanks, Sharded, ids) + idx, err := NewGpuIvfPq[float32, float32](ds, count, dim, L2Expanded, simIvfPqParams(), simDevices(), simRanks, Sharded, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -367,7 +367,7 @@ func TestSimulatedReplicatedExtendIvfPq(t *testing.T) { base := uint64(64) ds, ids := simData(base, dim) - idx, err := NewGpuIvfPq[float32](ds, base, dim, L2Expanded, simIvfPqParams(), simDevices(), simRanks, Replicated, ids) + idx, err := NewGpuIvfPq[float32, float32](ds, base, dim, L2Expanded, simIvfPqParams(), simDevices(), simRanks, Replicated, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -441,7 +441,7 @@ func TestSimulatedReplicatedCagra(t *testing.T) { count := uint64(64) ds, ids := simData(count, dim) - idx, err := NewGpuCagra[float32](ds, count, dim, L2Expanded, simCagraBuildParams(), simDevices(), simRanks, Replicated, ids) + idx, err := NewGpuCagra[float32, float32](ds, count, dim, L2Expanded, simCagraBuildParams(), simDevices(), simRanks, Replicated, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -473,7 +473,7 @@ func TestSimulatedShardedCagra(t *testing.T) { count := uint64(128) // 4 shards of 32 ds, ids := simData(count, dim) - idx, err := NewGpuCagra[float32](ds, count, dim, L2Expanded, simCagraBuildParams(), simDevices(), simRanks, Sharded, ids) + idx, err := NewGpuCagra[float32, float32](ds, count, dim, L2Expanded, simCagraBuildParams(), simDevices(), simRanks, Sharded, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -517,7 +517,7 @@ func TestSimulatedCagraSaveLoadAcrossModes(t *testing.T) { // Build REPLICATED under simulation and save the index files. { - idx, err := NewGpuCagra[float32](ds, count, dim, L2Expanded, simCagraBuildParams(), simDevices(), simRanks, Replicated, ids) + idx, err := NewGpuCagra[float32, float32](ds, count, dim, L2Expanded, simCagraBuildParams(), simDevices(), simRanks, Replicated, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -540,7 +540,7 @@ func TestSimulatedCagraSaveLoadAcrossModes(t *testing.T) { // Reload as REPLICATED (4 ranks). { - idx, err := NewGpuCagraFromDataDirectory[float32](dir, dim, L2Expanded, simCagraBuildParams(), simDevices(), simRanks, Replicated) + idx, err := NewGpuCagraFromDataDirectory[float32, float32](dir, dim, L2Expanded, simCagraBuildParams(), simDevices(), simRanks, Replicated) if err != nil { t.Fatalf("load replicated: %v", err) } @@ -557,7 +557,7 @@ func TestSimulatedCagraSaveLoadAcrossModes(t *testing.T) { // Reload the same files as SINGLE. { - idx, err := NewGpuCagraFromDataDirectory[float32](dir, dim, L2Expanded, simCagraBuildParams(), []int{0}, 1, SingleGpu) + idx, err := NewGpuCagraFromDataDirectory[float32, float32](dir, dim, L2Expanded, simCagraBuildParams(), []int{0}, 1, SingleGpu) if err != nil { t.Fatalf("load single: %v", err) } diff --git a/pkg/frontend/export.go b/pkg/frontend/export.go index 5d1b77c5739c9..7b928c78c96ad 100644 --- a/pkg/frontend/export.go +++ b/pkg/frontend/export.go @@ -718,6 +718,20 @@ func exportDataFromResultSetToCSVFile(oq *ExportConfig) error { } else if arr, ok := value.([]float64); ok { // this is for T_array_float64 type value = []byte(types.ArrayToString[float64](arr)) + } else if arr, ok := value.([]types.BF16); ok { + // this is for T_array_bf16 type + value = []byte(types.ArrayToString[types.BF16](arr)) + } else if arr, ok := value.([]types.Float16); ok { + // this is for T_array_float16 type + value = []byte(types.ArrayToString[types.Float16](arr)) + } else if arr, ok := value.([]int8); ok { + // this is for T_array_int8 type + value = []byte(types.ArrayToString[int8](arr)) + } else if s, ok := value.(string); ok { + // this is for T_array_uint8 (stored as its display string in + // extractRowFromVector, since []uint8 is indistinguishable from + // raw []byte) and any other string-valued varchar column + value = []byte(s) } if err = formatOutputString(oq, value.([]byte), symbol[i], closeby, true, buffer); err != nil { diff --git a/pkg/frontend/export_test.go b/pkg/frontend/export_test.go index c840ae594a9aa..8e3197e23ae53 100644 --- a/pkg/frontend/export_test.go +++ b/pkg/frontend/export_test.go @@ -265,6 +265,46 @@ func Test_exportDataToCSVFile(t *testing.T) { convey.So(exportDataFromResultSetToCSVFile(ep), convey.ShouldBeNil) }) + // Guards the narrow-vector export path: bf16/f16/int8 are emitted as their + // distinct slice types and vecuint8 as its display string (see + // extractRowFromVector). Before the fix the VARCHAR branch only special-cased + // []float32/[]float64 and then did value.([]byte) — a panic for []types.BF16 / + // []types.Float16 / []int8 and raw-byte corruption for []uint8. + convey.Convey("exportDataFromResultSetToCSVFile narrow vectors", t, func() { + ep := &ExportConfig{ + userConfig: &tree.ExportParam{ + Lines: &tree.Lines{TerminatedBy: &tree.Terminated{}}, + Fields: &tree.Fields{Terminated: &tree.Terminated{}, EnclosedBy: &tree.EnclosedBy{}, EscapedBy: &tree.EscapedBy{}}, + Header: true, + FilePath: "test/export_narrow.csv", + }, + mrs: &MysqlResultSet{}, + } + col := make([]MysqlColumn, 4) + for i := range col { + col[i].SetColumnType(defines.MYSQL_TYPE_VARCHAR) + ep.mrs.AddColumn(&col[i]) + } + f32 := []float32{1, 2, 3} + data := make([]interface{}, len(col)) + data[0] = types.Float32ToBF16Slice(f32) // bf16 slice + data[1] = types.Float32ToFloat16Slice(f32) // f16 slice + data[2] = []int8{1, 2, 3} // int8 slice + data[3] = types.ArrayToString[uint8]([]uint8{1, 2, 3}) // uint8 display string + ep.mrs.AddRow(data) + ep.Symbol = make([][]byte, len(col)) + ep.ColumnFlag = make([]bool, len(col)) + + stubs := gostub.StubFunc(&Close, nil) + defer stubs.Reset() + stubs = gostub.StubFunc(&openNewFile, nil) + defer stubs.Reset() + stubs = gostub.StubFunc(&writeDataToCSVFile, nil) + defer stubs.Reset() + + convey.So(exportDataFromResultSetToCSVFile(ep), convey.ShouldBeNil) + }) + convey.Convey("exportDataToCSVFile fail", t, func() { ep := &ExportConfig{ userConfig: &tree.ExportParam{ diff --git a/pkg/frontend/mysql_cmd_executor.go b/pkg/frontend/mysql_cmd_executor.go index f50308e7084c3..698e3104f5976 100644 --- a/pkg/frontend/mysql_cmd_executor.go +++ b/pkg/frontend/mysql_cmd_executor.go @@ -3982,7 +3982,8 @@ func convertEngineTypeToMysqlType(ctx context.Context, engineType types.T, col * col.SetColumnType(defines.MYSQL_TYPE_STRING) case types.T_varchar: col.SetColumnType(defines.MYSQL_TYPE_VAR_STRING) - case types.T_array_float32, types.T_array_float64: + case types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: col.SetColumnType(defines.MYSQL_TYPE_VARCHAR) case types.T_datalink: col.SetColumnType(defines.MYSQL_TYPE_TEXT) diff --git a/pkg/frontend/output.go b/pkg/frontend/output.go index 3dbb54c80d463..ee9f7609467ba 100644 --- a/pkg/frontend/output.go +++ b/pkg/frontend/output.go @@ -117,6 +117,36 @@ func extractRowFromVector(ctx context.Context, ses FeSession, vec *vector.Vector } else { row[i] = append([]float64(nil), arr...) } + case types.T_array_bf16: + arr := vector.GetArrayAt[types.BF16](vec, rowIndex) + if safeRefSlice { + row[i] = arr + } else { + row[i] = append([]types.BF16(nil), arr...) + } + case types.T_array_float16: + arr := vector.GetArrayAt[types.Float16](vec, rowIndex) + if safeRefSlice { + row[i] = arr + } else { + row[i] = append([]types.Float16(nil), arr...) + } + case types.T_array_int8: + arr := vector.GetArrayAt[int8](vec, rowIndex) + if safeRefSlice { + row[i] = arr + } else { + row[i] = append([]int8(nil), arr...) + } + case types.T_array_uint8: + // vecuint8's element slice is []uint8, which is indistinguishable from a + // raw []byte (binary/varbinary) value once the column is mapped to + // MYSQL_TYPE_VARCHAR. Every value-based consumer of mrs.Data (GetString, + // the legacy row encoders, CSV export) would then treat it as raw bytes + // and emit corrupt output. Store the display string instead so it is + // unambiguous; the main SELECT path (extractRowFromVector2/GetStringBased) + // is oid-aware and renders directly from the vector, unaffected by this. + row[i] = types.ArrayToString[uint8](vector.GetArrayAt[uint8](vec, rowIndex)) case types.T_date: row[i] = vector.GetFixedAtNoTypeCheck[types.Date](vec, rowIndex) case types.T_datetime: @@ -248,6 +278,34 @@ func extractRowFromVector2(ctx context.Context, ses FeSession, vec *vector.Vecto } else { row[i] = append([]float64(nil), arr...) } + case types.T_array_bf16: + arr := vector.GetArrayAt2[types.BF16](vec, colSlices.arrVarlena[sliceIdx], rowIndex) + if safeRefSlice { + row[i] = arr + } else { + row[i] = append([]types.BF16(nil), arr...) + } + case types.T_array_float16: + arr := vector.GetArrayAt2[types.Float16](vec, colSlices.arrVarlena[sliceIdx], rowIndex) + if safeRefSlice { + row[i] = arr + } else { + row[i] = append([]types.Float16(nil), arr...) + } + case types.T_array_int8: + arr := vector.GetArrayAt2[int8](vec, colSlices.arrVarlena[sliceIdx], rowIndex) + if safeRefSlice { + row[i] = arr + } else { + row[i] = append([]int8(nil), arr...) + } + case types.T_array_uint8: + arr := vector.GetArrayAt2[uint8](vec, colSlices.arrVarlena[sliceIdx], rowIndex) + if safeRefSlice { + row[i] = arr + } else { + row[i] = append([]uint8(nil), arr...) + } case types.T_date: row[i] = colSlices.arrDate[sliceIdx][rowIndex] case types.T_datetime: @@ -570,6 +628,14 @@ func (slices *ColumnSlices) GetStringBased(r uint64, i uint64) (string, error) { return types.ArrayToString[float32](vector.GetArrayAt2[float32](vec, slices.arrVarlena[sliceIdx], int(r))), nil case types.T_array_float64: return types.ArrayToString[float64](vector.GetArrayAt2[float64](vec, slices.arrVarlena[sliceIdx], int(r))), nil + case types.T_array_bf16: + return types.ArrayToString[types.BF16](vector.GetArrayAt2[types.BF16](vec, slices.arrVarlena[sliceIdx], int(r))), nil + case types.T_array_float16: + return types.ArrayToString[types.Float16](vector.GetArrayAt2[types.Float16](vec, slices.arrVarlena[sliceIdx], int(r))), nil + case types.T_array_int8: + return types.ArrayToString[int8](vector.GetArrayAt2[int8](vec, slices.arrVarlena[sliceIdx], int(r))), nil + case types.T_array_uint8: + return types.ArrayToString[uint8](vector.GetArrayAt2[uint8](vec, slices.arrVarlena[sliceIdx], int(r))), nil case types.T_Rowid: return slices.arrRowid[sliceIdx][r].String(), nil case types.T_Blockid: @@ -780,6 +846,9 @@ func convertVectorToSlice(ctx context.Context, ses FeSession, vec *vector.Vector case types.T_array_float64: colSlices.colIdx2SliceIdx[i] = len(colSlices.arrVarlena) colSlices.arrVarlena = append(colSlices.arrVarlena, vector.ToSliceNoTypeCheck2[types.Varlena](vec)) + case types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: + colSlices.colIdx2SliceIdx[i] = len(colSlices.arrVarlena) + colSlices.arrVarlena = append(colSlices.arrVarlena, vector.ToSliceNoTypeCheck2[types.Varlena](vec)) case types.T_date: colSlices.colIdx2SliceIdx[i] = len(colSlices.arrDate) colSlices.arrDate = append(colSlices.arrDate, vector.ToSliceNoTypeCheck2[types.Date](vec)) diff --git a/pkg/frontend/output_test.go b/pkg/frontend/output_test.go index 5625d7efb1e4c..5b155ac5a1c26 100644 --- a/pkg/frontend/output_test.go +++ b/pkg/frontend/output_test.go @@ -45,6 +45,79 @@ func TestExtractRowFromVector(t *testing.T) { } } +// TestExtractRowFromVectorNarrowVec guards the row-based (GetValue) display path +// for the narrow vector types. vecuint8 in particular must NOT be stored as +// []uint8: that is the same Go type as a raw []byte (binary/varbinary) value once +// the column is mapped to MYSQL_TYPE_VARCHAR, so every value-based consumer +// (GetString, the legacy row encoders, CSV export) would emit raw bytes / corrupt +// output. It is stored as its display string instead. bf16/f16/int8 stay as their +// distinct slice types (GetString renders them via ArrayToString). Every type must +// render as the human-readable "[1, 2, 3]" form, never raw bytes. +func TestExtractRowFromVectorNarrowVec(t *testing.T) { + mp := mpool.MustNewZero() + + f32 := []float32{1, 2, 3} + bf16 := types.Float32ToBF16Slice(f32) + f16 := types.Float32ToFloat16Slice(f32) + i8 := []int8{1, 2, 3} + u8 := []uint8{1, 2, 3} + + cases := []struct { + name string + oid types.T + bytes []byte + display string + assert func(t *testing.T, v any) + }{ + { + name: "bf16", oid: types.T_array_bf16, + bytes: types.ArrayToBytes[types.BF16](bf16), + display: types.ArrayToString[types.BF16](bf16), + assert: func(t *testing.T, v any) { _, ok := v.([]types.BF16); require.Truef(t, ok, "got %T", v) }, + }, + { + name: "f16", oid: types.T_array_float16, + bytes: types.ArrayToBytes[types.Float16](f16), + display: types.ArrayToString[types.Float16](f16), + assert: func(t *testing.T, v any) { _, ok := v.([]types.Float16); require.Truef(t, ok, "got %T", v) }, + }, + { + name: "int8", oid: types.T_array_int8, + bytes: types.ArrayToBytes[int8](i8), + display: types.ArrayToString[int8](i8), + assert: func(t *testing.T, v any) { _, ok := v.([]int8); require.Truef(t, ok, "got %T", v) }, + }, + { + name: "uint8", oid: types.T_array_uint8, + bytes: types.ArrayToBytes[uint8](u8), + display: types.ArrayToString[uint8](u8), + assert: func(t *testing.T, v any) { + s, ok := v.(string) + require.Truef(t, ok, "vecuint8 must be stored as a string (not []uint8/[]byte), got %T", v) + require.Equal(t, types.ArrayToString[uint8](u8), s) + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + vec := vector.NewVec(c.oid.ToType()) + require.NoError(t, vector.AppendAny(vec, c.bytes, false, mp)) + + row := make([]any, 1) + require.NoError(t, extractRowFromVector(context.TODO(), nil, vec, 0, row, 0, false)) + c.assert(t, row[0]) + + mrs := &MysqlResultSet{} + mrs.Data = [][]any{{row[0]}} + mrs.Columns = make([]Column, 1) + got, err := mrs.GetString(context.TODO(), 0, 0) + require.NoError(t, err) + require.Equal(t, c.display, got) + }) + } +} + func BenchmarkName(b *testing.B) { mp := mpool.MustNewZero() diff --git a/pkg/frontend/resultset.go b/pkg/frontend/resultset.go index 499c7c9b219b6..0daed86cf6221 100644 --- a/pkg/frontend/resultset.go +++ b/pkg/frontend/resultset.go @@ -473,6 +473,12 @@ func (mrs *MysqlResultSet) GetString(ctx context.Context, rindex, cindex uint64) return types.ArrayToString[float32](v), nil case []float64: return types.ArrayToString[float64](v), nil + case []types.BF16: + return types.ArrayToString[types.BF16](v), nil + case []types.Float16: + return types.ArrayToString[types.Float16](v), nil + case []int8: + return types.ArrayToString[int8](v), nil case int: return strconv.FormatInt(int64(v), 10), nil case uint: diff --git a/pkg/frontend/util.go b/pkg/frontend/util.go index f5478ec8af48f..7d6255391bf0e 100644 --- a/pkg/frontend/util.go +++ b/pkg/frontend/util.go @@ -352,6 +352,14 @@ func getValueFromVector(ctx context.Context, vec *vector.Vector, feSes FeSession return vector.GetArrayAt[float32](vec, 0), nil case types.T_array_float64: return vector.GetArrayAt[float64](vec, 0), nil + case types.T_array_bf16: + return vector.GetArrayAt[types.BF16](vec, 0), nil + case types.T_array_float16: + return vector.GetArrayAt[types.Float16](vec, 0), nil + case types.T_array_int8: + return vector.GetArrayAt[int8](vec, 0), nil + case types.T_array_uint8: + return vector.GetArrayAt[uint8](vec, 0), nil case types.T_decimal64: val := vector.GetFixedAtNoTypeCheck[types.Decimal64](vec, 0) return val.Format(expr.Typ.Scale), nil diff --git a/pkg/frontend/variables.go b/pkg/frontend/variables.go index e917ea9eb5788..0830a17715b75 100644 --- a/pkg/frontend/variables.go +++ b/pkg/frontend/variables.go @@ -3739,6 +3739,14 @@ var gSysVarsDefs = map[string]SystemVariable{ Type: InitSystemVariableBoolType("experimental_fulltext_index"), Default: int8(0), }, + "experimental_bm25_index": { + Name: "experimental_bm25_index", + Scope: ScopeBoth, + Dynamic: true, + SetVarHintApplies: false, + Type: InitSystemVariableBoolType("experimental_bm25_index"), + Default: int8(0), + }, "ft_relevancy_algorithm": { Name: fulltext.FulltextRelevancyAlgo, Scope: ScopeBoth, diff --git a/pkg/fulltext/plugin/compile/compile.go b/pkg/fulltext/plugin/compile/compile.go index 7ad4a9711e3ca..16e32a0ac5ddd 100644 --- a/pkg/fulltext/plugin/compile/compile.go +++ b/pkg/fulltext/plugin/compile/compile.go @@ -108,7 +108,7 @@ func (Hooks) HandleCreateIndex(ctx compileplugin.CompileContext, indexDefs map[s } // HandleReindex — fulltext does not support ALTER … REINDEX. -func (Hooks) HandleReindex(_ compileplugin.CompileContext, _ map[string]*plan.IndexDef, _ bool) error { +func (Hooks) HandleReindex(_ compileplugin.CompileContext, _ map[string]*plan.IndexDef, _ bool, _ bool) error { return moerr.NewNotSupportedNoCtx("ALTER ... REINDEX is not supported for fulltext indexes") } diff --git a/pkg/fulltext/plugin/runtime/runtime.go b/pkg/fulltext/plugin/runtime/runtime.go index fc984170bc1d4..b83beb8c482aa 100644 --- a/pkg/fulltext/plugin/runtime/runtime.go +++ b/pkg/fulltext/plugin/runtime/runtime.go @@ -88,6 +88,9 @@ func (CatalogHooks) SupportedVectorTypes() []types.T { return nil } // SupportedPrimaryKeyTypes: fulltext imposes no PK-type constraint. func (CatalogHooks) SupportedPrimaryKeyTypes() []types.T { return nil } +// ValidQuantization — full-text indexes have no quantization, so nothing to gate. +func (CatalogHooks) ValidQuantization(_, _ string) error { return nil } + // SupportedOpTypes — fulltext has no metric/op-type concept. // SupportedIncludeColumnTypes: this index has no INCLUDE-column support. func (CatalogHooks) SupportedIncludeColumnTypes() []types.T { return nil } diff --git a/pkg/indexplugin/all/all.go b/pkg/indexplugin/all/all.go index 4487956f7efb5..7074c657a0c9b 100644 --- a/pkg/indexplugin/all/all.go +++ b/pkg/indexplugin/all/all.go @@ -64,6 +64,7 @@ package all import ( + _ "github.com/matrixorigin/matrixone/pkg/bm25/plugin" _ "github.com/matrixorigin/matrixone/pkg/fulltext/plugin" _ "github.com/matrixorigin/matrixone/pkg/vectorindex/hnsw/plugin" _ "github.com/matrixorigin/matrixone/pkg/vectorindex/ivfflat/plugin" diff --git a/pkg/indexplugin/catalog/hooks.go b/pkg/indexplugin/catalog/hooks.go index 4e446db7cac8b..0531fc9d18d68 100644 --- a/pkg/indexplugin/catalog/hooks.go +++ b/pkg/indexplugin/catalog/hooks.go @@ -83,6 +83,18 @@ type Hooks interface { // and fulltext return nil, so SupportsIncludeColumnType reports false. SupportedIncludeColumnTypes() []types.T + // ValidQuantization reports whether QUANTIZATION='quant' is usable by this + // algorithm under op_type 'op', returning a descriptive error when not + // (nil = valid). It is the single per-algorithm rule for the + // (quantization, op_type) pair, so CREATE (plan-side schema validation) and + // REINDEX (compile-side ValidateReindexParams) gate it identically instead + // of duplicating the check. An empty quant means "no quantization / default + // storage" (valid); an empty op means "no metric in play" (only the + // storage-type rule applies). Example: the cuvs (CAGRA / IVF-PQ) backend + // rejects int8/uint8 with inner-product / cosine because its affine scalar + // quantizer only preserves L2 geometry. + ValidQuantization(quant, op string) error + // ExperimentalFlag returns the experimental-feature flag name that // must be enabled (set to true via SET / system var) for this // algorithm to be usable. Returns "" for non-experimental @@ -264,6 +276,13 @@ type SyncDescriptor struct { // false for cuvs algorithms (CAGRA, IVF-PQ) which have no "lists" // or training-sample concept — they always rebuild on cadence. IdxcronListsAware bool + + // IdxcronReindexOption is an extra REINDEX option keyword the executor + // inserts before FORCE_SYNC when building the cron-triggered ALTER — + // e.g. "MERGE" so a bm25 index runs incremental fold+tiered compaction + // instead of a full rebuild-from-source. Empty for the vector algorithms + // (plain rebuild); the executor omits it when unset. + IdxcronReindexOption string } // AlterTableCloneBehavior declares the per-hidden-table semantics diff --git a/pkg/indexplugin/compile/hooks.go b/pkg/indexplugin/compile/hooks.go index 04bd3beaa3493..9f0e2b8b77f64 100644 --- a/pkg/indexplugin/compile/hooks.go +++ b/pkg/indexplugin/compile/hooks.go @@ -155,8 +155,11 @@ type Hooks interface { // HandleReindex is the ALTER … REINDEX path. forceSync mirrors the // existing IVF-FLAT semantics (run synchronously inside the txn) and is - // ignored by algorithms that do not support it. - HandleReindex(ctx CompileContext, indexDefs map[string]*plan.IndexDef, forceSync bool) error + // ignored by algorithms that do not support it. merge requests incremental + // compaction (fold + tiered merge of already-built segments) instead of a + // full rebuild-from-source; only the bm25 index honors it, every other + // algorithm ignores it and rebuilds. + HandleReindex(ctx CompileContext, indexDefs map[string]*plan.IndexDef, forceSync, merge bool) error // RestoreInitSQL returns (startFromNow, initSQL) for the restored index's // CDC. initSQL rebuilds the index from the cloned rows — run post-commit by diff --git a/pkg/indexplugin/idxcron/hooks.go b/pkg/indexplugin/idxcron/hooks.go index 2430601391ecf..101378011d031 100644 --- a/pkg/indexplugin/idxcron/hooks.go +++ b/pkg/indexplugin/idxcron/hooks.go @@ -92,3 +92,13 @@ type Hooks interface { // IVF-FLAT, trivial-true for HNSW / fulltext. Updatable(in UpdatableInput) (ok bool, reason string, err error) } + +// ReindexOptioner is an OPTIONAL idxcron hook (checked via type assertion). A plugin that +// implements it chooses the REINDEX option per cron fire, OVERRIDING the static +// SyncDescriptor.IdxcronReindexOption. Fulltext uses it to return "MERGE" (incremental +// fold+tiered compaction) normally, but "" (a full REBUILD) once the dead-doc fraction is +// high enough that a rebuild reclaims more than it costs. Plugins that don't implement it +// keep their descriptor's fixed option. Called only when Updatable returned ok. +type ReindexOptioner interface { + ReindexOption(in UpdatableInput) (option string, err error) +} diff --git a/pkg/indexplugin/iscp/import.go b/pkg/indexplugin/iscp/import.go index 48befbf68559a..d257f351b63a7 100644 --- a/pkg/indexplugin/iscp/import.go +++ b/pkg/indexplugin/iscp/import.go @@ -37,6 +37,7 @@ package iscp import ( + _ "github.com/matrixorigin/matrixone/pkg/bm25/plugin/iscp" _ "github.com/matrixorigin/matrixone/pkg/fulltext/plugin/iscp" _ "github.com/matrixorigin/matrixone/pkg/vectorindex/hnsw/plugin/iscp" _ "github.com/matrixorigin/matrixone/pkg/vectorindex/ivfflat/plugin/iscp" diff --git a/pkg/iscp/cuvs_writer.go b/pkg/iscp/cuvs_writer.go index d79e18fd7ed3a..3a1099de16164 100644 --- a/pkg/iscp/cuvs_writer.go +++ b/pkg/iscp/cuvs_writer.go @@ -22,6 +22,7 @@ import ( "github.com/bytedance/sonic" "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/pb/plan" cuvscdc "github.com/matrixorigin/matrixone/pkg/vectorindex/cuvs" @@ -64,6 +65,8 @@ type CuvsCdcWriter struct { pkPos int32 partsPos []int32 dimension int32 + baseType types.T // vector column element type: vecf32 or vecf16 + vecBytesPer int // dim * base element size (4*dim for f32, 2*dim for f16) dbName string tblName string indexName string @@ -78,8 +81,8 @@ type CuvsCdcWriter struct { // switch on it); dbName/tblName/indexName are typically pulled from // the ISCP ConsumerInfo at the call site. // -// Both CAGRA and IVF-PQ on cuvs are fp32-only with a bigint PK; this -// constructor enforces both shapes. +// Both CAGRA and IVF-PQ on cuvs take a vecf32 or vecf16 base column with +// a bigint PK; this constructor enforces both shapes. func NewCuvsCdcWriter(algoName, dbName, tblName, indexName string, tabledef *plan.TableDef, indexdefs []*plan.IndexDef) (*CuvsCdcWriter, error) { @@ -121,11 +124,23 @@ func NewCuvsCdcWriter(algoName, dbName, tblName, indexName string, w.partsPos[i] = tabledef.Name2ColIndex[part] } vecTyp := tabledef.Cols[w.partsPos[0]].Typ - if vecTyp.Id != int32(types.T_array_float32) { + // cuvs accepts a vecf32 or vecf16 base column. The CDC record carries the + // vector as raw native base-type bytes (4*dim for f32, 2*dim for f16); the + // search-side overflow replay reinterprets them back to the base type B. + var baseElemSize int + switch types.T(vecTyp.Id) { + case types.T_array_float32: + w.baseType = types.T_array_float32 + baseElemSize = 4 + case types.T_array_float16: + w.baseType = types.T_array_float16 + baseElemSize = 2 + default: return nil, moerr.NewInternalErrorNoCtx(fmt.Sprintf( - "%s cuvs writer: vector column must be vecf32 (cuvs is fp32-only)", algoName)) + "%s cuvs writer: vector column must be vecf32 or vecf16", algoName)) } w.dimension = vecTyp.Width + w.vecBytesPer = int(vecTyp.Width) * baseElemSize // Resolve INCLUDE columns from indexAlgoParams. Returns zero // values when no INCLUDE columns are configured. @@ -172,6 +187,7 @@ func (w *CuvsCdcWriter) IndexName() string { return w.indexName } func (w *CuvsCdcWriter) IndexDef() []*plan.IndexDef { return w.indexdef } func (w *CuvsCdcWriter) Dimension() int32 { return w.dimension } func (w *CuvsCdcWriter) ColMetaJSON() string { return w.colMetaJSON } +func (w *CuvsCdcWriter) BaseVectorType() types.T { return w.baseType } // IndexSqlWriter implementation // @@ -218,7 +234,7 @@ func (w *CuvsCdcWriter) ToSql() ([]byte, error) { func (w *CuvsCdcWriter) appendDelete(key int64) error { out, err := cuvscdc.EncodeEventRecord(w.pendingRecords, cuvscdc.CdcOpDelete, - key, nil, nil, int(w.dimension), w.includeBytesPer) + key, nil, nil, w.vecBytesPer, w.includeBytesPer) if err != nil { return err } @@ -238,25 +254,36 @@ func (w *CuvsCdcWriter) encodeInsertOrUpsert(ctx context.Context, row []any, op // has a vector to index). return w.appendDelete(key) } - v, ok := rawVec.([]float32) - if !ok { - // A non-nil value of the wrong type is a real schema/type error, not a - // NULL vector — surface it instead of silently dropping the row to a - // DELETE (mirrors the HNSW sinker in index_sqlwriter.go). + // Extract the native base-type bytes verbatim. A vecf32 column arrives as + // []float32 (4 bytes/element), a vecf16 column as []types.Float16 (2 + // bytes/element); EncodeEventRecord validates the byte length against + // w.vecBytesPer. A typed-nil slice is an actually-absent vector → DELETE; + // any other type is a real schema error (mirrors the HNSW sinker). + var vecBytes []byte + switch v := rawVec.(type) { + case []float32: + if v == nil { + return w.appendDelete(key) + } + vecBytes = util.UnsafeSliceToBytes(v) + case []types.Float16: + if v == nil { + return w.appendDelete(key) + } + vecBytes = util.UnsafeSliceToBytes(v) + default: return moerr.NewInternalError(ctx, fmt.Sprintf( - "%s cuvs writer: invalid vector type, expected []float32, got %T", w.algoName, rawVec)) - } - if v == nil { - // Typed-nil slice — an actually absent vector; encode as DELETE. - return w.appendDelete(key) + "%s cuvs writer: invalid vector type, expected []float32 or []types.Float16, got %T", + w.algoName, rawVec)) } includeBytes, err := cuvscdc.EncodeIncludeRow(w.includeBindings, row, w.includeBytesPer) if err != nil { return err } + // Pass the raw native base-type bytes (4*dim for f32, 2*dim for f16). out, err := cuvscdc.EncodeEventRecord(w.pendingRecords, op, - key, v, includeBytes, int(w.dimension), w.includeBytesPer) + key, vecBytes, includeBytes, w.vecBytesPer, w.includeBytesPer) if err != nil { return err } diff --git a/pkg/iscp/cuvs_writer_test.go b/pkg/iscp/cuvs_writer_test.go index 47f31af51e8af..797fb807b6cff 100644 --- a/pkg/iscp/cuvs_writer_test.go +++ b/pkg/iscp/cuvs_writer_test.go @@ -203,7 +203,7 @@ func TestNewCuvsCdcWriter_RejectsVecF64(t *testing.T) { td.Cols[1].Typ.Id = int32(types.T_array_float64) _, err := NewCuvsCdcWriter("ivfpq", "db", "tbl", "idx", td, newTestCuvsIndexDefs(td)) require.Error(t, err) - require.Contains(t, err.Error(), "fp32-only") + require.Contains(t, err.Error(), "vecf32 or vecf16") } // --------------------------------------------------------------------------- diff --git a/pkg/iscp/index_consumer.go b/pkg/iscp/index_consumer.go index 4ae034c395395..bf5d68ac52b0f 100644 --- a/pkg/iscp/index_consumer.go +++ b/pkg/iscp/index_consumer.go @@ -447,12 +447,23 @@ func (c *IndexConsumer) Consume(ctx context.Context, r DataRetriever) error { return nil } +// valueRepr picks the value representation the paired writer needs: the WAND +// retrieval writer binary-encodes the pk (encodePk), so it needs native Go values; +// every other writer builds SQL text and needs the SQL-display string (the historical +// default). Keeping it a single helper localizes the one writer that opts into native. +func (c *IndexConsumer) valueRepr() ValueRepr { + if _, ok := c.sqlWriter.(*WandSqlWriter); ok { + return ReprNative + } + return ReprSQLString +} + func (c *IndexConsumer) sinkSnapshot(ctx context.Context, upsertBatch *AtomicBatch) error { var err error for _, bat := range upsertBatch.Batches { for i := 0; i < batchRowCount(bat); i++ { - if err = extractRowFromEveryVector(ctx, bat, i, c.rowdata); err != nil { + if err = extractRowFromEveryVector(ctx, bat, i, c.rowdata, c.valueRepr()); err != nil { return err } @@ -529,7 +540,7 @@ func (c *IndexConsumer) sinkTail(ctx context.Context, upsertBatch, deleteBatch * func (c *IndexConsumer) sinkInsert(ctx context.Context, upsertIter *atomicBatchRowIter) (err error) { // get row from the batch - if err = upsertIter.Row(ctx, c.rowdata); err != nil { + if err = upsertIter.Row(ctx, c.rowdata, c.valueRepr()); err != nil { return err } @@ -562,7 +573,7 @@ func (c *IndexConsumer) sinkInsert(ctx context.Context, upsertIter *atomicBatchR func (c *IndexConsumer) sinkDelete(ctx context.Context, deleteIter *atomicBatchRowIter) (err error) { // get row from the batch - if err = deleteIter.Row(ctx, c.rowdelete); err != nil { + if err = deleteIter.Row(ctx, c.rowdelete, c.valueRepr()); err != nil { return err } diff --git a/pkg/iscp/index_sqlwriter.go b/pkg/iscp/index_sqlwriter.go index 33157069f50a8..96384f58b0e88 100644 --- a/pkg/iscp/index_sqlwriter.go +++ b/pkg/iscp/index_sqlwriter.go @@ -28,6 +28,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/vectorindex" "github.com/matrixorigin/matrixone/pkg/vectorindex/hnsw" + "github.com/matrixorigin/matrixone/pkg/vectorindex/quantizer" "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" ) @@ -668,7 +669,36 @@ func (w *IvfflatSqlWriter) toIvfflatUpsert(upsert bool) ([]byte, error) { } cols := strings.Join(coldefs, ", ") - cnames_str := strings.Join(cnames, ", ") + + // Entry projection. The last src column is the vector that becomes the entry. + // For int8 QUANTIZATION the entry must be scaled by the trained quantizer + // (q(x)=x*mul+add, mul=255/(max-min), add=-min*mul-128) just like the + // synchronous build (compile.go) and search; otherwise the implicit + // vecf32->vecint8 cast on REPLACE does identity round+clamp and every + // CDC-maintained row gets wrong int8 codes. min/max come from the metadata + // table; COALESCE falls back to identity (mul=1,add=0) when they are absent + // (pure-async indexes that never trained bounds — search also uses identity + // there, so the two stay consistent). float16/bf16 narrow losslessly via the + // implicit cast, so only int8 needs this. + entryProj := cnames[len(cnames)-1] + if qt, ok := quantizer.ToVectorType(w.ivfparam.Quantization); ok && (qt == types.T_array_int8 || qt == types.T_array_uint8) { + metaTbl := sqlquote.QualifiedIdent(w.info.DBName, w.meta_tbl) + sub := func(k string) string { + return fmt.Sprintf("(SELECT CAST(`%s` AS DOUBLE) FROM %s WHERE `%s` = '%s')", + catalog.SystemSI_IVFFLAT_TblCol_Metadata_val, metaTbl, + catalog.SystemSI_IVFFLAT_TblCol_Metadata_key, k) + } + minS := sub(catalog.SystemSI_IVFFLAT_Metadata_QuantizeMin) + maxS := sub(catalog.SystemSI_IVFFLAT_Metadata_QuantizeMax) + if qt == types.T_array_uint8 { + entryProj = quantizer.Uint8EntrySQLFromBounds(cnames[len(cnames)-1], minS, maxS, w.partsType[0].Width) + } else { + entryProj = quantizer.Int8EntrySQLFromBounds(cnames[len(cnames)-1], minS, maxS, w.partsType[0].Width) + } + } + projCols := append([]string(nil), cnames...) + projCols[len(projCols)-1] = entryProj + cnames_str := strings.Join(projCols, ", ") if upsert { sql += fmt.Sprintf("REPLACE INTO %s ", sqlquote.QualifiedIdent(w.info.DBName, w.entries_tbl)) diff --git a/pkg/iscp/initsql_test.go b/pkg/iscp/initsql_test.go new file mode 100644 index 0000000000000..6d12dc3949e96 --- /dev/null +++ b/pkg/iscp/initsql_test.go @@ -0,0 +1,57 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iscp + +import ( + "encoding/json" + "reflect" + "testing" +) + +// TestSplitInitSQL covers the multi-statement InitSQL format: a JSON array of +// statements (new), a JSON string (one statement), and a raw non-JSON statement +// (backward-compat for pre-existing InitSQLs like "SELECT 1"). +func TestSplitInitSQL(t *testing.T) { + cases := []struct { + name string + in string + want []string + }{ + {"empty", "", nil}, + {"json array", `["INSERT INTO p SELECT ...", "SELECT f.* FROM p CROSS APPLY fulltext_wand_create(...)"]`, + []string{"INSERT INTO p SELECT ...", "SELECT f.* FROM p CROSS APPLY fulltext_wand_create(...)"}}, + {"json string", `"SELECT 1"`, []string{"SELECT 1"}}, + {"raw single", "SELECT 1", []string{"SELECT 1"}}, + {"raw insert", "INSERT INTO t VALUES (1)", []string{"INSERT INTO t VALUES (1)"}}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := splitInitSQL(c.in); !reflect.DeepEqual(got, c.want) { + t.Fatalf("splitInitSQL(%q) = %v, want %v", c.in, got, c.want) + } + }) + } + + // Round-trip: what the retrieval producer emits (json.Marshal of a []string) + // must split back to the same statements. + stmts := []string{"INSERT INTO db.posting SELECT ...", "SELECT f.* FROM db.posting CROSS APPLY fulltext_wand_create(...)"} + js, err := json.Marshal(stmts) + if err != nil { + t.Fatal(err) + } + if got := splitInitSQL(string(js)); !reflect.DeepEqual(got, stmts) { + t.Fatalf("round-trip: got %v, want %v", got, stmts) + } +} diff --git a/pkg/iscp/iteration.go b/pkg/iscp/iteration.go index fdd05dd65f467..334fb5687ae46 100644 --- a/pkg/iscp/iteration.go +++ b/pkg/iscp/iteration.go @@ -18,6 +18,7 @@ import ( "bytes" "context" "encoding/base64" + "encoding/json" "errors" "fmt" "strings" @@ -803,7 +804,17 @@ func ProcessInitSQL( 0) txnOp, err := cnTxnClient.New(ctx, nowTs, createByOpt) if txnOp != nil { - defer txnOp.Commit(ctx) + // Commit only when every InitSQL statement succeeded; roll back on any + // error so a multi-statement InitSQL (postings-populate + WAND build) is + // atomic — a mid-sequence failure must not leave the earlier statements + // committed (which an ISCP retry would then re-apply). + defer func() { + if err != nil { + _ = txnOp.Rollback(ctx) + } else { + err = txnOp.Commit(ctx) + } + }() } // injection is for ut @@ -850,10 +861,38 @@ func ProcessInitSQL( } sqlctx := sqlexec.NewSqlContext(ctx, cnUUID, txnOp, accountId, resolver) sqlproc := sqlexec.NewSqlProcessWithContext(sqlctx) - result, err := sqlexec.RunSql(sqlproc, sql) - if err != nil { - return + // InitSQL is a JSON array of statements (the multi-statement form), or a JSON + // string / raw single statement (backward-compat). ISCP has no multi-statement + // executor, so run each in sequence within this txn. + for _, stmt := range splitInitSQL(sql) { + if stmt == "" { + continue + } + res, e := sqlexec.RunSql(sqlproc, stmt) + if e != nil { + err = e + return + } + res.Close() } - defer result.Close() return } + +// splitInitSQL parses a decoded InitSQL payload into individual statements. The +// canonical form is a JSON array of statements; a JSON string is one statement; +// anything that isn't valid JSON is treated as a single raw statement so +// pre-existing InitSQLs (e.g. "SELECT 1", cagra/ivfpq builds) keep working. +func splitInitSQL(s string) []string { + if s == "" { + return nil + } + var arr []string + if json.Unmarshal([]byte(s), &arr) == nil { + return arr + } + var one string + if json.Unmarshal([]byte(s), &one) == nil { + return []string{one} + } + return []string{s} +} diff --git a/pkg/iscp/mock_consumer.go b/pkg/iscp/mock_consumer.go index fa2c813ba5dd4..d125ec5d6fe22 100644 --- a/pkg/iscp/mock_consumer.go +++ b/pkg/iscp/mock_consumer.go @@ -331,7 +331,7 @@ func (s *interalSqlConsumer) sinkSnapshot(ctx context.Context, bat *AtomicBatch) s.preRowType = UpsertRow } // step1: get row from the batch - if err = extractRowFromEveryVector(ctx, bat, i, s.insertRow); err != nil { + if err = extractRowFromEveryVector(ctx, bat, i, s.insertRow, ReprSQLString); err != nil { panic(err) } @@ -412,7 +412,7 @@ func (s *interalSqlConsumer) sinkInsert(ctx context.Context, insertIter *atomicB } // step1: get row from the batch - if err = insertIter.Row(ctx, s.insertRow); err != nil { + if err = insertIter.Row(ctx, s.insertRow, ReprSQLString); err != nil { return } @@ -436,7 +436,7 @@ func (s *interalSqlConsumer) sinkDelete(ctx context.Context, deleteIter *atomicB } // step1: get row from the batch - if err = deleteIter.Row(ctx, s.deleteRow); err != nil { + if err = deleteIter.Row(ctx, s.deleteRow, ReprSQLString); err != nil { return } diff --git a/pkg/iscp/types.go b/pkg/iscp/types.go index a810a90598e13..245e2303c1ffd 100644 --- a/pkg/iscp/types.go +++ b/pkg/iscp/types.go @@ -262,7 +262,7 @@ const ( type RowIterator interface { Next() bool - Row(ctx context.Context, row []any) error + Row(ctx context.Context, row []any, repr ValueRepr) error Close() } @@ -384,13 +384,14 @@ func (iter *atomicBatchRowIter) Next() bool { return iter.iter.Next() } -func (iter *atomicBatchRowIter) Row(ctx context.Context, row []any) error { +func (iter *atomicBatchRowIter) Row(ctx context.Context, row []any, repr ValueRepr) error { batchRow := iter.iter.Item() return extractRowFromEveryVector( ctx, batchRow.Src, batchRow.Offset, row, + repr, ) } diff --git a/pkg/iscp/util.go b/pkg/iscp/util.go index 6cbca560d8a88..749801c4935be 100644 --- a/pkg/iscp/util.go +++ b/pkg/iscp/util.go @@ -42,6 +42,21 @@ import ( "github.com/matrixorigin/matrixone/pkg/vm/engine" ) +// ValueRepr selects how extractRowFromVector renders the types whose native Go +// value and SQL-display string differ (temporal / decimal / uuid). A consumer that +// builds SQL text needs the display string; a consumer that binary-encodes the value +// (the WAND retrieval index) needs the native value so it can serialize it exactly. +type ValueRepr int + +const ( + // ReprSQLString is the historical behavior: datetime/time/timestamp/decimal/uuid + // come out as their SQL-display string (feeds convertColIntoSql). Default. + ReprSQLString ValueRepr = iota + // ReprNative returns those types as their native Go value (types.Datetime, + // types.Decimal128, types.Uuid, ...) so a binary encoder round-trips them exactly. + ReprNative +) + // extractRowFromEveryVector gets the j row from the every vector and outputs the row // bat columns layout: // 1. data: user defined cols | cpk (if needed) | commit-ts @@ -52,6 +67,7 @@ func extractRowFromEveryVector( dataSet *batch.Batch, rowIndex int, row []any, + repr ValueRepr, ) error { for i := 0; i < len(row); i++ { vec := dataSet.Vecs[i] @@ -64,7 +80,7 @@ func extractRowFromEveryVector( rowIndex = 0 } - if err := extractRowFromVector(ctx, vec, i, row, rowIndex); err != nil { + if err := extractRowFromVector(ctx, vec, i, row, rowIndex, repr); err != nil { return err } rowIndex = rowIndexBackup @@ -73,7 +89,7 @@ func extractRowFromEveryVector( } // extractRowFromVector gets the rowIndex row from the i vector -func extractRowFromVector(ctx context.Context, vec *vector.Vector, i int, row []any, rowIndex int) error { +func extractRowFromVector(ctx context.Context, vec *vector.Vector, i int, row []any, rowIndex int, repr ValueRepr) error { if vec.IsConstNull() || vec.GetNulls().Contains(uint64(rowIndex)) { row[i] = nil return nil @@ -116,30 +132,64 @@ func extractRowFromVector(ctx context.Context, vec *vector.Vector, i int, row [] //| �? @ @@ | //+------------------------------+ row[i] = vector.GetArrayAt[float32](vec, rowIndex) + case types.T_array_float16: + // vecf16: extract natively as []types.Float16 (2 bytes/element). The + // cuvs CDC writer reinterprets these bytes verbatim — no f32 widening. + row[i] = vector.GetArrayAt[types.Float16](vec, rowIndex) + case types.T_array_bf16: + row[i] = vector.GetArrayAt[types.BF16](vec, rowIndex) + case types.T_array_int8: + row[i] = vector.GetArrayAt[int8](vec, rowIndex) + case types.T_array_uint8: + row[i] = vector.GetArrayAt[uint8](vec, rowIndex) case types.T_array_float64: row[i] = vector.GetArrayAt[float64](vec, rowIndex) case types.T_date: row[i] = vector.GetFixedAtWithTypeCheck[types.Date](vec, rowIndex) case types.T_datetime: - scale := vec.GetType().Scale - row[i] = vector.GetFixedAtWithTypeCheck[types.Datetime](vec, rowIndex).String2(scale) + if repr == ReprNative { + row[i] = vector.GetFixedAtWithTypeCheck[types.Datetime](vec, rowIndex) + } else { + scale := vec.GetType().Scale + row[i] = vector.GetFixedAtWithTypeCheck[types.Datetime](vec, rowIndex).String2(scale) + } case types.T_time: - scale := vec.GetType().Scale - row[i] = vector.GetFixedAtWithTypeCheck[types.Time](vec, rowIndex).String2(scale) + if repr == ReprNative { + row[i] = vector.GetFixedAtWithTypeCheck[types.Time](vec, rowIndex) + } else { + scale := vec.GetType().Scale + row[i] = vector.GetFixedAtWithTypeCheck[types.Time](vec, rowIndex).String2(scale) + } case types.T_timestamp: - scale := vec.GetType().Scale - //TODO:get the right timezone - //timeZone := ses.GetTimeZone() - timeZone := time.UTC - row[i] = vector.GetFixedAtWithTypeCheck[types.Timestamp](vec, rowIndex).String2(timeZone, scale) + if repr == ReprNative { + row[i] = vector.GetFixedAtWithTypeCheck[types.Timestamp](vec, rowIndex) + } else { + scale := vec.GetType().Scale + //TODO:get the right timezone + //timeZone := ses.GetTimeZone() + timeZone := time.UTC + row[i] = vector.GetFixedAtWithTypeCheck[types.Timestamp](vec, rowIndex).String2(timeZone, scale) + } case types.T_decimal64: - scale := vec.GetType().Scale - row[i] = vector.GetFixedAtWithTypeCheck[types.Decimal64](vec, rowIndex).Format(scale) + if repr == ReprNative { + row[i] = vector.GetFixedAtWithTypeCheck[types.Decimal64](vec, rowIndex) + } else { + scale := vec.GetType().Scale + row[i] = vector.GetFixedAtWithTypeCheck[types.Decimal64](vec, rowIndex).Format(scale) + } case types.T_decimal128: - scale := vec.GetType().Scale - row[i] = vector.GetFixedAtWithTypeCheck[types.Decimal128](vec, rowIndex).Format(scale) + if repr == ReprNative { + row[i] = vector.GetFixedAtWithTypeCheck[types.Decimal128](vec, rowIndex) + } else { + scale := vec.GetType().Scale + row[i] = vector.GetFixedAtWithTypeCheck[types.Decimal128](vec, rowIndex).Format(scale) + } case types.T_uuid: - row[i] = vector.GetFixedAtWithTypeCheck[types.Uuid](vec, rowIndex).String() + if repr == ReprNative { + row[i] = vector.GetFixedAtWithTypeCheck[types.Uuid](vec, rowIndex) + } else { + row[i] = vector.GetFixedAtWithTypeCheck[types.Uuid](vec, rowIndex).String() + } case types.T_Rowid: row[i] = vector.GetFixedAtWithTypeCheck[types.Rowid](vec, rowIndex) case types.T_Blockid: @@ -257,6 +307,23 @@ func convertColIntoSql( value := data.([]float64) typstr := typ.DescString() sqlBuff = appendString(sqlBuff, fmt.Sprintf("CAST('%s' as %s)", types.ArrayToString(value), typstr)) + case types.T_array_float16: + // Narrow base columns (vecf16/bf16/int8/uint8). ArrayToString renders the + // half/bf16 bit pattern back to its decimal value and the int8/uint8 codes + // to integers, so CAST('[...]' as vecXXX(n)) reconstructs the same vector + // the ivfflat entry projection expects (matches the synchronous build, + // which reads the base column directly in SQL). + value := data.([]types.Float16) + sqlBuff = appendString(sqlBuff, fmt.Sprintf("CAST('%s' as %s)", types.ArrayToString(value), typ.DescString())) + case types.T_array_bf16: + value := data.([]types.BF16) + sqlBuff = appendString(sqlBuff, fmt.Sprintf("CAST('%s' as %s)", types.ArrayToString(value), typ.DescString())) + case types.T_array_int8: + value := data.([]int8) + sqlBuff = appendString(sqlBuff, fmt.Sprintf("CAST('%s' as %s)", types.ArrayToString(value), typ.DescString())) + case types.T_array_uint8: + value := data.([]uint8) + sqlBuff = appendString(sqlBuff, fmt.Sprintf("CAST('%s' as %s)", types.ArrayToString(value), typ.DescString())) case types.T_date: value := data.(types.Date) sqlBuff = appendByte(sqlBuff, '\'') diff --git a/pkg/iscp/util_test.go b/pkg/iscp/util_test.go index e3064c3ab4f30..e9fece9195537 100644 --- a/pkg/iscp/util_test.go +++ b/pkg/iscp/util_test.go @@ -31,7 +31,7 @@ import ( func mockUtilVector(t *testing.T, proc *process.Process) (*batch.Batch, []string) { i := 0 - nvec := 15 + nvec := 19 bat := batch.NewWithSize(nvec) res := make([]string, nvec) @@ -75,6 +75,40 @@ func mockUtilVector(t *testing.T, proc *process.Process) (*batch.Batch, []string i += 1 } + { + // []float16 (narrow base column) + bat.Vecs[i] = vector.NewVec(types.New(types.T_array_float16, 3, 0)) + vf16 := types.Float32ToFloat16Slice([]float32{0, 1, 2}) + vector.AppendArray[types.Float16](bat.Vecs[i], vf16, false, proc.Mp()) + res[i] = "CAST('[0, 1, 2]' as VECF16(3))" + i += 1 + } + + { + // []bf16 (narrow base column) + bat.Vecs[i] = vector.NewVec(types.New(types.T_array_bf16, 3, 0)) + vbf16 := types.Float32ToBF16Slice([]float32{0, 1, 2}) + vector.AppendArray[types.BF16](bat.Vecs[i], vbf16, false, proc.Mp()) + res[i] = "CAST('[0, 1, 2]' as VECBF16(3))" + i += 1 + } + + { + // []int8 (narrow base column) + bat.Vecs[i] = vector.NewVec(types.New(types.T_array_int8, 3, 0)) + vector.AppendArray[int8](bat.Vecs[i], []int8{0, 1, 2}, false, proc.Mp()) + res[i] = "CAST('[0, 1, 2]' as VECINT8(3))" + i += 1 + } + + { + // []uint8 (narrow base column) + bat.Vecs[i] = vector.NewVec(types.New(types.T_array_uint8, 3, 0)) + vector.AppendArray[uint8](bat.Vecs[i], []uint8{0, 1, 2}, false, proc.Mp()) + res[i] = "CAST('[0, 1, 2]' as VECUINT8(3))" + i += 1 + } + { // date bat.Vecs[i] = vector.NewVec(types.New(types.T_date, 4, 0)) @@ -191,7 +225,7 @@ func TestRowFromVector(t *testing.T) { sql := make([]byte, 0, 1024) for i, vec := range bat.Vecs { - err := extractRowFromVector(ctx, vec, 0, res, 0) + err := extractRowFromVector(ctx, vec, 0, res, 0, ReprSQLString) require.Nil(t, err) sql, err := convertColIntoSql(ctx, res[0], vec.GetType(), sql) @@ -202,3 +236,66 @@ func TestRowFromVector(t *testing.T) { } } + +// TestExtractRowNativeRepr covers the ReprNative path added for binary CDC consumers +// (the WAND retrieval index): the temporal / decimal / uuid types must come out as +// their exact native Go value under ReprNative, while ReprSQLString keeps yielding the +// SQL-display string; a non-differing control type (int64) must be identical in both. +func TestExtractRowNativeRepr(t *testing.T) { + m := mpool.MustNewZero() + proc := testutil.NewProcessWithMPool(t, "", m) + ctx := context.Background() + + mk := func(typ types.Type, appendFn func(v *vector.Vector)) *vector.Vector { + v := vector.NewVec(typ) + appendFn(v) + return v + } + + cases := []struct { + name string + vec *vector.Vector + native any // expected value under ReprNative + differs bool // native form differs from the SQL-display string (so ReprSQLString is a string) + }{ + {"datetime", mk(types.New(types.T_datetime, 8, 0), func(v *vector.Vector) { + vector.AppendFixed(v, types.Datetime(0x0123456789ABCDEF), false, proc.Mp()) + }), types.Datetime(0x0123456789ABCDEF), true}, + {"time", mk(types.New(types.T_time, 8, 0), func(v *vector.Vector) { + vector.AppendFixed(v, types.Time(0x0011223344556677), false, proc.Mp()) + }), types.Time(0x0011223344556677), true}, + {"timestamp", mk(types.New(types.T_timestamp, 8, 0), func(v *vector.Vector) { + vector.AppendFixed(v, types.Timestamp(0x7FFFFFFFFFFFFFFF), false, proc.Mp()) + }), types.Timestamp(0x7FFFFFFFFFFFFFFF), true}, + {"decimal64", mk(types.New(types.T_decimal64, 8, 0), func(v *vector.Vector) { + vector.AppendFixed(v, types.Decimal64(1000), false, proc.Mp()) + }), types.Decimal64(1000), true}, + {"decimal128", mk(types.New(types.T_decimal128, 16, 0), func(v *vector.Vector) { + vector.AppendFixed(v, types.Decimal128{B0_63: 1000, B64_127: 7}, false, proc.Mp()) + }), types.Decimal128{B0_63: 1000, B64_127: 7}, true}, + {"uuid", mk(types.New(types.T_uuid, 16, 0), func(v *vector.Vector) { + vector.AppendFixed(v, types.Uuid([16]byte{0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0, 8}), false, proc.Mp()) + }), types.Uuid([16]byte{0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0, 8}), true}, + // control: int64 is native in both modes, so the two reprs must agree. + {"int64", mk(types.New(types.T_int64, 8, 0), func(v *vector.Vector) { + vector.AppendFixed[int64](v, int64(100), false, proc.Mp()) + }), int64(100), false}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + nat := make([]any, 1) + require.NoError(t, extractRowFromVector(ctx, c.vec, 0, nat, 0, ReprNative)) + require.Equal(t, c.native, nat[0], "ReprNative must yield the exact native value") + + str := make([]any, 1) + require.NoError(t, extractRowFromVector(ctx, c.vec, 0, str, 0, ReprSQLString)) + if c.differs { + _, ok := str[0].(string) + require.Truef(t, ok, "ReprSQLString must yield a string for %s, got %T", c.name, str[0]) + } else { + require.Equal(t, nat[0], str[0], "a non-differing type must be identical across reprs") + } + }) + } +} diff --git a/pkg/iscp/wand_consumer.go b/pkg/iscp/wand_consumer.go new file mode 100644 index 0000000000000..6f0880cf4a8f1 --- /dev/null +++ b/pkg/iscp/wand_consumer.go @@ -0,0 +1,175 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iscp + +import ( + "context" + "time" + + "github.com/matrixorigin/matrixone/pkg/bm25/wand" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/monlp/tokenizer" + veccache "github.com/matrixorigin/matrixone/pkg/vectorindex/cache" + "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" +) + +// RunWand is the ISCP consumer loop for the WAND "retrieval" index. Like +// RunHnsw it is model-building (not the generic SQL RunIndex), but leaner: it +// never loads the full index. It STREAMS each flush's CDC blob straight into a +// TailBuilder — tokenizing insert rows into capacity-capped segments and spilling +// each sealed segment to a temp file the moment it fills — so peak memory is one +// open segment, not the whole stream. On channel close it appends the spilled +// segments (+ one delete batch) as tag=1 CdcTail frames at the next chunk_id, in +// one txn, advancing the watermark. +// +// Why streaming: the old path buffered every event (acc.Events) before building, +// so a large initial sync (e.g. 88M rows) OOM'd holding all (pk, text) in RAM. +// This mirrors hnsw's HnswSync (Update rolls/unloads full models to files; +// Save persists), bounding memory to ~one max_index_capacity segment. +// +// NOTE: this consumer is CDC/txn-coupled and is NOT exercised by the package +// unit tests; it needs a live mo_ctl + CDC pipeline to validate end-to-end. The +// WAND-specific build/frame logic it calls (TailBuilder, TailFileInsertSqls, +// NextTailChunkIdSql) is unit-tested in pkg/bm25/wand. +func RunWand(c *IndexConsumer, ctx context.Context, errch chan error, r DataRetriever) { + w, ok := c.sqlWriter.(*WandSqlWriter) + if !ok { + errch <- moerr.NewInternalError(ctx, "wand iscp Run: unexpected writer type") + return + } + + tok, err := tokenizer.SharedJiebaTokenizer(false) + if err != nil { + errch <- err + return + } + // Same jieba path the search side uses (parsePatternInNLModeJieba), so build + // and query tokens match. + tokenize := func(text string) []string { + var words []string + for t, e := range tok.Tokenize([]byte(text)) { + if e != nil { + break + } + slen := t.TokenBytes[0] + words = append(words, string(t.TokenBytes[1:slen+1])) + } + return words + } + + // w.capacity was resolved at writer construction (flat algo-param > captured + // fulltext_max_index_capacity session var > default), so no live resolve here. + tb, err := wand.NewTailBuilder(w.pkType, w.capacity, tokenize) + if err != nil { + errch <- err + return + } + defer tb.Cleanup() + + datatype := r.GetDataType() + nevents := 0 + + for { + select { + case <-ctx.Done(): + return + case e := <-errch: + errch <- e + return + case blob, ok := <-c.sqlBufSendCh: + if !ok { + // channel closed: seal the final segment + delete frame (all spilled to + // files, delete first) and persist them as tag=1 frames in one txn. + segs, ferr := tb.Finish() + if ferr != nil { + errch <- ferr + return + } + changed := false + err = sqlexec.RunTxnWithSqlContext(ctx, c.cnEngine, c.cnTxnClient, c.cnUUID, r.GetAccountID(), time.Hour, nil, nil, + func(sqlproc *sqlexec.SqlProcess, cbdata any) (err error) { + startChunk, err := wandNextTailChunkId(sqlproc, w.cfg) + if err != nil { + return err + } + chunkID := startChunk + for _, seg := range segs { + // The frame is ALREADY on disk (TailBuilder spilled it), so + // INSERT it via load_file straight from the file — no read-back + // to memory, no hex/unhex — split across MaxChunkSize rows. + for _, s := range wand.TailFileInsertSqls(w.cfg, chunkID, seg.Path, seg.FrameLen) { + res, e := sqlexec.RunSql(sqlproc, s) + if e != nil { + return e + } + res.Close() + } + chunkID += wand.FrameChunkCount(seg.FrameLen) + } + changed = len(segs) > 0 + logutil.Infof("[wand-sink] db=%s index=%s type=%d events=%d frames=%d chunk_id=%d..%d", + w.cfg.DbName, w.cfg.IndexTable, datatype, nevents, len(segs), startChunk, chunkID) + // advance the CDC watermark only on the tail stream. + if datatype == ISCPDataType_Tail { + sqlctx := sqlproc.SqlCtx + return r.UpdateWatermark(sqlproc.GetContext(), sqlctx.GetService(), sqlctx.Txn()) + } + return nil + }) + if err != nil { + errch <- err + return + } + // Evict the cached search index so the next query reloads tag=0 + + // the freshly-appended tag=1 frames, instead of serving the warm + // (stale) cache until its idle TTL. Local to this CN's cache. + if changed { + veccache.Cache.Remove(w.cfg.IndexTable) + logutil.Infof("[wand-sink] evicted search cache for index=%s", w.cfg.IndexTable) + } + return + } + + cdc, derr := wand.DecodeWandCdc(blob) + if derr != nil { + errch <- derr + return + } + nevents += len(cdc.Events) + if aerr := tb.AddBatch(cdc); aerr != nil { + errch <- aerr + return + } + } + } +} + +// wandNextTailChunkId runs the COALESCE(MAX(chunk_id)+1,0) query for the tag=1 +// CdcTail and returns the next append position. +func wandNextTailChunkId(sqlproc *sqlexec.SqlProcess, cfg wand.TableConfig) (int64, error) { + res, err := sqlexec.RunSql(sqlproc, wand.NextTailChunkIdSql(cfg)) + if err != nil { + return 0, err + } + defer res.Close() + for _, bat := range res.Batches { + if bat != nil && bat.RowCount() > 0 { + return vector.GetFixedAtNoTypeCheck[int64](bat.Vecs[0], 0), nil + } + } + return 0, nil +} diff --git a/pkg/iscp/wand_sqlwriter.go b/pkg/iscp/wand_sqlwriter.go new file mode 100644 index 0000000000000..a3897a69adcdb --- /dev/null +++ b/pkg/iscp/wand_sqlwriter.go @@ -0,0 +1,170 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iscp + +import ( + "context" + + "github.com/matrixorigin/matrixone/pkg/bm25/wand" + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + indexplugin "github.com/matrixorigin/matrixone/pkg/indexplugin" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" +) + +// defaultWandCapacity caps docs-per-segment for the CDC delta build. A segment +// frame larger than MaxChunkSize is split across chunk rows and reassembled at +// load (Bug 1), so this is NOT bounded by the storage-row size — it's the +// segment-count/size tradeoff knob (fewer, larger segments → faster search but +// more RAM per open segment during the streaming build). Matches HNSW's 1M +// default. Overridable via the max_index_capacity algo-param (though nothing +// user-facing sets it on a retrieval index yet — see fulltext_wand.md Phase C). +const defaultWandCapacity int64 = 1000000 + +// WandSqlWriter is the ISCP sink adapter for the WAND "retrieval" fulltext index. +// Unlike the postings FulltextSqlWriter (which emits SQL), it is model-building +// like HnswSqlWriter: it accumulates the CDC rows of one flush into a binary +// WandCdc blob (ToSql), which RunWand decodes, tokenizes, and turns into tag=1 +// CdcTail frames. The blob is binary (typed pk) because a retrieval pk is `any` +// (int64 OR varchar) — a JSON blob would corrupt a non-integer pk. +type WandSqlWriter struct { + cfg wand.TableConfig // DbName + ft_index (storage) + ft_meta (metadata) + pkType int32 // types.T of the source primary key + pkPos int32 // pk column index in the extracted row + textPos int32 // indexed text column index (idxdef.Parts[0]) + capacity int64 // max docs per delta segment (max_index_capacity) + + cdc *wand.WandCdc // accumulated events for the current flush + ndata int // approx bytes buffered, for Full() + lastOp string // last CDC op, for CheckLastOp batching +} + +var _ IndexSqlWriter = (*WandSqlWriter)(nil) + +// NewWandSqlWriter resolves the pk / text columns and the ft_index/ft_meta +// storage tables (reused with a tag column: tag=0 base, tag=1 CdcTail) from the +// retrieval index def. +func NewWandSqlWriter(algo string, jobID JobID, info *ConsumerInfo, tabledef *plan.TableDef, indexdef []*plan.IndexDef) (IndexSqlWriter, error) { + idxdef := indexdef[0] + + var storage, meta string + for _, idx := range indexdef { + switch idx.IndexAlgoTableType { + case catalog.Bm25Index_TblType_Storage: + storage = idx.IndexTableName + case catalog.Bm25Index_TblType_Metadata: + meta = idx.IndexTableName + } + } + if len(storage) == 0 || len(meta) == 0 { + return nil, moerr.NewInternalErrorNoCtx("wand sink: ft_index/ft_meta hidden tables not found on retrieval index") + } + if len(idxdef.Parts) == 0 { + return nil, moerr.NewInternalErrorNoCtx("wand sink: retrieval index has no source column") + } + + pkPos := tabledef.Name2ColIndex[tabledef.Pkey.PkeyColName] + pkTyp := tabledef.Cols[pkPos].Typ + textPos := tabledef.Name2ColIndex[idxdef.Parts[0]] + + // capacity precedence (AlgoParamInt): a flat max_index_capacity algo-param + // (explicit CREATE INDEX option) > the fulltext_max_index_capacity value + // CAPTURED into algo_params.session_vars at CREATE (BuildSessionVars) > + // defaultWandCapacity. The sinker runs in an internal ISCP proc with NO live + // resolver, so we resolve the captured session_vars blob directly here rather + // than through GetResolveVariableFunc (which is nil) — mirroring initSQLResolver. + flat := "" + if m, e := catalog.IndexParamsStringToMap(idxdef.IndexAlgoParams); e == nil { + flat = m[catalog.IndexAlgoParamMaxIndexCapacity] + } + var resolve indexplugin.ResolveVarFunc + if sv, e := catalog.IndexParamsSessionVars(idxdef.IndexAlgoParams); e == nil && len(sv) > 0 { + if md, e2 := sqlexec.NewMetadataFromJson(string(sv)); e2 == nil && md != nil { + resolve = md.ResolveVariableFunc + } + } + capacity, err := indexplugin.AlgoParamInt(flat, resolve, "fulltext_max_index_capacity", defaultWandCapacity) + if err != nil { + return nil, err + } + + return &WandSqlWriter{ + cfg: wand.TableConfig{DbName: info.DBName, IndexTable: storage, MetadataTable: meta}, + pkType: int32(pkTyp.Id), + pkPos: pkPos, + textPos: textPos, + capacity: capacity, + cdc: wand.NewWandCdc(int32(pkTyp.Id)), + }, nil +} + +func (w *WandSqlWriter) CheckLastOp(op string) bool { return len(w.lastOp) == 0 || w.lastOp == op } +func (w *WandSqlWriter) Empty() bool { return w.cdc.Len() == 0 } +func (w *WandSqlWriter) Full() bool { return w.ndata >= MAX_CDC_DATA_SIZE } +func (w *WandSqlWriter) ToSql() ([]byte, error) { return w.cdc.Encode() } + +func (w *WandSqlWriter) Reset() { + w.cdc = wand.NewWandCdc(w.pkType) + w.ndata = 0 + w.lastOp = "" +} + +func (w *WandSqlWriter) Insert(ctx context.Context, row []any) error { + w.lastOp = vectorindex.CDC_INSERT + text := wandRowText(row[w.textPos]) + w.cdc.Insert(wandCopyPk(row[w.pkPos]), text) + w.ndata += len(text) + 16 + return nil +} + +func (w *WandSqlWriter) Upsert(ctx context.Context, row []any) error { + w.lastOp = vectorindex.CDC_UPSERT + text := wandRowText(row[w.textPos]) + w.cdc.Upsert(wandCopyPk(row[w.pkPos]), text) + w.ndata += len(text) + 16 + return nil +} + +func (w *WandSqlWriter) Delete(ctx context.Context, row []any) error { + // a delete row carries only the pk in position 0 (mirrors HnswSqlWriter). + w.lastOp = vectorindex.CDC_DELETE + w.cdc.Delete(wandCopyPk(row[0])) + w.ndata += 16 + return nil +} + +// wandRowText reads the source text column as a string (varchar → []byte/string; +// a NULL text yields ""; such a doc simply contributes no terms). +func wandRowText(v any) string { + switch t := v.(type) { + case []byte: + return string(t) + case string: + return t + default: + return "" + } +} + +// wandCopyPk defensively copies a byte-slice pk out of the reused row buffer; +// value pks (int64, etc.) are copied by assignment. +func wandCopyPk(v any) any { + if b, ok := v.([]byte); ok { + return append([]byte(nil), b...) + } + return v +} diff --git a/pkg/iscp/wand_sqlwriter_test.go b/pkg/iscp/wand_sqlwriter_test.go new file mode 100644 index 0000000000000..ee32880eea4ad --- /dev/null +++ b/pkg/iscp/wand_sqlwriter_test.go @@ -0,0 +1,87 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iscp + +import ( + "context" + "testing" + + "github.com/matrixorigin/matrixone/pkg/bm25/wand" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/vectorindex" +) + +// TestWandSqlWriterAccumulate checks the writer maps extracted CDC rows to the +// right WandCdc events (pk/text column resolution, op tracking, defensive pk +// copy) and that ToSql round-trips them. Fields are set directly to bypass the +// tabledef/indexdef-dependent constructor. +func TestWandSqlWriterAccumulate(t *testing.T) { + pkType := int32(types.T_int64) + w := &WandSqlWriter{ + pkType: pkType, + pkPos: 0, + textPos: 1, + capacity: 100, + cdc: wand.NewWandCdc(pkType), + } + ctx := context.Background() + + if !w.Empty() { + t.Fatal("fresh writer should be empty") + } + if !w.CheckLastOp(vectorindex.CDC_INSERT) { + t.Fatal("empty writer should accept any op") + } + + // pk in col 0, text (varchar → []byte) in col 1; delete carries pk in col 0. + if err := w.Insert(ctx, []any{int64(1), []byte("营养 早餐")}); err != nil { + t.Fatal(err) + } + if err := w.Upsert(ctx, []any{int64(2), "视频"}); err != nil { + t.Fatal(err) + } + if err := w.Delete(ctx, []any{int64(3)}); err != nil { + t.Fatal(err) + } + + if w.Empty() { + t.Fatal("writer should have buffered events") + } + if w.lastOp != vectorindex.CDC_DELETE || w.CheckLastOp(vectorindex.CDC_INSERT) { + t.Fatalf("lastOp tracking wrong: %q", w.lastOp) + } + + blob, err := w.ToSql() + if err != nil { + t.Fatal(err) + } + cdc, err := wand.DecodeWandCdc(blob) + if err != nil { + t.Fatal(err) + } + if len(cdc.Events) != 3 { + t.Fatalf("want 3 events, got %d", len(cdc.Events)) + } + if cdc.Events[0].Pk.(int64) != 1 || cdc.Events[0].Text != "营养 早餐" || + cdc.Events[1].Pk.(int64) != 2 || cdc.Events[1].Text != "视频" || + cdc.Events[2].Pk.(int64) != 3 { + t.Fatalf("events wrong: %+v", cdc.Events) + } + + w.Reset() + if !w.Empty() || w.lastOp != "" || w.ndata != 0 { + t.Fatal("Reset should clear the writer") + } +} diff --git a/pkg/monlp/tokenizer/word_id.go b/pkg/monlp/tokenizer/word_id.go new file mode 100644 index 0000000000000..f6838e28b61df --- /dev/null +++ b/pkg/monlp/tokenizer/word_id.go @@ -0,0 +1,104 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tokenizer + +import ( + "bufio" + "os" + "strings" + "sync" +) + +// Word-id support for the WAND retrieval index. +// +// jieba.dict.utf8 is a fixed list of ~349k dictionary words ("word freq POS" +// per line). Each word's line index is a stable, process-wide word id, shared +// by every index — so dictionary words need no per-index term storage. Tokens +// NOT in the dictionary (English words, numbers, user-dict / HMM-discovered +// tokens) return ok=false; the caller assigns them per-index overflow ids in a +// range above DictWordIDLimit. + +// DictWordIDLimit is the exclusive upper bound of global (dictionary) word ids. +// Out-of-dictionary overflow ids must be assigned at or above this value so the +// two id spaces never collide, regardless of the dictionary's exact size. +const DictWordIDLimit = int32(1) << 24 // 16,777,216 (dict has ~349k entries) + +var ( + wordIDOnce sync.Once + wordIDMap map[string]int32 + wordIDErr error +) + +func loadWordIDMap() { + path := jiebaDictPaths()[0] // jieba.dict.utf8 + f, err := os.Open(path) + if err != nil { + wordIDErr = err + return + } + defer f.Close() + + m := make(map[string]int32, 400000) + var id int32 + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 0, 64*1024), 1024*1024) + for sc.Scan() { + line := sc.Text() + if line == "" { + id++ + continue + } + // "word freq POS" — the word is the first space-separated field. + word := line + if sp := strings.IndexByte(line, ' '); sp >= 0 { + word = line[:sp] + } + // First occurrence wins (the line index is the stable id); the dict has + // a few case variants but no exact-duplicate words. + if _, exists := m[word]; !exists { + m[word] = id + } + id++ + } + if err := sc.Err(); err != nil { + wordIDErr = err + return + } + if id >= DictWordIDLimit { + // Defensive: the dict grew past the reserved global id space. + wordIDErr = errDictTooLarge + return + } + wordIDMap = m +} + +var errDictTooLarge = &dictError{"jieba dictionary exceeds DictWordIDLimit"} + +type dictError struct{ msg string } + +func (e *dictError) Error() string { return e.msg } + +// WordID returns the global word id of a jieba-dictionary word (its line index +// in jieba.dict.utf8). ok is false for out-of-dictionary tokens, which the +// caller maps to per-index overflow ids (>= DictWordIDLimit). The dictionary is +// loaded once on first call and shared process-wide. +func WordID(word string) (id int32, ok bool, err error) { + wordIDOnce.Do(loadWordIDMap) + if wordIDErr != nil { + return 0, false, wordIDErr + } + id, ok = wordIDMap[word] + return id, ok, nil +} diff --git a/pkg/monlp/tokenizer/word_id_test.go b/pkg/monlp/tokenizer/word_id_test.go new file mode 100644 index 0000000000000..e9e2966d1f83f --- /dev/null +++ b/pkg/monlp/tokenizer/word_id_test.go @@ -0,0 +1,42 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tokenizer + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestWordID(t *testing.T) { + // A common dictionary word resolves to a stable non-negative id. + id, ok, err := WordID("中国") + require.NoError(t, err) + require.True(t, ok, "expected a common jieba-dict word to resolve") + require.GreaterOrEqual(t, id, int32(0)) + require.Less(t, id, int32(DictWordIDLimit)) + + // The same word resolves to the same id (map is stable / loaded once). + id2, ok2, err := WordID("中国") + require.NoError(t, err) + require.True(t, ok2) + require.Equal(t, id, id2) + + // An out-of-dictionary token returns ok=false with no error (the caller maps + // these to per-index overflow ids). + _, ok, err = WordID("zzz_definitely_not_a_dict_word_qwerty") + require.NoError(t, err) + require.False(t, ok) +} diff --git a/pkg/partition/partition.go b/pkg/partition/partition.go index 68e5a8d285e8f..6691d79730912 100644 --- a/pkg/partition/partition.go +++ b/pkg/partition/partition.go @@ -176,7 +176,9 @@ func Partition(sels []int64, diffs []bool, partitions []int64, vec *vector.Vecto return genericPartition[types.Blockid](sels, diffs, partitions, vec) case types.T_char, types.T_varchar, types.T_json, types.T_text, types.T_binary, types.T_varbinary, types.T_blob, - types.T_array_float32, types.T_array_float64, types.T_datalink: + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, + types.T_datalink: return bytesPartition(sels, diffs, partitions, vec) //Used by ORDER_BY SQL clause. //Byte partition logic doesn't use byte.Compare or Str. diff --git a/pkg/pb/plan/plan.pb.go b/pkg/pb/plan/plan.pb.go index 034bbc6941420..9f789b7d81a88 100644 --- a/pkg/pb/plan/plan.pb.go +++ b/pkg/pb/plan/plan.pb.go @@ -10327,6 +10327,7 @@ type AlterTableAlterReIndex struct { IndexName string `protobuf:"bytes,3,opt,name=index_name,json=indexName,proto3" json:"index_name,omitempty"` IndexAlgoParamList int64 `protobuf:"varint,4,opt,name=index_algo_param_list,json=indexAlgoParamList,proto3" json:"index_algo_param_list,omitempty"` ForceSync bool `protobuf:"varint,5,opt,name=force_sync,json=forceSync,proto3" json:"force_sync,omitempty"` + Merge bool `protobuf:"varint,6,opt,name=merge,proto3" json:"merge,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -10400,6 +10401,13 @@ func (m *AlterTableAlterReIndex) GetForceSync() bool { return false } +func (m *AlterTableAlterReIndex) GetMerge() bool { + if m != nil { + return m.Merge + } + return false +} + type AlterTableAlterAutoUpdate struct { DbName string `protobuf:"bytes,1,opt,name=db_name,json=dbName,proto3" json:"db_name,omitempty"` TableName string `protobuf:"bytes,2,opt,name=table_name,json=tableName,proto3" json:"table_name,omitempty"` @@ -23339,6 +23347,16 @@ func (m *AlterTableAlterReIndex) MarshalToSizedBuffer(dAtA []byte) (int, error) i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.Merge { + i-- + if m.Merge { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x30 + } if m.ForceSync { i-- if m.ForceSync { @@ -30235,6 +30253,9 @@ func (m *AlterTableAlterReIndex) ProtoSize() (n int) { if m.ForceSync { n += 2 } + if m.Merge { + n += 2 + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -53502,6 +53523,26 @@ func (m *AlterTableAlterReIndex) Unmarshal(dAtA []byte) error { } } m.ForceSync = bool(v != 0) + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Merge", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Merge = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPlan(dAtA[iNdEx:]) diff --git a/pkg/sort/sort.go b/pkg/sort/sort.go index 2def8288ccb09..020fd66723804 100644 --- a/pkg/sort/sort.go +++ b/pkg/sort/sort.go @@ -40,7 +40,8 @@ type sortType interface { ~[]types.Time | ~[]types.Enum | ~[]types.MoYear | ~[]types.TS | ~[]types.Decimal64 | ~[]types.Decimal128 | ~[]types.Decimal256 | ~[]types.Rowid | ~[]types.Blockid | ~[]types.Uuid | - ~[][]float32 | ~[][]float64 + ~[][]float32 | ~[][]float64 | + ~[][]types.BF16 | ~[][]types.Float16 | ~[][]int8 | ~[][]uint8 } type xorshift uint64 @@ -287,6 +288,34 @@ func Sort(desc, nullsLast, hasNull bool, os []int64, vec *vector.Vector) { } else { genericSort(col, os, arrayGreater[float64]) } + case types.T_array_bf16: + col := vector.MustArrayCol[types.BF16](vec) + if !desc { + genericSort(col, os, arrayElementLess[types.BF16]) + } else { + genericSort(col, os, arrayElementGreater[types.BF16]) + } + case types.T_array_float16: + col := vector.MustArrayCol[types.Float16](vec) + if !desc { + genericSort(col, os, arrayElementLess[types.Float16]) + } else { + genericSort(col, os, arrayElementGreater[types.Float16]) + } + case types.T_array_int8: + col := vector.MustArrayCol[int8](vec) + if !desc { + genericSort(col, os, arrayElementLess[int8]) + } else { + genericSort(col, os, arrayElementGreater[int8]) + } + case types.T_array_uint8: + col := vector.MustArrayCol[uint8](vec) + if !desc { + genericSort(col, os, arrayElementLess[uint8]) + } else { + genericSort(col, os, arrayElementGreater[uint8]) + } case types.T_TS: col := vector.MustFixedColNoTypeCheck[types.TS](vec) if !desc { @@ -394,6 +423,16 @@ func arrayGreater[T types.RealNumbers](data [][]T, i, j int64) bool { return types.ArrayCompare[T](data[i], data[j]) > 0 } +// Narrow vector element types (bf16/f16/int8) order through the float32 bridge +// so bf16/f16 sign bits do not corrupt the ordering. +func arrayElementLess[T types.ArrayElement](data [][]T, i, j int64) bool { + return types.ArrayElementCompare[T](data[i], data[j]) < 0 +} + +func arrayElementGreater[T types.ArrayElement](data [][]T, i, j int64) bool { + return types.ArrayElementCompare[T](data[i], data[j]) > 0 +} + func genericLess[T types.OrderedT](data []T, i, j int64) bool { return data[i] < data[j] } diff --git a/pkg/sql/colexec/evalExpression.go b/pkg/sql/colexec/evalExpression.go index bbaa366e253e6..c77336392f24f 100644 --- a/pkg/sql/colexec/evalExpression.go +++ b/pkg/sql/colexec/evalExpression.go @@ -851,10 +851,19 @@ func generateConstExpressionExecutor(proc *process.Process, typ types.Type, con case *plan.Literal_EnumVal: vec, err = vector.NewConstFixed(constEnumType, types.Enum(val.EnumVal), 1, proc.Mp()) case *plan.Literal_VecVal: - if typ.Oid == types.T_array_float32 { + switch typ.Oid { + case types.T_array_float32: vec, err = vector.NewConstArray(typ, types.BytesToArray[float32]([]byte(val.VecVal)), 1, proc.Mp()) - } else if typ.Oid == types.T_array_float64 { + case types.T_array_float64: vec, err = vector.NewConstArray(typ, types.BytesToArray[float64]([]byte(val.VecVal)), 1, proc.Mp()) + case types.T_array_bf16: + vec, err = vector.NewConstArray(typ, types.BytesToArray[types.BF16]([]byte(val.VecVal)), 1, proc.Mp()) + case types.T_array_float16: + vec, err = vector.NewConstArray(typ, types.BytesToArray[types.Float16]([]byte(val.VecVal)), 1, proc.Mp()) + case types.T_array_int8: + vec, err = vector.NewConstArray(typ, types.BytesToArray[int8]([]byte(val.VecVal)), 1, proc.Mp()) + case types.T_array_uint8: + vec, err = vector.NewConstArray(typ, types.BytesToArray[uint8]([]byte(val.VecVal)), 1, proc.Mp()) } default: return nil, moerr.NewNYI(proc.Ctx, fmt.Sprintf("const expression %v", con.GetValue())) diff --git a/pkg/sql/colexec/external/external.go b/pkg/sql/colexec/external/external.go index c1bb608088de3..68918f778afdc 100644 --- a/pkg/sql/colexec/external/external.go +++ b/pkg/sql/colexec/external/external.go @@ -738,6 +738,26 @@ func isLegalLine(param *tree.ExternParam, cols []*plan.ColDef, fields []csvparse if err != nil { return false } + case types.T_array_bf16: + _, err := types.StringToArrayToBytes[types.BF16](field.Val) + if err != nil { + return false + } + case types.T_array_float16: + _, err := types.StringToArrayToBytes[types.Float16](field.Val) + if err != nil { + return false + } + case types.T_array_int8: + _, err := types.StringToArrayToBytes[int8](field.Val) + if err != nil { + return false + } + case types.T_array_uint8: + _, err := types.StringToArrayToBytes[uint8](field.Val) + if err != nil { + return false + } case types.T_json: if param.Format == tree.CSV { field.Val = fmt.Sprintf("%v", strings.Trim(field.Val, "\"")) @@ -1404,6 +1424,50 @@ func getColData(bat *batch.Batch, line []csvparser.Field, rowIdx int, param *Ext if err = vector.AppendBytes(vec, types.ArrayToBytes[float64](arr), false, mp); err != nil { return err } + case types.T_array_bf16: + arr, err := types.StringToArray[types.BF16](field.Val) + if err != nil { + return err + } + if int(vec.GetType().Width) != types.MaxArrayDimension && int(vec.GetType().Width) != len(arr) { + return moerr.NewArrayDefMismatchNoCtx(int(vec.GetType().Width), len(arr)) + } + if err = vector.AppendBytes(vec, types.ArrayToBytes[types.BF16](arr), false, mp); err != nil { + return err + } + case types.T_array_float16: + arr, err := types.StringToArray[types.Float16](field.Val) + if err != nil { + return err + } + if int(vec.GetType().Width) != types.MaxArrayDimension && int(vec.GetType().Width) != len(arr) { + return moerr.NewArrayDefMismatchNoCtx(int(vec.GetType().Width), len(arr)) + } + if err = vector.AppendBytes(vec, types.ArrayToBytes[types.Float16](arr), false, mp); err != nil { + return err + } + case types.T_array_int8: + arr, err := types.StringToArray[int8](field.Val) + if err != nil { + return err + } + if int(vec.GetType().Width) != types.MaxArrayDimension && int(vec.GetType().Width) != len(arr) { + return moerr.NewArrayDefMismatchNoCtx(int(vec.GetType().Width), len(arr)) + } + if err = vector.AppendBytes(vec, types.ArrayToBytes[int8](arr), false, mp); err != nil { + return err + } + case types.T_array_uint8: + arr, err := types.StringToArray[uint8](field.Val) + if err != nil { + return err + } + if int(vec.GetType().Width) != types.MaxArrayDimension && int(vec.GetType().Width) != len(arr) { + return moerr.NewArrayDefMismatchNoCtx(int(vec.GetType().Width), len(arr)) + } + if err = vector.AppendBytes(vec, types.ArrayToBytes[uint8](arr), false, mp); err != nil { + return err + } case types.T_json: var jsonBytes []byte if param.Extern.Format != tree.CSV { diff --git a/pkg/sql/colexec/external/hive_partition_fill.go b/pkg/sql/colexec/external/hive_partition_fill.go index 41bbe4757a072..f4b779e379ed9 100644 --- a/pkg/sql/colexec/external/hive_partition_fill.go +++ b/pkg/sql/colexec/external/hive_partition_fill.go @@ -362,7 +362,9 @@ func fillConstantVector( } return vector.SetConstFixed(vec, v, rowCount, mp) - case types.T_array_float32, types.T_array_float64: + case types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, + types.T_array_int8, types.T_array_uint8: return moerr.NewNotSupportedf(proc.Ctx, "unsupported partition column type VECTOR for col=%s, path=%s", col.Name, filePath) diff --git a/pkg/sql/colexec/external/hive_partition_test.go b/pkg/sql/colexec/external/hive_partition_test.go index 35ee1ce1913f6..1100d3a16ed53 100644 --- a/pkg/sql/colexec/external/hive_partition_test.go +++ b/pkg/sql/colexec/external/hive_partition_test.go @@ -1906,12 +1906,21 @@ func TestFillConstantVector_Bool(t *testing.T) { func TestFillConstantVector_UnsupportedVector(t *testing.T) { proc := testutil.NewProc(t) - vec := vector.NewVec(types.T_array_float32.ToType()) - col := &plan.ColDef{Name: "emb", Typ: plan.Type{Id: int32(types.T_array_float32)}} - - err := fillConstantVector(vec, "[1,2,3]", col, 1, proc, "/test") - require.Error(t, err) - assert.Contains(t, err.Error(), "unsupported") + // Vectors (any element width) cannot be a hive partition column. + for _, id := range []types.T{ + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, + types.T_array_int8, types.T_array_uint8, + } { + t.Run(id.String(), func(t *testing.T) { + vec := vector.NewVec(id.ToType()) + col := &plan.ColDef{Name: "emb", Typ: plan.Type{Id: int32(id)}} + + err := fillConstantVector(vec, "[1,2,3]", col, 1, proc, "/test") + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported partition column type VECTOR") + }) + } } func TestFillPartitionColumns_DefaultPartNull(t *testing.T) { diff --git a/pkg/sql/colexec/external/parquet.go b/pkg/sql/colexec/external/parquet.go index b8c6a55ad0124..700e8b1dbfb34 100644 --- a/pkg/sql/colexec/external/parquet.go +++ b/pkg/sql/colexec/external/parquet.go @@ -314,7 +314,9 @@ func (h *ParquetHandler) prepare(param *ExternalParam) error { if !col.Leaf() { targetType := types.T(def.Typ.Id) switch targetType { - case types.T_array_float32, types.T_array_float64: + case types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, + types.T_array_int8, types.T_array_uint8: physicalCol, fn = h.getNestedListMapper(col, def.Typ) default: if !isNestedTargetTypeSupported(targetType) { @@ -451,6 +453,53 @@ func (*ParquetHandler) getNestedListMapper(sc *parquet.Column, dt plan.Type) (*p default: return nil, nil } + case types.T_array_bf16: + // bf16/f16 vectors are stored in parquet as FLOAT leaves and narrowed on load. + if leaf.Type().Kind() != parquet.Float { + return nil, nil + } + mp.mapper = func(mp *columnMapper, page parquet.Page, proc *process.Process, vec *vector.Vector) error { + return processParquetListToArray(proc.Ctx, mp, page, proc, vec, width, func(v parquet.Value) (types.BF16, error) { + return types.BF16FromFloat32(v.Float()), nil + }) + } + case types.T_array_float16: + if leaf.Type().Kind() != parquet.Float { + return nil, nil + } + mp.mapper = func(mp *columnMapper, page parquet.Page, proc *process.Process, vec *vector.Vector) error { + return processParquetListToArray(proc.Ctx, mp, page, proc, vec, width, func(v parquet.Value) (types.Float16, error) { + return types.Float16FromFloat32(v.Float()), nil + }) + } + case types.T_array_int8: + // int8/uint8 vectors are stored in parquet as INT32 leaves; load is strict + // (out-of-range values are rejected, mirroring the int8 string parse). + if leaf.Type().Kind() != parquet.Int32 { + return nil, nil + } + mp.mapper = func(mp *columnMapper, page parquet.Page, proc *process.Process, vec *vector.Vector) error { + return processParquetListToArray(proc.Ctx, mp, page, proc, vec, width, func(v parquet.Value) (int8, error) { + x := v.Int32() + if x < math.MinInt8 || x > math.MaxInt8 { + return 0, moerr.NewOutOfRangeNoCtxf("vecint8", "value %d out of range [-128,127]", x) + } + return int8(x), nil + }) + } + case types.T_array_uint8: + if leaf.Type().Kind() != parquet.Int32 { + return nil, nil + } + mp.mapper = func(mp *columnMapper, page parquet.Page, proc *process.Process, vec *vector.Vector) error { + return processParquetListToArray(proc.Ctx, mp, page, proc, vec, width, func(v parquet.Value) (uint8, error) { + x := v.Int32() + if x < 0 || x > math.MaxUint8 { + return 0, moerr.NewOutOfRangeNoCtxf("vecuint8", "value %d out of range [0,255]", x) + } + return uint8(x), nil + }) + } default: return nil, nil } @@ -1757,6 +1806,50 @@ func (*ParquetHandler) getMapper(sc *parquet.Column, dt plan.Type) *columnMapper mp.mapper = func(mp *columnMapper, page parquet.Page, proc *process.Process, vec *vector.Vector) error { return processStringToArray[float64](proc.Ctx, mp, page, proc, vec, width) } + case types.T_array_bf16: + if !isPlainStringLikeType(st) { + break + } + width := int(dt.Width) + if width <= 0 { + width = types.MaxArrayDimension + } + mp.mapper = func(mp *columnMapper, page parquet.Page, proc *process.Process, vec *vector.Vector) error { + return processStringToArray[types.BF16](proc.Ctx, mp, page, proc, vec, width) + } + case types.T_array_float16: + if !isPlainStringLikeType(st) { + break + } + width := int(dt.Width) + if width <= 0 { + width = types.MaxArrayDimension + } + mp.mapper = func(mp *columnMapper, page parquet.Page, proc *process.Process, vec *vector.Vector) error { + return processStringToArray[types.Float16](proc.Ctx, mp, page, proc, vec, width) + } + case types.T_array_int8: + if !isPlainStringLikeType(st) { + break + } + width := int(dt.Width) + if width <= 0 { + width = types.MaxArrayDimension + } + mp.mapper = func(mp *columnMapper, page parquet.Page, proc *process.Process, vec *vector.Vector) error { + return processStringToArray[int8](proc.Ctx, mp, page, proc, vec, width) + } + case types.T_array_uint8: + if !isPlainStringLikeType(st) { + break + } + width := int(dt.Width) + if width <= 0 { + width = types.MaxArrayDimension + } + mp.mapper = func(mp *columnMapper, page parquet.Page, proc *process.Process, vec *vector.Vector) error { + return processStringToArray[uint8](proc.Ctx, mp, page, proc, vec, width) + } } if mp.mapper != nil { return mp @@ -2062,7 +2155,7 @@ func processStringToJson( return nil } -func processStringToArray[T types.RealNumbers]( +func processStringToArray[T types.ArrayElement]( ctx context.Context, mp *columnMapper, page parquet.Page, @@ -2137,7 +2230,7 @@ func processStringToArray[T types.RealNumbers]( return nil } -func parseStringArrayValue[T types.RealNumbers](data []byte) ([]T, error) { +func parseStringArrayValue[T types.ArrayElement](data []byte) ([]T, error) { text := strings.TrimSpace(util.UnsafeBytesToString(data)) if isEmptyArrayText(text) { return []T{}, nil @@ -2178,7 +2271,7 @@ func readParquetPageAllValues(ctx context.Context, page parquet.Page) ([]parquet return values, nil } -func processParquetListToArray[T types.RealNumbers]( +func processParquetListToArray[T types.ArrayElement]( ctx context.Context, mp *columnMapper, page parquet.Page, diff --git a/pkg/sql/colexec/external/parquet_string_to_narrow_array_test.go b/pkg/sql/colexec/external/parquet_string_to_narrow_array_test.go new file mode 100644 index 0000000000000..cc7e702229e33 --- /dev/null +++ b/pkg/sql/colexec/external/parquet_string_to_narrow_array_test.go @@ -0,0 +1,174 @@ +// Copyright 2025 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package external + +import ( + "bytes" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/testutil" + "github.com/parquet-go/parquet-go" + "github.com/stretchr/testify/require" +) + +// TestParquet_StringToNarrowArray covers the plain-string -> narrow vector +// (bf16/f16/int8/uint8) leaf mappers added to getMapper. Before the fix these +// target types fell through the switch and getMapper returned nil (NYI on +// import), unlike the equivalent vecf32/vecf64 columns. +func TestParquet_StringToNarrowArray(t *testing.T) { + proc := testutil.NewProc(t) + + tests := []struct { + name string + dt types.T + width int32 + strValues []string + }{ + {"STRING → VECINT8", types.T_array_int8, 3, []string{"[1,2,3]", "[-128,0,127]"}}, + {"STRING → VECUINT8", types.T_array_uint8, 3, []string{"[0,1,2]", "[255,128,0]"}}, + {"STRING → VECBF16", types.T_array_bf16, 3, []string{"[1,2,3]", "[-1.5,0,2.5]"}}, + {"STRING → VECFLOAT16", types.T_array_float16, 3, []string{"[1,2,3]", "[-1.5,0,2.5]"}}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // Build a plain-string leaf parquet column holding the array text. + st := parquet.String().Type() + var buf bytes.Buffer + schema := parquet.NewSchema("x", parquet.Group{"c": parquet.Leaf(st)}) + w := parquet.NewWriter(&buf, schema) + for _, s := range tc.strValues { + _, err := w.WriteRows([]parquet.Row{{parquet.ByteArrayValue([]byte(s))}}) + require.NoError(t, err) + } + require.NoError(t, w.Close()) + + f, err := parquet.OpenFile(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.NoError(t, err) + col := f.Root().Column("c") + page, err := col.Pages().ReadPage() + require.NoError(t, err) + + vec := vector.NewVec(types.New(tc.dt, tc.width, 0)) + var h ParquetHandler + mp := h.getMapper(col, plan.Type{Id: int32(tc.dt), Width: tc.width, NotNullable: true}) + require.NotNil(t, mp, "STRING → %s conversion should be supported", tc.dt) + require.NoError(t, mp.mapping(page, proc, vec)) + require.Equal(t, len(tc.strValues), vec.Length()) + + // The stored bytes must match parsing the same text directly. + for i, s := range tc.strValues { + switch tc.dt { + case types.T_array_int8: + want, err := types.StringToArray[int8](s) + require.NoError(t, err) + require.Equal(t, want, vector.GetArrayAt[int8](vec, i)) + case types.T_array_uint8: + want, err := types.StringToArray[uint8](s) + require.NoError(t, err) + require.Equal(t, want, vector.GetArrayAt[uint8](vec, i)) + case types.T_array_bf16: + want, err := types.StringToArray[types.BF16](s) + require.NoError(t, err) + require.Equal(t, want, vector.GetArrayAt[types.BF16](vec, i)) + case types.T_array_float16: + want, err := types.StringToArray[types.Float16](s) + require.NoError(t, err) + require.Equal(t, want, vector.GetArrayAt[types.Float16](vec, i)) + } + } + }) + } +} + +// TestParquet_StringToNarrowArray_DimMismatch covers the width-check branch: +// a value whose element count differs from the column dimension must error. +func TestParquet_StringToNarrowArray_DimMismatch(t *testing.T) { + proc := testutil.NewProc(t) + + st := parquet.String().Type() + var buf bytes.Buffer + schema := parquet.NewSchema("x", parquet.Group{"c": parquet.Leaf(st)}) + w := parquet.NewWriter(&buf, schema) + _, err := w.WriteRows([]parquet.Row{{parquet.ByteArrayValue([]byte("[1,2,3,4]"))}}) + require.NoError(t, err) + require.NoError(t, w.Close()) + + f, err := parquet.OpenFile(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.NoError(t, err) + col := f.Root().Column("c") + page, err := col.Pages().ReadPage() + require.NoError(t, err) + + vec := vector.NewVec(types.New(types.T_array_int8, 3, 0)) + var h ParquetHandler + mp := h.getMapper(col, plan.Type{Id: int32(types.T_array_int8), Width: 3, NotNullable: true}) + require.NotNil(t, mp) + require.Error(t, mp.mapping(page, proc, vec), "dimension mismatch (4 != 3) should error") +} + +// TestParquet_StringToNarrowArray_Branches covers the two per-case sub-branches +// for every narrow type: width<=0 defaulting to MaxArrayDimension, and a +// non-plain-string physical source being unsupported (getMapper returns nil). +func TestParquet_StringToNarrowArray_Branches(t *testing.T) { + proc := testutil.NewProc(t) + narrowTypes := []types.T{ + types.T_array_int8, types.T_array_uint8, types.T_array_bf16, types.T_array_float16, + } + + // (a) Width == 0 in the plan type → falls back to MaxArrayDimension (no dim check). + for _, dt := range narrowTypes { + st := parquet.String().Type() + var buf bytes.Buffer + schema := parquet.NewSchema("x", parquet.Group{"c": parquet.Leaf(st)}) + w := parquet.NewWriter(&buf, schema) + _, err := w.WriteRows([]parquet.Row{{parquet.ByteArrayValue([]byte("[1,2,3]"))}}) + require.NoError(t, err) + require.NoError(t, w.Close()) + + f, err := parquet.OpenFile(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.NoError(t, err) + col := f.Root().Column("c") + page, err := col.Pages().ReadPage() + require.NoError(t, err) + + vec := vector.NewVec(types.New(dt, 3, 0)) + var h ParquetHandler + mp := h.getMapper(col, plan.Type{Id: int32(dt) /* Width: 0 */, NotNullable: true}) + require.NotNil(t, mp, "%s with width 0 should still map", dt) + require.NoError(t, mp.mapping(page, proc, vec)) + require.Equal(t, 1, vec.Length()) + } + + // (b) Non-plain-string physical source (INT64 leaf) → unsupported, nil mapper. + for _, dt := range narrowTypes { + var buf bytes.Buffer + schema := parquet.NewSchema("x", parquet.Group{"c": parquet.Leaf(parquet.Int64Type)}) + w := parquet.NewWriter(&buf, schema) + _, err := w.WriteRows([]parquet.Row{{parquet.Int64Value(1)}}) + require.NoError(t, err) + require.NoError(t, w.Close()) + + f, err := parquet.OpenFile(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.NoError(t, err) + col := f.Root().Column("c") + var h ParquetHandler + mp := h.getMapper(col, plan.Type{Id: int32(dt), Width: 3, NotNullable: true}) + require.Nil(t, mp, "non-string source → %s should be unsupported", dt) + } +} diff --git a/pkg/sql/colexec/external/parquet_test.go b/pkg/sql/colexec/external/parquet_test.go index 226ee7887fcdb..1e6ac525527e9 100644 --- a/pkg/sql/colexec/external/parquet_test.go +++ b/pkg/sql/colexec/external/parquet_test.go @@ -542,6 +542,149 @@ func TestParquetListToVectorMapping(t *testing.T) { vec := vector.NewVec(types.New(types.T_array_float32, 3, 0)) require.ErrorContains(t, mp.mapping(page, proc, vec), "parquet list NULL elements are not supported") }) + + // Narrow vector targets: bf16/f16 decode from FLOAT leaves, int8/uint8 from + // INT32 leaves. Values are chosen exactly representable so the round-trip is + // loss-free and can be asserted by exact equality. + t.Run("float list to vecbf16", func(t *testing.T) { + f, page := writeListAndGetPage(t, parquet.Leaf(parquet.FloatType), []parquet.Row{ + { + parquet.FloatValue(1).Level(0, 1, 0), + parquet.FloatValue(2).Level(1, 1, 0), + parquet.FloatValue(-0.5).Level(1, 1, 0), + }, + }) + + var h ParquetHandler + leaf, mp := h.getNestedListMapper(f.Root().Column("c"), plan.Type{Id: int32(types.T_array_bf16), Width: 3}) + require.NotNil(t, leaf) + require.NotNil(t, mp) + + vec := vector.NewVec(types.New(types.T_array_bf16, 3, 0)) + require.NoError(t, mp.mapping(page, proc, vec)) + require.Equal(t, 1, vec.Length()) + require.Equal(t, []types.BF16{ + types.BF16FromFloat32(1), types.BF16FromFloat32(2), types.BF16FromFloat32(-0.5), + }, vector.GetArrayAt[types.BF16](vec, 0)) + }) + + t.Run("float list to vecf16", func(t *testing.T) { + f, page := writeListAndGetPage(t, parquet.Leaf(parquet.FloatType), []parquet.Row{ + { + parquet.FloatValue(0.5).Level(0, 1, 0), + parquet.FloatValue(0.25).Level(1, 1, 0), + parquet.FloatValue(4).Level(1, 1, 0), + }, + }) + + var h ParquetHandler + leaf, mp := h.getNestedListMapper(f.Root().Column("c"), plan.Type{Id: int32(types.T_array_float16), Width: 3}) + require.NotNil(t, leaf) + require.NotNil(t, mp) + + vec := vector.NewVec(types.New(types.T_array_float16, 3, 0)) + require.NoError(t, mp.mapping(page, proc, vec)) + require.Equal(t, 1, vec.Length()) + require.Equal(t, []types.Float16{ + types.Float16FromFloat32(0.5), types.Float16FromFloat32(0.25), types.Float16FromFloat32(4), + }, vector.GetArrayAt[types.Float16](vec, 0)) + }) + + t.Run("int32 list to vecint8", func(t *testing.T) { + f, page := writeListAndGetPage(t, parquet.Leaf(parquet.Int32Type), []parquet.Row{ + { + parquet.Int32Value(-128).Level(0, 1, 0), + parquet.Int32Value(0).Level(1, 1, 0), + parquet.Int32Value(127).Level(1, 1, 0), + }, + }) + + var h ParquetHandler + leaf, mp := h.getNestedListMapper(f.Root().Column("c"), plan.Type{Id: int32(types.T_array_int8), Width: 3}) + require.NotNil(t, leaf) + require.NotNil(t, mp) + + vec := vector.NewVec(types.New(types.T_array_int8, 3, 0)) + require.NoError(t, mp.mapping(page, proc, vec)) + require.Equal(t, 1, vec.Length()) + require.Equal(t, []int8{-128, 0, 127}, vector.GetArrayAt[int8](vec, 0)) + }) + + t.Run("int32 list to vecuint8", func(t *testing.T) { + f, page := writeListAndGetPage(t, parquet.Leaf(parquet.Int32Type), []parquet.Row{ + { + parquet.Int32Value(0).Level(0, 1, 0), + parquet.Int32Value(128).Level(1, 1, 0), + parquet.Int32Value(255).Level(1, 1, 0), + }, + }) + + var h ParquetHandler + leaf, mp := h.getNestedListMapper(f.Root().Column("c"), plan.Type{Id: int32(types.T_array_uint8), Width: 3}) + require.NotNil(t, leaf) + require.NotNil(t, mp) + + vec := vector.NewVec(types.New(types.T_array_uint8, 3, 0)) + require.NoError(t, mp.mapping(page, proc, vec)) + require.Equal(t, 1, vec.Length()) + require.Equal(t, []uint8{0, 128, 255}, vector.GetArrayAt[uint8](vec, 0)) + }) + + t.Run("vecint8 out of range rejected", func(t *testing.T) { + f, page := writeListAndGetPage(t, parquet.Leaf(parquet.Int32Type), []parquet.Row{ + { + parquet.Int32Value(200).Level(0, 1, 0), + parquet.Int32Value(0).Level(1, 1, 0), + parquet.Int32Value(0).Level(1, 1, 0), + }, + }) + + var h ParquetHandler + _, mp := h.getNestedListMapper(f.Root().Column("c"), plan.Type{Id: int32(types.T_array_int8), Width: 3}) + require.NotNil(t, mp) + + vec := vector.NewVec(types.New(types.T_array_int8, 3, 0)) + require.ErrorContains(t, mp.mapping(page, proc, vec), "out of range") + }) + + t.Run("vecuint8 out of range rejected", func(t *testing.T) { + f, page := writeListAndGetPage(t, parquet.Leaf(parquet.Int32Type), []parquet.Row{ + { + parquet.Int32Value(-1).Level(0, 1, 0), + parquet.Int32Value(0).Level(1, 1, 0), + parquet.Int32Value(0).Level(1, 1, 0), + }, + }) + + var h ParquetHandler + _, mp := h.getNestedListMapper(f.Root().Column("c"), plan.Type{Id: int32(types.T_array_uint8), Width: 3}) + require.NotNil(t, mp) + + vec := vector.NewVec(types.New(types.T_array_uint8, 3, 0)) + require.ErrorContains(t, mp.mapping(page, proc, vec), "out of range") + }) + + t.Run("vecbf16 rejects int32 leaf", func(t *testing.T) { + f, _ := writeListAndGetPage(t, parquet.Leaf(parquet.Int32Type), []parquet.Row{ + {parquet.Int32Value(1).Level(0, 1, 0)}, + }) + + var h ParquetHandler + leaf, mp := h.getNestedListMapper(f.Root().Column("c"), plan.Type{Id: int32(types.T_array_bf16), Width: 1}) + require.Nil(t, leaf) + require.Nil(t, mp) + }) + + t.Run("vecint8 rejects float leaf", func(t *testing.T) { + f, _ := writeListAndGetPage(t, parquet.Leaf(parquet.FloatType), []parquet.Row{ + {parquet.FloatValue(1).Level(0, 1, 0)}, + }) + + var h ParquetHandler + leaf, mp := h.getNestedListMapper(f.Root().Column("c"), plan.Type{Id: int32(types.T_array_int8), Width: 1}) + require.Nil(t, leaf) + require.Nil(t, mp) + }) } func TestParquetCrossTypeMappings(t *testing.T) { diff --git a/pkg/sql/colexec/group/exec2.go b/pkg/sql/colexec/group/exec2.go index 8f8e9aae46758..9f17dc9d420a5 100644 --- a/pkg/sql/colexec/group/exec2.go +++ b/pkg/sql/colexec/group/exec2.go @@ -180,6 +180,11 @@ func GetKeyWidth(id types.T, width0 int32, nullable bool) (width int) { if id == types.T_array_float64 { width *= 8 } + if id == types.T_array_bf16 || id == types.T_array_float16 { + width *= 2 + } + // T_array_int8 / T_array_uint8 are 1 byte/element -> width unchanged + // (width0 already counts). } else { width = id.TypeLen() } diff --git a/pkg/sql/colexec/productl2/product_l2.go b/pkg/sql/colexec/productl2/product_l2.go index 05c31ae8e40e1..27decd51c840d 100644 --- a/pkg/sql/colexec/productl2/product_l2.go +++ b/pkg/sql/colexec/productl2/product_l2.go @@ -274,27 +274,55 @@ func newMat[T types.RealNumbers](ctr *container, ap *Productl2, probes [][]T, nu } } + // T is the centroid (index) element type. T==float32 covers f32 centroids, + // which a base of any type (f32/f64/narrow) is decoded to; T==float64 is the + // plain f64 index where the base is f64 and reinterpreted directly. + _, toF32 := any(*new(T)).(float32) + oid := tblColVec.GetType().Oid for j := 0; j < probeCount; j++ { if tblColVec.IsNull(uint64(j)) { probes[j] = nullvec continue } - v := types.BytesToArray[T](tblColVec.GetBytesAt(j)) - probes[j] = v + b := tblColVec.GetBytesAt(j) + if !toF32 { + probes[j] = types.BytesToArray[T](b) // f64 centroids: base is f64 + continue + } + var f32 []float32 + switch oid { + case types.T_array_float64: + f64 := types.BytesToArray[float64](b) + f32 = make([]float32, len(f64)) + for i, x := range f64 { + f32[i] = float32(x) + } + case types.T_array_bf16: + f32 = types.BF16ToFloat32Slice(types.BytesToArray[types.BF16](b)) + case types.T_array_float16: + f32 = types.Float16ToFloat32Slice(types.BytesToArray[types.Float16](b)) + case types.T_array_int8: + f32 = types.Int8ToFloat32Slice(types.BytesToArray[int8](b)) + case types.T_array_uint8: + f32 = types.Uint8ToFloat32Slice(types.BytesToArray[uint8](b)) + default: // T_array_float32 + f32 = types.BytesToArray[float32](b) + } + probes[j] = any(f32).([]T) } return probes, nil } func (ctr *container) probe(ap *Productl2, proc *process.Process, result *vm.CallResult) error { - tblColPos := ap.OnExpr.GetF().GetArgs()[1].GetCol().GetColPos() - switch ctr.inBat.Vecs[tblColPos].GetType().Oid { - case types.T_array_float32: - return probeRun[float32](ctr, ap, proc, result) - case types.T_array_float64: + // Dispatch on the CENTROID (index) type, not the base type: under QUANTIZATION + // an f64/narrow base is assigned against f32 centroids, so the base must be + // decoded to f32 (in newMat) to match. Only a plain f64 index keeps f64. + centroidColPos := ap.OnExpr.GetF().GetArgs()[0].GetCol().GetColPos() + if ctr.bat.Vecs[centroidColPos].GetType().Oid == types.T_array_float64 { return probeRun[float64](ctr, ap, proc, result) } - return nil + return probeRun[float32](ctr, ap, proc, result) } func (ctr *container) release() { diff --git a/pkg/sql/colexec/table_function/bm25_compact.go b/pkg/sql/colexec/table_function/bm25_compact.go new file mode 100644 index 0000000000000..9239a167de19d --- /dev/null +++ b/pkg/sql/colexec/table_function/bm25_compact.go @@ -0,0 +1,129 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package table_function + +import ( + "strconv" + + "github.com/matrixorigin/matrixone/pkg/bm25/wand" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/sql/colexec" + veccache "github.com/matrixorigin/matrixone/pkg/vectorindex/cache" + "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" + "github.com/matrixorigin/matrixone/pkg/vm" + "github.com/matrixorigin/matrixone/pkg/vm/process" +) + +// bm25CompactState runs a WAND retrieval-index tiered merge-compaction as +// a standalone table function: `SELECT * FROM bm25_compact(db, store, meta)` +// (no CROSS APPLY / no driving table). It reads the three identifying args in +// start() and, in end(), folds the tag=0 base + the tag=1 CdcTail into a fresh, +// capacity-split tag=0 base and deletes the inputs — via wand.CompactSegments, in +// the statement's transaction. Reached from idxcron / ALTER … REINDEX … FULLTEXT +// MERGE. Its output is a single discarded status row (mirrors bm25_create). +type bm25CompactState struct { + inited bool + tblcfg wand.TableConfig + capacity int64 + batch *batch.Batch +} + +func (u *bm25CompactState) reset(tf *TableFunction, proc *process.Process) { + if u.batch != nil { + u.batch.CleanOnlyData() + } +} + +func (u *bm25CompactState) call(tf *TableFunction, proc *process.Process) (vm.CallResult, error) { + if u.batch != nil { + u.batch.CleanOnlyData() + } + return vm.CancelResult, nil +} + +func (u *bm25CompactState) free(tf *TableFunction, proc *process.Process, pipelineFailed bool, err error) { + if u.batch != nil { + u.batch.Clean(proc.Mp()) + } +} + +// start reads the four varchar args once — [0]=db, [1]=store table, [2]=metadata +// table, [3]=max_index_capacity — into the TableConfig + capacity the compaction runs +// against. Capacity is passed explicitly (resolved by the compile layer from the index's +// persisted algo_params) rather than resolved here, so a manual MERGE and the background +// idxcron MERGE always use the SAME build-time capacity regardless of session. +func (u *bm25CompactState) start(tf *TableFunction, proc *process.Process, nthRow int, analyzer process.Analyzer) error { + if u.inited { + return nil + } + for i := 0; i < 4; i++ { + v := tf.ctr.argVecs[i] + if v.GetType().Oid != types.T_varchar { + return moerr.NewInvalidInput(proc.Ctx, "bm25_compact: args (db, store, meta, capacity) must be strings") + } + if !v.IsConst() { + return moerr.NewInternalError(proc.Ctx, "bm25_compact: args must be string constants") + } + } + u.tblcfg = wand.TableConfig{ + DbName: tf.ctr.argVecs[0].UnsafeGetStringAt(0), + IndexTable: tf.ctr.argVecs[1].UnsafeGetStringAt(0), + MetadataTable: tf.ctr.argVecs[2].UnsafeGetStringAt(0), + } + if u.tblcfg.DbName == "" || u.tblcfg.IndexTable == "" || u.tblcfg.MetadataTable == "" { + return moerr.NewInternalError(proc.Ctx, "bm25_compact: db/store/meta must be non-empty") + } + cap, err := strconv.ParseInt(tf.ctr.argVecs[3].UnsafeGetStringAt(0), 10, 64) + if err != nil { + return moerr.NewInvalidInput(proc.Ctx, "bm25_compact: capacity must be an integer") + } + u.capacity = cap + u.batch = tf.createResultBatch() + u.inited = true + return nil +} + +// end runs the tiered merge-compaction in the statement transaction. +func (u *bm25CompactState) end(tf *TableFunction, proc *process.Process) error { + if !u.inited { + return nil + } + sqlproc := sqlexec.NewSqlProcess(proc) + + // capacity was resolved by the compile layer from the index's persisted algo_params + // (the immutable max_index_capacity flat param) and passed in as arg[3], so fold-split + // and tiered-merge fullness always match what the base was built with. + if _, err := wand.CompactSegments(sqlproc, u.tblcfg, u.capacity); err != nil { + return err + } + // The tag=0 base changed — evict any cached search index so the next query + // reloads the merged base instead of the stale one held until the TTL. + veccache.Cache.Remove(u.tblcfg.IndexTable) + return nil +} + +func bm25CompactPrepare(proc *process.Process, arg *TableFunction) (tvfState, error) { + if len(arg.Args) != 4 { + return nil, moerr.NewInvalidInput(proc.Ctx, "bm25_compact: expects 4 args (db, store, meta, capacity)") + } + var err error + st := &bm25CompactState{} + arg.ctr.executorsForArgs, err = colexec.NewExpressionExecutorsFromPlanExpressions(proc, arg.Args) + arg.ctr.argVecs = make([]*vector.Vector, len(arg.Args)) + return st, err +} diff --git a/pkg/sql/colexec/table_function/bm25_compact_test.go b/pkg/sql/colexec/table_function/bm25_compact_test.go new file mode 100644 index 0000000000000..90d7adeca243d --- /dev/null +++ b/pkg/sql/colexec/table_function/bm25_compact_test.go @@ -0,0 +1,89 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package table_function + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/testutil" +) + +func TestBm25CompactPrepare_WrongArgCount(t *testing.T) { + m := mpool.MustNewZero() + proc := testutil.NewProcessWithMPool(t, "", m) + proc.Ctx = context.Background() + + arg := &TableFunction{Args: []*plan.Expr{bm25VarcharCol(), bm25VarcharCol()}} // only 2 + _, err := bm25CompactPrepare(proc, arg) + require.Error(t, err) + require.Contains(t, err.Error(), "expects 4 args") +} + +func TestBm25CompactStart_Errors(t *testing.T) { + m := mpool.MustNewZero() + proc := testutil.NewProcessWithMPool(t, "", m) + proc.Ctx = context.Background() + + // argVecs: [0]=db, [1]=store, [2]=meta, [3]=capacity — all const strings on the + // happy path. Each case corrupts one. + cases := []struct { + name string + vecs func() []*vector.Vector + wantErr string + }{ + { + name: "arg not string", + vecs: func() []*vector.Vector { + return []*vector.Vector{bm25TestConstInt(m), bm25TestConstVarchar(m, "s"), bm25TestConstVarchar(m, "meta"), bm25TestConstVarchar(m, "1000")} + }, + wantErr: "must be strings", + }, + { + name: "arg not const", + vecs: func() []*vector.Vector { + return []*vector.Vector{bm25TestNonConstVarchar(m), bm25TestConstVarchar(m, "s"), bm25TestConstVarchar(m, "meta"), bm25TestConstVarchar(m, "1000")} + }, + wantErr: "", + }, + { + name: "capacity not integer", + vecs: func() []*vector.Vector { + return []*vector.Vector{bm25TestConstVarchar(m, "db"), bm25TestConstVarchar(m, "s"), bm25TestConstVarchar(m, "meta"), bm25TestConstVarchar(m, "abc")} + }, + wantErr: "capacity must be an integer", + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + arg := &TableFunction{Args: []*plan.Expr{bm25VarcharCol(), bm25VarcharCol(), bm25VarcharCol(), bm25VarcharCol()}} + st, err := bm25CompactPrepare(proc, arg) + require.NoError(t, err) + + arg.ctr.argVecs = tt.vecs() + err = st.start(arg, proc, 0, nil) + require.Error(t, err) + if tt.wantErr != "" { + require.Contains(t, err.Error(), tt.wantErr) + } + }) + } +} diff --git a/pkg/sql/colexec/table_function/bm25_create.go b/pkg/sql/colexec/table_function/bm25_create.go new file mode 100644 index 0000000000000..d5f727e22f74f --- /dev/null +++ b/pkg/sql/colexec/table_function/bm25_create.go @@ -0,0 +1,281 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package table_function + +import ( + "bytes" + "fmt" + "time" + + "github.com/bytedance/sonic" + "github.com/matrixorigin/matrixone/pkg/bm25/wand" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/datalink" + "github.com/matrixorigin/matrixone/pkg/monlp/tokenizer" + "github.com/matrixorigin/matrixone/pkg/sql/colexec" + veccache "github.com/matrixorigin/matrixone/pkg/vectorindex/cache" + "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" + "github.com/matrixorigin/matrixone/pkg/vm" + "github.com/matrixorigin/matrixone/pkg/vm/process" +) + +var wand_runSql = sqlexec.RunSql + +// bm25DocLenSentinel is the reserved word the classic postings pipeline uses to +// carry per-doc length; the bm25 builder tracks doc length itself, so an Add of +// this word is skipped. Kept identical to the classic fulltext sentinel so a +// postings-fed build (if ever used) stays compatible. +const bm25DocLenSentinel = "__DocLen" + +// bm25CreateState builds a WAND retrieval index from a postings stream +// fed in (word, doc_id, tf) order — the engine-sorted/grouped query +// `SELECT word, doc_id, FROM GROUP BY word, doc_id +// ORDER BY word, doc_id`. It accumulates into a wand.Builder and, at end(), +// serializes + persists the index (metadata + chunk rows) via SQL. Its own +// output is a single discarded status row (mirrors hnsw_create). +type bm25CreateState struct { + inited bool + tblcfg wand.TableConfig + builder *wand.Builder + batch *batch.Batch +} + +func (u *bm25CreateState) reset(tf *TableFunction, proc *process.Process) { + if u.batch != nil { + u.batch.CleanOnlyData() + } +} + +func (u *bm25CreateState) call(tf *TableFunction, proc *process.Process) (vm.CallResult, error) { + u.batch.CleanOnlyData() + if u.batch.RowCount() == 0 { + return vm.CancelResult, nil + } + return vm.CallResult{Status: vm.ExecNext, Batch: u.batch}, nil +} + +func (u *bm25CreateState) free(tf *TableFunction, proc *process.Process, pipelineFailed bool, err error) { + if u.batch != nil { + u.batch.Clean(proc.Mp()) + } +} + +// end finalizes the build and persists the index (idempotent: existing chunks +// for this id are deleted first). +func (u *bm25CreateState) end(tf *TableFunction, proc *process.Process) error { + if !u.inited || u.builder == nil { + return nil + } + sqlproc := sqlexec.NewSqlProcess(proc) + + // Capacity is carried in the cfg by the compile layer (resolved from the index's immutable + // max_index_capacity flat param), so the base splits at the same value every compaction + // later reads. Fall back to the resolver only for an index built before the flat param + // existed (cfg.Capacity == 0). Unresolved / 0 => a single unbounded base. + capacity := u.tblcfg.Capacity + if capacity == 0 { + if rf := sqlproc.GetResolveVariableFunc(); rf != nil { + if v, verr := rf("fulltext_max_index_capacity", true, false); verr == nil { + if c, ok := v.(int64); ok { + capacity = c + } + } + } + } + + // Split the compacted base into capacity-bounded sub-indexes (a single model when + // the corpus fits within capacity), each stored under its own index_id so a large + // corpus builds several tag=0 bases instead of one monolith. + models := u.builder.FinishSegments(capacity) + + // Drop empty sub-models (a segment whose rows carried no searchable tokens). If the + // whole corpus is empty, persist nothing — matching the single-index build. + nonEmpty := models[:0] + for _, m := range models { + if m.NumTerms() == 0 { + m.Free() + continue + } + nonEmpty = append(nonEmpty, m) + } + if len(nonEmpty) == 0 { + return nil + } + // Clear any previous tag=0 bases so the build is idempotent (the tag=1 CdcTail is + // untouched — CREATE has no tail yet anyway). + for _, s := range wand.DeleteAllBasesSqls(u.tblcfg) { + res, err := wand_runSql(sqlproc, s) + if err != nil { + return err + } + res.Close() + } + + // Synchronous CREATE build → the compacted main index (tag=0). Each sub-model is + // spilled to a temp file and read via load_file, so keep the temps until the INSERTs + // have run. A per-build-unique id prefix (index table + build ts) keeps concurrent / + // repeated builds from writing colliding sub-index ids (mirrors HNSW's uid:n). + ts := time.Now().UnixMicro() + uid := fmt.Sprintf("%s:%d", u.tblcfg.IndexTable, ts) + cleanups := make([]func(), 0, len(nonEmpty)) + defer func() { + for _, c := range cleanups { + c() + } + }() + for i, m := range nonEmpty { + m.Id = wand.SubIndexId(uid, i) + sqls, cleanup, err := m.ToInsertSqls(u.tblcfg, ts, 0) + if err != nil { + return err + } + cleanups = append(cleanups, cleanup) + for _, s := range sqls { + res, err := wand_runSql(sqlproc, s) + if err != nil { + return err + } + res.Close() + } + } + // A fresh tag=0 was written (CREATE build) — evict any cached search index so the + // next query reloads the new base(s) instead of the stale one held until the TTL. + veccache.Cache.Remove(u.tblcfg.IndexTable) + return nil +} + +func bm25CreatePrepare(proc *process.Process, arg *TableFunction) (tvfState, error) { + var err error + st := &bm25CreateState{} + arg.ctr.executorsForArgs, err = colexec.NewExpressionExecutorsFromPlanExpressions(proc, arg.Args) + arg.ctr.argVecs = make([]*vector.Vector, len(arg.Args)) + return st, err +} + +// start feeds one row into the builder. Two input shapes, selected by cfg.FromSource: +// - postings mode (default): argVecs [0]=cfg, [1]=word, [2]=doc_id — one row is one +// token occurrence; the builder sums tf per (word, doc_id), skipping __DocLen. +// - source mode: argVecs [0]=cfg, [1]=pk, [2..]=text cols — the row is tokenized in-Go +// (jieba, HMM=false) and every token is Add'd, so no separate postings table or +// tokenize pass is needed. Builder.Add caps tf and tracks doc length internally, so +// there is no __DocLen sentinel here. +func (u *bm25CreateState) start(tf *TableFunction, proc *process.Process, nthRow int, analyzer process.Analyzer) (err error) { + if !u.inited { + cfgVec := tf.ctr.argVecs[0] + if cfgVec.GetType().Oid != types.T_varchar { + return moerr.NewInvalidInput(proc.Ctx, "bm25_create: first argument (config) must be a string") + } + if !cfgVec.IsConst() { + return moerr.NewInternalError(proc.Ctx, "bm25_create: config must be a string constant") + } + cfgstr := cfgVec.UnsafeGetStringAt(0) + if len(cfgstr) == 0 { + return moerr.NewInternalError(proc.Ctx, "bm25_create: config is empty") + } + if err = sonic.Unmarshal([]byte(cfgstr), &u.tblcfg); err != nil { + return err + } + + // The builder's pk type comes from the doc_id column (postings mode) or the pk + // column (source mode); either is the source pk type. + pkVec := tf.ctr.argVecs[2] + if u.tblcfg.FromSource { + pkVec = tf.ctr.argVecs[1] + } else if tf.ctr.argVecs[1].GetType().Oid != types.T_varchar { + return moerr.NewInvalidInput(proc.Ctx, "bm25_create: second argument (word) must be a string") + } + u.builder = wand.NewBuilder(u.tblcfg.IndexTable, int32(pkVec.GetType().Oid)) + u.batch = tf.createResultBatch() + u.inited = true + } + + u.batch.CleanOnlyData() + + if u.tblcfg.FromSource { + return u.addSourceRow(tf, proc, nthRow) + } + + wordVec := tf.ctr.argVecs[1] + docVec := tf.ctr.argVecs[2] + if wordVec.IsNull(uint64(nthRow)) || docVec.IsNull(uint64(nthRow)) { + return nil + } + word := wordVec.GetStringAt(nthRow) + if word == bm25DocLenSentinel { + return nil // BM25 doc-length sentinel, not a real term + } + pk := vector.GetAny(docVec, nthRow, false) + return u.builder.Add(word, pk) +} + +// addSourceRow tokenizes one source row (argVecs [1]=pk, [2..]=text cols) with the +// retrieval jieba tokenizer and Add's every token to the builder. It mirrors +// fulltext_index_tokenize's retrieval branch (concat columns with '\n', datalink → +// plain text, HMM=false), minus the position/__DocLen bookkeeping the WAND builder +// does not need. +func (u *bm25CreateState) addSourceRow(tf *TableFunction, proc *process.Process, nthRow int) error { + argVecs := tf.ctr.argVecs + pkVec := argVecs[1] + if pkVec.IsNull(uint64(nthRow)) { + return nil + } + // Match fulltext_index_tokenize: if any text column is NULL the doc yields no tokens. + for i := 2; i < len(argVecs); i++ { + if argVecs[i].IsNull(uint64(nthRow)) { + return nil + } + } + var content bytes.Buffer + for i := 2; i < len(argVecs); i++ { + if i > 2 { + content.WriteByte('\n') + } + data := argVecs[i].GetStringAt(nthRow) + if types.T(tf.Args[i].Typ.Id) == types.T_datalink { + dl, err := datalink.NewDatalink(data, proc) + if err != nil { + return err + } + b, err := dl.GetPlainText(proc) + if err != nil { + return err + } + content.Write(b) + } else { + content.WriteString(data) + } + } + if content.Len() == 0 { + return nil + } + jtok, err := tokenizer.SharedJiebaTokenizer(false) + if err != nil { + return err + } + pk := vector.GetAny(pkVec, nthRow, false) + for t, terr := range jtok.Tokenize(content.Bytes()) { + if terr != nil { + return terr + } + slen := t.TokenBytes[0] + if aerr := u.builder.Add(string(t.TokenBytes[1:slen+1]), pk); aerr != nil { + return aerr + } + } + return nil +} diff --git a/pkg/sql/colexec/table_function/bm25_create_test.go b/pkg/sql/colexec/table_function/bm25_create_test.go new file mode 100644 index 0000000000000..988262da3760b --- /dev/null +++ b/pkg/sql/colexec/table_function/bm25_create_test.go @@ -0,0 +1,95 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package table_function + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/testutil" +) + +// bm25CreateArg builds a TableFunction with three varchar Args (cfg, word, doc_id), +// the postings-mode shape bm25_create expects. +func bm25CreateArg() *TableFunction { + col := func() *plan.Expr { + return &plan.Expr{ + Expr: &plan.Expr_Col{Col: &plan.ColRef{}}, + Typ: plan.Type{Id: int32(types.T_varchar)}, + } + } + return &TableFunction{Args: []*plan.Expr{col(), col(), col()}} +} + +func TestBm25CreateStart_Errors(t *testing.T) { + m := mpool.MustNewZero() + proc := testutil.NewProcessWithMPool(t, "", m) + proc.Ctx = context.Background() + + constCfg := func(s string) *vector.Vector { + v, err := vector.NewConstBytes(types.T_varchar.ToType(), []byte(s), 1, m) + require.NoError(t, err) + return v + } + constVarchar := func(s string) *vector.Vector { return constCfg(s) } + constInt := func() *vector.Vector { + v, err := vector.NewConstFixed(types.T_int64.ToType(), int64(1), 1, m) + require.NoError(t, err) + return v + } + nonConstVarchar := func() *vector.Vector { + v := vector.NewVec(types.T_varchar.ToType()) + require.NoError(t, vector.AppendBytes(v, []byte(`{}`), false, m)) + return v + } + + cases := []struct { + name string + cfg *vector.Vector + word *vector.Vector + wantErr string + }{ + {"cfg not string", constInt(), constVarchar("w"), "must be a string"}, + {"cfg not const", nonConstVarchar(), constVarchar("w"), "must be a string constant"}, + {"cfg empty", constCfg(""), constVarchar("w"), "config is empty"}, + {"cfg bad json", constCfg(`{`), constVarchar("w"), ""}, + {"word not string", constCfg(`{}`), constInt(), "must be a string"}, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + arg := bm25CreateArg() + st, err := bm25CreatePrepare(proc, arg) + require.NoError(t, err) + + arg.ctr.argVecs = make([]*vector.Vector, 3) + arg.ctr.argVecs[0] = tt.cfg + arg.ctr.argVecs[1] = tt.word + arg.ctr.argVecs[2] = constVarchar("doc1") + + err = st.start(arg, proc, 0, nil) + require.Error(t, err) + if tt.wantErr != "" { + require.Contains(t, err.Error(), tt.wantErr) + } + }) + } +} diff --git a/pkg/sql/colexec/table_function/bm25_search.go b/pkg/sql/colexec/table_function/bm25_search.go new file mode 100644 index 0000000000000..a8f41ba1d44df --- /dev/null +++ b/pkg/sql/colexec/table_function/bm25_search.go @@ -0,0 +1,306 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package table_function + +import ( + "context" + + "github.com/bytedance/sonic" + "github.com/matrixorigin/matrixone/pkg/bm25/wand" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/monlp/tokenizer" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/colexec" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + veccache "github.com/matrixorigin/matrixone/pkg/vectorindex/cache" + "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" + "github.com/matrixorigin/matrixone/pkg/vm" + "github.com/matrixorigin/matrixone/pkg/vm/process" +) + +// bm25SearchState answers a retrieval-mode MATCH by running WAND top-K +// over the in-memory index (loaded + cached via VectorIndexCache), emitting +// (doc_id, score) rows. No SQL ORDER BY/LIMIT sort: the top-K is produced by +// the WAND walk itself. Mirrors hnsw_search. +type bm25SearchState struct { + inited bool + tblcfg wand.TableConfig + limit uint64 + offset int + keys []any // doc_id values of the source pk type + distances []float64 + filterBytes []byte // serialized docfilter membership (WHERE-clause prefilter), if any + batch *batch.Batch + + // Streaming no-LIMIT path (u.limit == 0): rather than materialize every matching + // doc, a producer goroutine runs the WAND search with an Emit callback that hands + // bounded batches to streamCh; call() drains one batch per invocation and the + // upstream ORDER BY score node ranks them. cancel stops the producer (and releases + // the cache read-lock it holds) if the consumer aborts early. + streaming bool + streamCh chan wandStreamBatch + errCh chan error + cancel context.CancelFunc + done bool +} + +// wandStreamBatch is one emitted batch (<= streamBatch rows); the producer hands +// ownership to the consumer, so the slices are not reused. +type wandStreamBatch struct { + keys []any + distances []float64 +} + +func (u *bm25SearchState) end(tf *TableFunction, proc *process.Process) error { return nil } + +func (u *bm25SearchState) reset(tf *TableFunction, proc *process.Process) { + u.stopStream() + if u.batch != nil { + u.batch.CleanOnlyData() + } + u.offset = 0 + u.keys = nil + u.distances = nil + u.filterBytes = nil + u.streaming = false + u.errCh = nil + u.done = false +} + +// stopStream cancels the producer goroutine (if streaming) and drains streamCh +// until the producer closes it, so no goroutine — nor the cache read-lock it holds +// — leaks past this query. Idempotent; a no-op when not streaming. +func (u *bm25SearchState) stopStream() { + if u.cancel == nil { + return + } + u.cancel() // unblocks the producer's Emit (its select sees ctx.Done()) + if u.streamCh != nil { + for range u.streamCh { // drain to the producer's close() + } + } + u.cancel = nil + u.streamCh = nil +} + +func (u *bm25SearchState) call(tf *TableFunction, proc *process.Process) (vm.CallResult, error) { + u.batch.CleanOnlyData() + // The projection may request only doc_id (1 column, e.g. COUNT(*) or a bare + // WHERE match) or doc_id+score (2 columns) — mirror fulltext_index_scan and + // only emit score when the batch has it. + withScore := u.batch.VectorCount() > 1 + + if u.streaming { + if u.done { + return vm.CancelResult, nil + } + select { + case b, ok := <-u.streamCh: + if !ok { + // producer finished; surface any search error. errCh is sent before + // the channel is closed, so it is ready here. + u.done = true + u.cancel = nil + if e := <-u.errCh; e != nil { + return vm.CancelResult, e + } + return vm.CancelResult, nil + } + for i := range b.keys { + vector.AppendAny(u.batch.Vecs[0], b.keys[i], false, proc.Mp()) + if withScore { + vector.AppendFixed[float64](u.batch.Vecs[1], b.distances[i], false, proc.Mp()) + } + } + u.batch.SetRowCount(len(b.keys)) + return vm.CallResult{Status: vm.ExecNext, Batch: u.batch}, nil + case <-proc.Ctx.Done(): + return vm.CancelResult, proc.Ctx.Err() + } + } + + nkeys := len(u.keys) + n := 0 + for i := u.offset; i < nkeys && n < 8192; i++ { + vector.AppendAny(u.batch.Vecs[0], u.keys[i], false, proc.Mp()) + if withScore { + vector.AppendFixed[float64](u.batch.Vecs[1], u.distances[i], false, proc.Mp()) + } + n++ + } + u.offset += n + u.batch.SetRowCount(n) + + if u.batch.RowCount() == 0 { + return vm.CancelResult, nil + } + return vm.CallResult{Status: vm.ExecNext, Batch: u.batch}, nil +} + +func (u *bm25SearchState) free(tf *TableFunction, proc *process.Process, pipelineFailed bool, err error) { + u.stopStream() + if u.batch != nil { + u.batch.Clean(proc.Mp()) + } +} + +func bm25SearchPrepare(proc *process.Process, arg *TableFunction) (tvfState, error) { + var err error + st := &bm25SearchState{} + arg.ctr.executorsForArgs, err = colexec.NewExpressionExecutorsFromPlanExpressions(proc, arg.Args) + arg.ctr.argVecs = make([]*vector.Vector, len(arg.Args)) + + // Top-K limit, pushed down onto the node by the planner (apply_indices). + // When absent (e.g. the LIMIT lives on a SORT above the join), leave it 0 — + // the search then returns all matches and the SORT bounds them, matching + // fulltext_index_scan. Do NOT default to 1. + if arg.Limit != nil { + if cExpr, ok := arg.Limit.Expr.(*plan.Expr_Lit); ok { + switch v := cExpr.Lit.Value.(type) { + case *plan.Literal_U64Val: + st.limit = v.U64Val + case *plan.Literal_I64Val: + if v.I64Val > 0 { + st.limit = uint64(v.I64Val) + } + } + } + } + return st, err +} + +// start runs one query. argVecs: [0]=cfg(json const), [1]=pattern(varchar). +func (u *bm25SearchState) start(tf *TableFunction, proc *process.Process, nthRow int, analyzer process.Analyzer) (err error) { + if !u.inited { + cfgVec := tf.ctr.argVecs[0] + if cfgVec.GetType().Oid != types.T_varchar { + return moerr.NewInvalidInput(proc.Ctx, "bm25_search: first argument (config) must be a string") + } + if !cfgVec.IsConst() { + return moerr.NewInternalError(proc.Ctx, "bm25_search: config must be a string constant") + } + cfgstr := cfgVec.UnsafeGetStringAt(0) + if len(cfgstr) == 0 { + return moerr.NewInternalError(proc.Ctx, "bm25_search: config is empty") + } + if err = sonic.Unmarshal([]byte(cfgstr), &u.tblcfg); err != nil { + return err + } + patVec := tf.ctr.argVecs[1] + if patVec.GetType().Oid != types.T_varchar { + return moerr.NewInvalidInput(proc.Ctx, "bm25_search: second argument (pattern) must be a string") + } + u.batch = tf.createResultBatch() + u.inited = true + } + + u.stopStream() + u.offset = 0 + u.keys = nil + u.distances = nil + u.streaming = false + u.done = false + u.batch.CleanOnlyData() + + patVec := tf.ctr.argVecs[1] + if patVec.IsNull(uint64(nthRow)) { + return nil + } + pattern := patVec.GetStringAt(nthRow) + + // Tokenize the query exactly as the index was built: jieba (HMM=false), + // the same tokenizer bm25_create uses to build the index. + jtok, err := tokenizer.SharedJiebaTokenizer(false) + if err != nil { + return err + } + terms := make([]string, 0, 8) + for t, terr := range jtok.Tokenize([]byte(pattern)) { + if terr != nil { + return terr + } + slen := t.TokenBytes[0] + if w := string(t.TokenBytes[1 : slen+1]); w != "" { + terms = append(terms, w) + } + } + if len(terms) == 0 { + return nil // empty query → no hits + } + + // Prefilter pushdown: when the WHERE clause is pushed down as a unique-join- + // keys runtime filter, wait for it and build the docfilter membership bytes — + // the same mechanism fulltext_index_scan uses. Applied inside the WAND walk + // so the returned top-K is already filtered (no over-fetch). + if u.filterBytes == nil && len(tf.RuntimeFilterSpecs) > 0 { + res, ferr := waitFulltextMembershipFilter(proc, tf.RuntimeFilterSpecs) + if ferr != nil { + return ferr + } + if res != nil { + u.filterBytes = res.membershipFilterBytes + } + } + + veccache.Cache.Once() + + algo := wand.NewWandSearch(u.tblcfg) + q := wand.WandQuery{Terms: terms, FilterBytes: u.filterBytes} + + if u.limit == 0 { + // No pushed LIMIT: STREAM every matching doc in bounded batches (no top-K + // heap, no materialization of the whole result set). A producer goroutine runs + // the search with an Emit callback that hands batches to streamCh; call() drains + // one per invocation and the upstream ORDER BY score node ranks. cancel/ctx let + // reset()/free() stop the producer and release the cache read-lock it holds. + u.streaming = true + u.streamCh = make(chan wandStreamBatch, 4) + u.errCh = make(chan error, 1) + ctx, cancel := context.WithCancel(proc.Ctx) + u.cancel = cancel + rt := vectorindex.RuntimeConfig{Emit: func(keys []any, dists []float64) error { + select { + case u.streamCh <- wandStreamBatch{keys: keys, distances: dists}: + return nil + case <-ctx.Done(): + return ctx.Err() + } + }} + sp := sqlexec.NewSqlProcess(proc) + go func() { + _, _, serr := veccache.Cache.Search(sp, u.tblcfg.IndexTable, algo, q, rt) + u.errCh <- serr // buffered(1): send before close so call() reads it after drain + close(u.streamCh) + }() + return nil + } + + // With a pushed LIMIT: WAND top-K, returned all at once (bounded by the LIMIT). + rt := vectorindex.RuntimeConfig{Limit: uint(u.limit)} + keys, dists, err := veccache.Cache.Search(sqlexec.NewSqlProcess(proc), u.tblcfg.IndexTable, algo, q, rt) + if err != nil { + return err + } + ks, ok := keys.([]any) + if !ok { + return moerr.NewInternalError(proc.Ctx, "wand search: keys is not []any") + } + u.keys = ks + u.distances = dists + return nil +} diff --git a/pkg/sql/colexec/table_function/bm25_search_test.go b/pkg/sql/colexec/table_function/bm25_search_test.go new file mode 100644 index 0000000000000..5e48b0eb5dbed --- /dev/null +++ b/pkg/sql/colexec/table_function/bm25_search_test.go @@ -0,0 +1,95 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package table_function + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/testutil" +) + +// bm25TestConstVarchar / …ConstInt / …NonConstVarchar are shared vector builders for +// the bm25 TVF start()-error tests. +func bm25TestConstVarchar(m *mpool.MPool, s string) *vector.Vector { + v, err := vector.NewConstBytes(types.T_varchar.ToType(), []byte(s), 1, m) + if err != nil { + panic(err) + } + return v +} + +func bm25TestConstInt(m *mpool.MPool) *vector.Vector { + v, err := vector.NewConstFixed(types.T_int64.ToType(), int64(1), 1, m) + if err != nil { + panic(err) + } + return v +} + +func bm25TestNonConstVarchar(m *mpool.MPool) *vector.Vector { + v := vector.NewVec(types.T_varchar.ToType()) + if err := vector.AppendBytes(v, []byte(`{}`), false, m); err != nil { + panic(err) + } + return v +} + +func bm25VarcharCol() *plan.Expr { + return &plan.Expr{Expr: &plan.Expr_Col{Col: &plan.ColRef{}}, Typ: plan.Type{Id: int32(types.T_varchar)}} +} + +func TestBm25SearchStart_Errors(t *testing.T) { + m := mpool.MustNewZero() + proc := testutil.NewProcessWithMPool(t, "", m) + proc.Ctx = context.Background() + + cases := []struct { + name string + cfg *vector.Vector + pat *vector.Vector + wantErr string + }{ + {"cfg not string", bm25TestConstInt(m), bm25TestConstVarchar(m, "apple"), "must be a string"}, + {"cfg not const", bm25TestNonConstVarchar(m), bm25TestConstVarchar(m, "apple"), "must be a string constant"}, + {"cfg empty", bm25TestConstVarchar(m, ""), bm25TestConstVarchar(m, "apple"), "config is empty"}, + {"cfg bad json", bm25TestConstVarchar(m, `{`), bm25TestConstVarchar(m, "apple"), ""}, + {"pattern not string", bm25TestConstVarchar(m, `{}`), bm25TestConstInt(m), "must be a string"}, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + arg := &TableFunction{Args: []*plan.Expr{bm25VarcharCol(), bm25VarcharCol()}} + st, err := bm25SearchPrepare(proc, arg) + require.NoError(t, err) + + arg.ctr.argVecs = make([]*vector.Vector, 2) + arg.ctr.argVecs[0] = tt.cfg + arg.ctr.argVecs[1] = tt.pat + + err = st.start(arg, proc, 0, nil) + require.Error(t, err) + if tt.wantErr != "" { + require.Contains(t, err.Error(), tt.wantErr) + } + }) + } +} diff --git a/pkg/sql/colexec/table_function/cagra_create_gpu.go b/pkg/sql/colexec/table_function/cagra_create_gpu.go index bbf52fcf54d68..8a3a531547d15 100644 --- a/pkg/sql/colexec/table_function/cagra_create_gpu.go +++ b/pkg/sql/colexec/table_function/cagra_create_gpu.go @@ -23,6 +23,7 @@ import ( "github.com/bytedance/sonic" "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" @@ -48,16 +49,34 @@ var cagraCatalogHooks = cagrart.CatalogHooks{} var cagra_runSql = sqlexec.RunSql +// cagraBuilder is the (B, Q)-erased build interface the create state drives. +// *cagraPkg.CagraBuild[B, Q] satisfies it for every wired (base, storage) +// combo. GetIndexes is [B,Q]-typed and intentionally NOT on the interface — +// end() routes through ToInsertSql instead. +type cagraBuilder interface { + // AddRow takes the raw base-type bytes of one vector (4*dim for an f32 base, + // 2*dim for an f16 base); the concrete builder reinterprets them to its + // []B/[]Q with UnsafeSliceCast (the interface can't name B). Passing []byte + // rather than `any` keeps the per-row build hot path allocation-free. + AddRow(id int64, vecBytes []byte) error + SetFilterColumns(colMetaJSON string) + AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error + ToInsertSql(ts int64) ([]string, error) + Destroy() error +} + type cagraCreateState struct { - inited bool - buildf32 *cagraPkg.CagraBuild[float32] - buildf16 *cagraPkg.CagraBuild[cuvs.Float16] - buildi8 *cagraPkg.CagraBuild[int8] - buildui8 *cagraPkg.CagraBuild[uint8] - param vectorindex.CagraParam - tblcfg vectorindex.IndexTableConfig - idxcfg vectorindex.IndexConfig - offset int + inited bool + builder cagraBuilder + param vectorindex.CagraParam + tblcfg vectorindex.IndexTableConfig + idxcfg vectorindex.IndexConfig + offset int + + // baseOid is the base (source) vector column element type — f32 or f16. + // The storage/quantization type (which builder is non-nil) may differ: + // f16 base is stored as half (direct) or quantized to int8/uint8. + baseOid types.T // filterCols is the INCLUDE column metadata derived at start() from // param.IncludedColumns (names) + argVecs[3:] (types). Empty when the @@ -95,19 +114,11 @@ func (u *cagraCreateState) end(tf *TableFunction, proc *process.Process) error { ) ts := time.Now().UnixMicro() - switch { - case u.buildf32 != nil: - sqls, err = u.buildf32.ToInsertSql(ts) - case u.buildf16 != nil: - sqls, err = u.buildf16.ToInsertSql(ts) - case u.buildi8 != nil: - sqls, err = u.buildi8.ToInsertSql(ts) - case u.buildui8 != nil: - sqls, err = u.buildui8.ToInsertSql(ts) - default: - // No builder selected → init didn't set one. Nothing to do for - // the cuvs side; the CDC tail (if any) below still emits. + if u.builder != nil { + sqls, err = u.builder.ToInsertSql(ts) } + // No builder selected → init didn't set one. Nothing to do for the cuvs + // side; the CDC tail (if any) below still emits. if err != nil { return err } @@ -121,9 +132,14 @@ func (u *cagraCreateState) end(tf *TableFunction, proc *process.Process) error { // record 0. Search-side can recover the INCLUDE-column layout // for tag=1 replay even when no tag=0 sub-index exists. colMetaJSON := colMetaJSONFromCols(u.filterCols) + // vecBytesPerRow = dim * base element size (2 for vecf16, else 4). + elemSize := 4 + if u.baseOid == types.T_array_float16 { + elemSize = 2 + } + vecBytesPerRow := int(u.idxcfg.CuvsCagra.Dimensions) * elemSize tailSqls, err := cuvscdc.SaveSmallTailAsCdc( - u.tblcfg, u.cdcTail, - int(u.idxcfg.CuvsCagra.Dimensions), ibpr, colMetaJSON) + u.tblcfg, u.cdcTail, vecBytesPerRow, ibpr, colMetaJSON) if err != nil { return err } @@ -160,17 +176,8 @@ func (u *cagraCreateState) free(tf *TableFunction, proc *process.Process, pipeli if u.batch != nil { u.batch.Clean(proc.Mp()) } - if u.buildf32 != nil { - u.buildf32.Destroy() - } - if u.buildf16 != nil { - u.buildf16.Destroy() - } - if u.buildi8 != nil { - u.buildi8.Destroy() - } - if u.buildui8 != nil { - u.buildui8.Destroy() + if u.builder != nil { + u.builder.Destroy() } } @@ -340,7 +347,16 @@ func (u *cagraCreateState) start(tf *TableFunction, proc *process.Process, nthRo faVec := tf.ctr.argVecs[2] if !catalogplugin.SupportsVectorType(cagraCatalogHooks, faVec.GetType().Oid) { - return moerr.NewInvalidInput(proc.Ctx, "third argument (vector) must be a float32 array") + return moerr.NewInvalidInput(proc.Ctx, "third argument (vector) must be a float32 / float16 array") + } + u.baseOid = faVec.GetType().Oid + + // Derive the storage qtype from the base column type when no QUANTIZATION + // was given: a vecf16 base with no quantization is stored natively as half. + // (vecf16 + QUANTIZATION=int8/uint8 keeps qt = int8/uint8 — quantize path.) + if u.baseOid == types.T_array_float16 && qt == metric.Quantization_F32 { + qt = metric.Quantization_F16 + u.idxcfg.CuvsCagra.Quantization = uint16(qt) } // dimension @@ -357,15 +373,25 @@ func (u *cagraCreateState) start(tf *TableFunction, proc *process.Process, nthRo uid := fmt.Sprintf("%s:%d:%d", tf.CnAddr, tf.MaxParallel, tf.ParallelID) // ---- create builder ---- - switch qt { - case metric.Quantization_F16: - u.buildf16, err = cagraPkg.NewCagraBuild[cuvs.Float16](uid, u.idxcfg, u.tblcfg, nthread, devices) - case metric.Quantization_INT8: - u.buildi8, err = cagraPkg.NewCagraBuild[int8](uid, u.idxcfg, u.tblcfg, nthread, devices) - case metric.Quantization_UINT8: - u.buildui8, err = cagraPkg.NewCagraBuild[uint8](uid, u.idxcfg, u.tblcfg, nthread, devices) + // One real [B, Q] builder keyed on (base column type, storage qtype). + // The 7 wired combos: f32 base × {f32, f16, int8, uint8}; f16 base × + // {f16, int8, uint8}. + isF16Base := u.baseOid == types.T_array_float16 + switch { + case isF16Base && qt == metric.Quantization_F16: + u.builder, err = cagraPkg.NewCagraBuild[cuvs.Float16, cuvs.Float16](uid, u.idxcfg, u.tblcfg, nthread, devices) + case isF16Base && qt == metric.Quantization_INT8: + u.builder, err = cagraPkg.NewCagraBuild[cuvs.Float16, int8](uid, u.idxcfg, u.tblcfg, nthread, devices) + case isF16Base && qt == metric.Quantization_UINT8: + u.builder, err = cagraPkg.NewCagraBuild[cuvs.Float16, uint8](uid, u.idxcfg, u.tblcfg, nthread, devices) + case qt == metric.Quantization_F16: + u.builder, err = cagraPkg.NewCagraBuild[float32, cuvs.Float16](uid, u.idxcfg, u.tblcfg, nthread, devices) + case qt == metric.Quantization_INT8: + u.builder, err = cagraPkg.NewCagraBuild[float32, int8](uid, u.idxcfg, u.tblcfg, nthread, devices) + case qt == metric.Quantization_UINT8: + u.builder, err = cagraPkg.NewCagraBuild[float32, uint8](uid, u.idxcfg, u.tblcfg, nthread, devices) default: - u.buildf32, err = cagraPkg.NewCagraBuild[float32](uid, u.idxcfg, u.tblcfg, nthread, devices) + u.builder, err = cagraPkg.NewCagraBuild[float32, float32](uid, u.idxcfg, u.tblcfg, nthread, devices) } if err != nil { return err @@ -381,7 +407,7 @@ func (u *cagraCreateState) start(tf *TableFunction, proc *process.Process, nthRo if len(u.filterCols) > 0 { logutil.Infof("CAGRA create: INCLUDE columns = %v (from %d arg vectors)", u.filterCols, len(tf.ctr.argVecs)-3) - if err = initFilterColumns(u.activeBuilder(), u.filterCols); err != nil { + if err = initFilterColumns(u.builder, u.filterCols); err != nil { return err } } @@ -413,16 +439,28 @@ func (u *cagraCreateState) start(tf *TableFunction, proc *process.Process, nthRo u.rowsSeen++ id := vector.GetFixedAtNoTypeCheck[int64](tf.ctr.argVecs[1], nthRow) - fa := types.BytesToArray[float32](faVec.GetBytesAt(nthRow)) - if uint(len(fa)) != u.idxcfg.CuvsCagra.Dimensions { - return moerr.NewInternalError(proc.Ctx, "vector dimension mismatch") + // Decode the base vector to its native type (see ivfpq_create_gpu.go for the + // rationale). f16 base -> native []cuvs.Float16 for both the direct (half) + // add and the CDC tail (stored as native half bytes — no f32 detour). + var fa []float32 + var hf []cuvs.Float16 + if u.baseOid == types.T_array_float16 { + h := types.BytesToArray[types.Float16](faVec.GetBytesAt(nthRow)) + if uint(len(h)) != u.idxcfg.CuvsCagra.Dimensions { + return moerr.NewInternalError(proc.Ctx, "vector dimension mismatch") + } + hf = f16ToCuvs(h) + } else { + fa = types.BytesToArray[float32](faVec.GetBytesAt(nthRow)) + if uint(len(fa)) != u.idxcfg.CuvsCagra.Dimensions { + return moerr.NewInternalError(proc.Ctx, "vector dimension mismatch") + } } // Trailing rows below the cuvs threshold route to the CDC tail // (search-side brute-force replay) instead of the cuvs builder. if srcPos >= u.cdcCutoff { - vecCopy := append([]float32(nil), fa...) var incBytes []byte if len(u.filterCols) > 0 { incBytes, err = encodeIncludeRowFromArgVecs(u.filterCols, tf.ctr.argVecs, 3, nthRow) @@ -430,50 +468,38 @@ func (u *cagraCreateState) start(tf *TableFunction, proc *process.Process, nthRo return err } } + // Buffer the tail row as raw native base-type bytes so a vecf16 base is + // stored as half (2 bytes/elem) in the CDC record — no f32 detour. + var vecBytes []byte + if u.baseOid == types.T_array_float16 { + vecBytes = append([]byte(nil), util.UnsafeSliceToBytes(hf)...) + } else { + vecBytes = append([]byte(nil), util.UnsafeSliceToBytes(fa)...) + } u.cdcTail = append(u.cdcTail, cuvscdc.PendingRecord{ Pkid: id, - Vec: vecCopy, + Vec: vecBytes, Include: incBytes, }) return nil } - switch { - case u.buildf32 != nil: - err = u.buildf32.AddFloat(id, fa) - case u.buildf16 != nil: - err = u.buildf16.AddFloat(id, fa) - case u.buildi8 != nil: - err = u.buildi8.AddFloat(id, fa) - case u.buildui8 != nil: - err = u.buildui8.AddFloat(id, fa) + // Pass the vector as raw base-type bytes (f32 base -> fa, f16 base -> hf), + // reinterpreted with UnsafeSliceToBytes (zero-copy); the concrete + // CagraBuild[B,Q] casts them back to its own []B/[]Q. No per-row alloc. + vecBytes := util.UnsafeSliceToBytes(fa) + if u.baseOid == types.T_array_float16 { + vecBytes = util.UnsafeSliceToBytes(hf) } - if err != nil { + if err = u.builder.AddRow(id, vecBytes); err != nil { return err } // ---- per-row: append filter column values (if any) ---- if len(u.filterCols) > 0 { - if err = appendFilterRow(u.activeBuilder(), u.filterCols, tf.ctr.argVecs, 3, nthRow); err != nil { + if err = appendFilterRow(u.builder, u.filterCols, tf.ctr.argVecs, 3, nthRow); err != nil { return err } } return nil } - -// activeBuilder returns whichever quantization-specialised builder is live, -// exposed through the narrow filterColumnBuilder interface. Exactly one of -// the four fields is non-nil after a successful NewCagraBuild dispatch. -func (u *cagraCreateState) activeBuilder() filterColumnBuilder { - switch { - case u.buildf32 != nil: - return u.buildf32 - case u.buildf16 != nil: - return u.buildf16 - case u.buildi8 != nil: - return u.buildi8 - case u.buildui8 != nil: - return u.buildui8 - } - return nil -} diff --git a/pkg/sql/colexec/table_function/cagra_search_gpu.go b/pkg/sql/colexec/table_function/cagra_search_gpu.go index dc9771bc819c5..09a075c4b65d4 100644 --- a/pkg/sql/colexec/table_function/cagra_search_gpu.go +++ b/pkg/sql/colexec/table_function/cagra_search_gpu.go @@ -60,15 +60,27 @@ func newCagraAlgoFn(idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTabl // test-only: mirror the build-side device simulation so search loads the same // SHARDED / REPLICATED topology. No-op when gpu_multi_simulation < 2. devices = vectorindex.SimulateDevices(devices, tblcfg.GpuMultiSimulation) - switch metric.QuantizationType(idxcfg.CuvsCagra.Quantization) { + // Dispatch on (base type B, storage type Q): indices store Q, overflow is B. + q := metric.QuantizationType(idxcfg.CuvsCagra.Quantization) + if types.T(tblcfg.KeyPartType) == types.T_array_float16 { + switch q { + case metric.Quantization_INT8: + return cagraPkg.NewCagraSearch[cuvs.Float16, int8](idxcfg, tblcfg, devices) + case metric.Quantization_UINT8: + return cagraPkg.NewCagraSearch[cuvs.Float16, uint8](idxcfg, tblcfg, devices) + default: // F16 (direct) + return cagraPkg.NewCagraSearch[cuvs.Float16, cuvs.Float16](idxcfg, tblcfg, devices) + } + } + switch q { case metric.Quantization_F16: - return cagraPkg.NewCagraSearch[cuvs.Float16](idxcfg, tblcfg, devices) + return cagraPkg.NewCagraSearch[float32, cuvs.Float16](idxcfg, tblcfg, devices) case metric.Quantization_INT8: - return cagraPkg.NewCagraSearch[int8](idxcfg, tblcfg, devices) + return cagraPkg.NewCagraSearch[float32, int8](idxcfg, tblcfg, devices) case metric.Quantization_UINT8: - return cagraPkg.NewCagraSearch[uint8](idxcfg, tblcfg, devices) + return cagraPkg.NewCagraSearch[float32, uint8](idxcfg, tblcfg, devices) default: // Quantization_F32 and unknown - return cagraPkg.NewCagraSearch[float32](idxcfg, tblcfg, devices) + return cagraPkg.NewCagraSearch[float32, float32](idxcfg, tblcfg, devices) } } @@ -213,6 +225,24 @@ func (u *cagraSearchState) start(tf *TableFunction, proc *process.Process, nthRo u.idxcfg.CuvsCagra.Dimensions = uint(faVec.GetType().Width) u.idxcfg.Type = vectorindex.CAGRA + // The query vector type must equal the index's base column type. The + // planner pushdown normally forces this, but the table function has no + // other guard: without it a mismatched query (e.g. a vecf16 query against + // an f32-base/f32-storage index) would drive the f32->f16 storage override + // below and runCagraSearchHalf off the QUERY type and deserialize the + // on-disk index with the wrong storage type. Mirrors the CPU ivf_search guard. + if int32(faVec.GetType().Oid) != u.tblcfg.KeyPartType { + return moerr.NewInvalidInput(proc.Ctx, "query vector type does not match the index base column type") + } + + // A vecf16 base with no QUANTIZATION stores natively as half: derive the + // storage qtype from the (f16) base type so newCagraAlgo dispatches + // NewCagraSearch[cuvs.Float16]. (vecf16 + QUANTIZATION keeps int8/uint8.) + if types.T(u.tblcfg.KeyPartType) == types.T_array_float16 && + metric.QuantizationType(u.idxcfg.CuvsCagra.Quantization) == metric.Quantization_F32 { + u.idxcfg.CuvsCagra.Quantization = uint16(metric.Quantization_F16) + } + u.batch = tf.createResultBatch() u.inited = true } @@ -242,6 +272,13 @@ func (u *cagraSearchState) start(tf *TableFunction, proc *process.Process, nthRo veccache.Cache.Once() + // A vecf16 query is decoded natively to half. CagraSearch.Search dispatches: + // f16-direct (T==Float16) searches the half index natively; a quantized + // f16->int8/uint8 index quantizes the half query to T via the half quantizer. + if faVec.GetType().Oid == types.T_array_float16 { + return runCagraSearchHalf(proc, u, faVec, nthRow) + } + return runCagraSearch[float32](proc, u, faVec, nthRow) } @@ -250,7 +287,20 @@ func runCagraSearch[T types.RealNumbers](proc *process.Process, u *cagraSearchSt if uint(len(fa)) != u.idxcfg.CuvsCagra.Dimensions { return moerr.NewInvalidInput(proc.Ctx, fmt.Sprintf("vector ops between different dimensions (%d, %d) is not permitted.", u.idxcfg.CuvsCagra.Dimensions, len(fa))) } + return cagraRunSearchQuery(proc, u, fa) +} + +// runCagraSearchHalf decodes a vecf16 query natively to []cuvs.Float16 (no f32 +// detour) for a half-storage index. +func runCagraSearchHalf(proc *process.Process, u *cagraSearchState, faVec *vector.Vector, nthRow int) (err error) { + h := types.BytesToArray[types.Float16](faVec.GetBytesAt(nthRow)) + if uint(len(h)) != u.idxcfg.CuvsCagra.Dimensions { + return moerr.NewInvalidInput(proc.Ctx, fmt.Sprintf("vector ops between different dimensions (%d, %d) is not permitted.", u.idxcfg.CuvsCagra.Dimensions, len(h))) + } + return cagraRunSearchQuery(proc, u, f16ToCuvs(h)) +} +func cagraRunSearchQuery(proc *process.Process, u *cagraSearchState, fa any) (err error) { algo := newCagraAlgo(u.idxcfg, u.tblcfg) rt := vectorindex.RuntimeConfig{ diff --git a/pkg/sql/colexec/table_function/ivf_create.go b/pkg/sql/colexec/table_function/ivf_create.go index 7106395a7b29b..820bbc6ef5f3e 100644 --- a/pkg/sql/colexec/table_function/ivf_create.go +++ b/pkg/sql/colexec/table_function/ivf_create.go @@ -36,6 +36,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/vectorindex/ivfflat/kmeans/device" ivfflatrt "github.com/matrixorigin/matrixone/pkg/vectorindex/ivfflat/plugin/runtime" "github.com/matrixorigin/matrixone/pkg/vectorindex/metric" + "github.com/matrixorigin/matrixone/pkg/vectorindex/quantizer" "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" "github.com/matrixorigin/matrixone/pkg/vm" "github.com/matrixorigin/matrixone/pkg/vm/process" @@ -153,6 +154,32 @@ func clustering[T types.RealNumbers](u *ivfCreateState, tf *TableFunction, proc } logutil.Infof("IVFFLAT END: After Kmeans clustering, insert centroids to table") + // int8/uint8 QUANTIZATION (cuVS-style asymmetric scalar quantizer): train + // [min,max] over the sample and persist them in the metadata table. Entries + // (build) and the query (search) map [min,max] onto the full int8 range + // [-128,127] (or uint8 [0,255]) with the same q(x)=round(x*mul+add). Using both + // bounds (not a symmetric scale) uses the whole range for offset data. The + // percentile bound-training is identical for int8 and uint8 (only the target + // range differs, applied later by compile/search). (bf16/float16 are float + // formats and need none.) + if qt, ok := quantizer.ToVectorType(u.param.Quantization); ok && (qt == types.T_array_int8 || qt == types.T_array_uint8) { + qmin, qmax := quantizer.TrainInt8(data) + insSQL := fmt.Sprintf( + "INSERT INTO `%s`.`%s` (`%s`, `%s`) VALUES ('%s', '%.9g'), ('%s', '%.9g') "+ + "ON DUPLICATE KEY UPDATE `%s` = VALUES(`%s`)", + u.tblcfg.DbName, u.tblcfg.MetadataTable, + catalog.SystemSI_IVFFLAT_TblCol_Metadata_key, catalog.SystemSI_IVFFLAT_TblCol_Metadata_val, + catalog.SystemSI_IVFFLAT_Metadata_QuantizeMin, qmin, + catalog.SystemSI_IVFFLAT_Metadata_QuantizeMax, qmax, + catalog.SystemSI_IVFFLAT_TblCol_Metadata_val, catalog.SystemSI_IVFFLAT_TblCol_Metadata_val) + res, err := ivf_runSql(sqlexec.NewSqlProcess(proc), insSQL) + if err != nil { + return err + } + res.Close() + logutil.Infof("IVFFLAT: int8 quantizer min=%g max=%g stored", qmin, qmax) + } + return nil } @@ -329,10 +356,12 @@ func (u *ivfCreateState) start(tf *TableFunction, proc *process.Process, nthRow return moerr.NewInvalidInput(proc.Ctx, "Second argument (vector must be a vecf32 or vecf64 type") } - if embedvec.GetType().Oid == types.T_array_float32 { - u.data32 = make([][]float32, 0, u.nsample) - } else { + // kmeans always clusters in float32 (or float64). Narrow input types + // (bf16/f16/int8) decode to float32 -> data32; only native float64 uses data64. + if embedvec.GetType().Oid == types.T_array_float64 { u.data64 = make([][]float64, 0, u.nsample) + } else { + u.data32 = make([][]float32, 0, u.nsample) } // dimension @@ -343,20 +372,32 @@ func (u *ivfCreateState) start(tf *TableFunction, proc *process.Process, nthRow for _, bat := range res.Batches { evec := bat.Vecs[0] for i := 0; i < bat.RowCount(); i++ { + var f32a []float32 switch evec.GetType().Oid { case types.T_array_float32: - f32a := types.BytesToArray[float32](evec.GetBytesAt(i)) - if uint(len(f32a)) != u.idxcfg.Ivfflat.Dimensions { - return moerr.NewInternalError(proc.Ctx, "vector dimension mismatch") - } - u.data32 = append(u.data32, append(make([]float32, 0, len(f32a)), f32a...)) + f32a = types.BytesToArray[float32](evec.GetBytesAt(i)) case types.T_array_float64: f64a := types.BytesToArray[float64](evec.GetBytesAt(i)) if uint(len(f64a)) != u.idxcfg.Ivfflat.Dimensions { return moerr.NewInternalError(proc.Ctx, "vector dimension mismatch") } u.data64 = append(u.data64, append(make([]float64, 0, len(f64a)), f64a...)) + continue + case types.T_array_bf16: + f32a = types.BF16ToFloat32Slice(types.BytesToArray[types.BF16](evec.GetBytesAt(i))) + case types.T_array_float16: + f32a = types.Float16ToFloat32Slice(types.BytesToArray[types.Float16](evec.GetBytesAt(i))) + case types.T_array_int8: + f32a = types.Int8ToFloat32Slice(types.BytesToArray[int8](evec.GetBytesAt(i))) + case types.T_array_uint8: + f32a = types.Uint8ToFloat32Slice(types.BytesToArray[uint8](evec.GetBytesAt(i))) + default: + return moerr.NewInternalError(proc.Ctx, "unsupported ivfflat vector type") + } + if uint(len(f32a)) != u.idxcfg.Ivfflat.Dimensions { + return moerr.NewInternalError(proc.Ctx, "vector dimension mismatch") } + u.data32 = append(u.data32, append(make([]float32, 0, len(f32a)), f32a...)) } } diff --git a/pkg/sql/colexec/table_function/ivf_search.go b/pkg/sql/colexec/table_function/ivf_search.go index f094a10c8b919..9104c8d525a0f 100644 --- a/pkg/sql/colexec/table_function/ivf_search.go +++ b/pkg/sql/colexec/table_function/ivf_search.go @@ -30,6 +30,7 @@ import ( veccache "github.com/matrixorigin/matrixone/pkg/vectorindex/cache" "github.com/matrixorigin/matrixone/pkg/vectorindex/ivfflat" "github.com/matrixorigin/matrixone/pkg/vectorindex/metric" + "github.com/matrixorigin/matrixone/pkg/vectorindex/quantizer" "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" "github.com/matrixorigin/matrixone/pkg/vm" "github.com/matrixorigin/matrixone/pkg/vm/process" @@ -58,13 +59,21 @@ var ( ) func newIvfAlgoFn(idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig) (veccache.VectorIndexSearchIf, error) { - switch idxcfg.Ivfflat.VectorType { + // The centroid search index is typed by the CENTROID storage type, not the + // entry/input type. For narrow entries the centroids are f32 (decoupled), so + // this returns IvfflatSearch[float32]. CentroidType == 0 (old indexes) means + // "same as entry". + ct := idxcfg.Ivfflat.CentroidType + if ct == 0 { + ct = idxcfg.Ivfflat.VectorType + } + switch ct { case int32(types.T_array_float32): return ivfflat.NewIvfflatSearch[float32](idxcfg, tblcfg), nil case int32(types.T_array_float64): return ivfflat.NewIvfflatSearch[float64](idxcfg, tblcfg), nil default: - return nil, moerr.NewInternalErrorNoCtx("newIvfAlgoFn: invalid vector type") + return nil, moerr.NewInternalErrorNoCtx("newIvfAlgoFn: invalid centroid type") } } @@ -204,7 +213,26 @@ func (u *ivfSearchState) start(tf *TableFunction, proc *process.Process, nthRow return err } u.idxcfg.Ivfflat.Version = version // version from meta table - u.idxcfg.Ivfflat.VectorType = u.tblcfg.KeyPartType // array float32 or array float64 + u.idxcfg.Ivfflat.VectorType = u.tblcfg.KeyPartType // entry/input type + // Centroid type is decoupled: f32 for narrow entries (must match the f32 + // centroid hidden table from schema.go), else same as the entry type. + switch types.T(u.tblcfg.KeyPartType) { + case types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: + u.idxcfg.Ivfflat.CentroidType = int32(types.T_array_float32) + default: + u.idxcfg.Ivfflat.CentroidType = u.tblcfg.KeyPartType + } + // QUANTIZATION: entries are stored as the quantization (down-cast) type, + // independent of the base column. The centroids are forced to f32 (decoupled + // — accurate assignment, fast f32 search) for ANY base type, including f64; + // the query is decoded to f32 for the centroid search and to the entry type + // for the re-rank. VectorType = the entry/quantization type. + if u.param.Quantization != "" { + if qt, ok := quantizer.ToVectorType(u.param.Quantization); ok { + u.idxcfg.Ivfflat.VectorType = int32(qt) + u.idxcfg.Ivfflat.CentroidType = int32(types.T_array_float32) + } + } u.batch = tf.createResultBatch() u.inited = true @@ -223,22 +251,56 @@ func (u *ivfSearchState) start(tf *TableFunction, proc *process.Process, nthRow faVec := tf.ctr.argVecs[1] - switch faVec.GetType().Oid { - case types.T_array_float32: - return runIvfSearchVector[float32](tf, u, proc, faVec, nthRow) - case types.T_array_float64: + // Dispatch on the CENTROID type, not the base type. Only a plain f64 index + // (f64 base, no quantization) keeps f64 centroids; every other case — f32 base, + // narrow base, or any base under QUANTIZATION — searches f32 centroids, so the + // query is decoded to float32 regardless of its column type. + if u.idxcfg.Ivfflat.CentroidType == int32(types.T_array_float64) { return runIvfSearchVector[float64](tf, u, proc, faVec, nthRow) - default: - return moerr.NewInternalError(proc.Ctx, "vector is not array_float32 or array_float64") } + return runIvfSearchVectorToF32(tf, u, proc, faVec, nthRow) } func runIvfSearchVector[T types.RealNumbers](tf *TableFunction, u *ivfSearchState, proc *process.Process, faVec *vector.Vector, nthRow int) (err error) { if faVec.IsNull(uint64(nthRow)) { return nil } + return runIvfSearchQuery(tf, u, proc, types.BytesToArray[T](faVec.GetBytesAt(nthRow))) +} + +// runIvfSearchVectorToF32 decodes the query (of any vector column type: f32, f64, +// or narrow bf16/f16/int8) to float32 and runs the float32 centroid search. The +// SQL re-rank then encodes the query in the entry/quantization type. +func runIvfSearchVectorToF32(tf *TableFunction, u *ivfSearchState, proc *process.Process, faVec *vector.Vector, nthRow int) error { + if faVec.IsNull(uint64(nthRow)) { + return nil + } + b := faVec.GetBytesAt(nthRow) + var fa []float32 + switch faVec.GetType().Oid { + case types.T_array_float32: + fa = types.BytesToArray[float32](b) + case types.T_array_float64: + f64 := types.BytesToArray[float64](b) + fa = make([]float32, len(f64)) + for i, x := range f64 { + fa[i] = float32(x) + } + case types.T_array_bf16: + fa = types.BF16ToFloat32Slice(types.BytesToArray[types.BF16](b)) + case types.T_array_float16: + fa = types.Float16ToFloat32Slice(types.BytesToArray[types.Float16](b)) + case types.T_array_int8: + fa = types.Int8ToFloat32Slice(types.BytesToArray[int8](b)) + case types.T_array_uint8: + fa = types.Uint8ToFloat32Slice(types.BytesToArray[uint8](b)) + default: + return moerr.NewInternalError(proc.Ctx, "unsupported ivfflat vector type") + } + return runIvfSearchQuery(tf, u, proc, fa) +} - fa := types.BytesToArray[T](faVec.GetBytesAt(nthRow)) +func runIvfSearchQuery[T types.RealNumbers](tf *TableFunction, u *ivfSearchState, proc *process.Process, fa []T) (err error) { if uint(len(fa)) != u.idxcfg.Ivfflat.Dimensions { return moerr.NewInvalidInput(proc.Ctx, fmt.Sprintf("vector ops between different dimensions (%d, %d) is not permitted.", u.idxcfg.Ivfflat.Dimensions, len(fa))) } diff --git a/pkg/sql/colexec/table_function/ivfpq_create_gpu.go b/pkg/sql/colexec/table_function/ivfpq_create_gpu.go index b748951e56245..d8a28bc49e67f 100644 --- a/pkg/sql/colexec/table_function/ivfpq_create_gpu.go +++ b/pkg/sql/colexec/table_function/ivfpq_create_gpu.go @@ -20,9 +20,11 @@ import ( "fmt" "strconv" "time" + "unsafe" "github.com/bytedance/sonic" "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" @@ -48,16 +50,45 @@ var ivfpqCatalogHooks = ivfpqrt.CatalogHooks{} var ivfpq_runSql = sqlexec.RunSql +// f16ToCuvs reinterprets a []types.Float16 as []cuvs.Float16. Both are uint16 +// with identical layout; this is a zero-copy view (the caller does not retain it +// past the GPU add, which copies to device). Shared by the ivfpq/cagra GPU +// table functions for the native f16 (half) path. +func f16ToCuvs(s []types.Float16) []cuvs.Float16 { + if len(s) == 0 { + return nil + } + return unsafe.Slice((*cuvs.Float16)(unsafe.Pointer(&s[0])), len(s)) +} + +// ivfpqBuilder is the (B, Q)-erased build interface the create state drives. +// *ivfpqPkg.IvfpqBuild[B, Q] satisfies it for every wired (base, storage) +// combo. GetIndexes is [B,Q]-typed and intentionally NOT on the interface — +// end() routes through ToInsertSql instead. +type ivfpqBuilder interface { + // AddRow takes the raw base-type bytes of one vector (4*dim for an f32 base, + // 2*dim for an f16 base); the concrete builder reinterprets them to its + // []B/[]Q with UnsafeSliceCast (the interface can't name B). Passing []byte + // rather than `any` keeps the per-row build hot path allocation-free. + AddRow(id int64, vecBytes []byte) error + SetFilterColumns(colMetaJSON string) + AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error + ToInsertSql(ts int64) ([]string, error) + Destroy() error +} + type ivfpqCreateState struct { - inited bool - buildf32 *ivfpqPkg.IvfpqBuild[float32] - buildf16 *ivfpqPkg.IvfpqBuild[cuvs.Float16] - buildi8 *ivfpqPkg.IvfpqBuild[int8] - buildui8 *ivfpqPkg.IvfpqBuild[uint8] - param vectorindex.IvfpqParam - tblcfg vectorindex.IndexTableConfig - idxcfg vectorindex.IndexConfig - offset int + inited bool + builder ivfpqBuilder + param vectorindex.IvfpqParam + tblcfg vectorindex.IndexTableConfig + idxcfg vectorindex.IndexConfig + offset int + + // baseOid is the base (source) vector column element type — f32 or f16. + // The storage/quantization type (which builder is non-nil) may differ: + // f16 base is stored as half (direct) or quantized to int8/uint8. + baseOid types.T // filterCols is the INCLUDE column metadata derived at start() from // param.IncludedColumns (names) + argVecs[3:] (types). Empty when the @@ -72,7 +103,9 @@ type ivfpqCreateState struct { // records under vectorindex.CdcTailId. cdcCutoff int64 rowsSeen int64 - cdcTail []cuvscdc.PendingRecord + // CDC tail records, with each vector stored as raw native base-type bytes + // (f16 stays 2-byte — no f32 widening). vecBytesPerRow = dim * base elem size. + cdcTail []cuvscdc.PendingRecord // srcEmpty short-circuits the per-row code when SELECT COUNT(*) // at init time returned zero — nothing to build, nothing to CDC. @@ -93,19 +126,11 @@ func (u *ivfpqCreateState) end(tf *TableFunction, proc *process.Process) error { ) ts := time.Now().UnixMicro() - switch { - case u.buildf32 != nil: - sqls, err = u.buildf32.ToInsertSql(ts) - case u.buildf16 != nil: - sqls, err = u.buildf16.ToInsertSql(ts) - case u.buildi8 != nil: - sqls, err = u.buildi8.ToInsertSql(ts) - case u.buildui8 != nil: - sqls, err = u.buildui8.ToInsertSql(ts) - default: - // No builder selected → init didn't set one. Nothing to do for - // the cuvs side; the CDC tail (if any) below still emits. + if u.builder != nil { + sqls, err = u.builder.ToInsertSql(ts) } + // No builder selected → init didn't set one. Nothing to do for the cuvs + // side; the CDC tail (if any) below still emits. if err != nil { return err } @@ -119,9 +144,14 @@ func (u *ivfpqCreateState) end(tf *TableFunction, proc *process.Process) error { // record 0. Search-side can recover the INCLUDE-column layout // for tag=1 replay even when no tag=0 sub-index exists. colMetaJSON := colMetaJSONFromCols(u.filterCols) + // vecBytesPerRow = dim * base element size (2 for vecf16, else 4). + elemSize := 4 + if u.baseOid == types.T_array_float16 { + elemSize = 2 + } + vecBytesPerRow := int(u.idxcfg.CuvsIvfpq.Dimensions) * elemSize tailSqls, err := cuvscdc.SaveSmallTailAsCdc( - u.tblcfg, u.cdcTail, - int(u.idxcfg.CuvsIvfpq.Dimensions), ibpr, colMetaJSON) + u.tblcfg, u.cdcTail, vecBytesPerRow, ibpr, colMetaJSON) if err != nil { return err } @@ -158,17 +188,8 @@ func (u *ivfpqCreateState) free(tf *TableFunction, proc *process.Process, pipeli if u.batch != nil { u.batch.Clean(proc.Mp()) } - if u.buildf32 != nil { - u.buildf32.Destroy() - } - if u.buildf16 != nil { - u.buildf16.Destroy() - } - if u.buildi8 != nil { - u.buildi8.Destroy() - } - if u.buildui8 != nil { - u.buildui8.Destroy() + if u.builder != nil { + u.builder.Destroy() } } @@ -350,7 +371,16 @@ func (u *ivfpqCreateState) start(tf *TableFunction, proc *process.Process, nthRo faVec := tf.ctr.argVecs[2] if !catalogplugin.SupportsVectorType(ivfpqCatalogHooks, faVec.GetType().Oid) { - return moerr.NewInvalidInput(proc.Ctx, "third argument (vector) must be a float32 array") + return moerr.NewInvalidInput(proc.Ctx, "third argument (vector) must be a float32 / float16 array") + } + u.baseOid = faVec.GetType().Oid + + // Derive the storage qtype from the base column type when no QUANTIZATION + // was given: a vecf16 base with no quantization is stored natively as half. + // (vecf16 + QUANTIZATION=int8/uint8 keeps qt = int8/uint8 — quantize path.) + if u.baseOid == types.T_array_float16 && qt == metric.Quantization_F32 { + qt = metric.Quantization_F16 + u.idxcfg.CuvsIvfpq.Quantization = uint16(qt) } // dimension @@ -367,15 +397,25 @@ func (u *ivfpqCreateState) start(tf *TableFunction, proc *process.Process, nthRo uid := fmt.Sprintf("%s:%d:%d", tf.CnAddr, tf.MaxParallel, tf.ParallelID) // ---- create builder ---- - switch qt { - case metric.Quantization_F16: - u.buildf16, err = ivfpqPkg.NewIvfpqBuild[cuvs.Float16](uid, u.idxcfg, u.tblcfg, nthread, devices) - case metric.Quantization_INT8: - u.buildi8, err = ivfpqPkg.NewIvfpqBuild[int8](uid, u.idxcfg, u.tblcfg, nthread, devices) - case metric.Quantization_UINT8: - u.buildui8, err = ivfpqPkg.NewIvfpqBuild[uint8](uid, u.idxcfg, u.tblcfg, nthread, devices) + // One real [B, Q] builder keyed on (base column type, storage qtype). + // The 7 wired combos: f32 base × {f32, f16, int8, uint8}; f16 base × + // {f16, int8, uint8}. + isF16Base := u.baseOid == types.T_array_float16 + switch { + case isF16Base && qt == metric.Quantization_F16: + u.builder, err = ivfpqPkg.NewIvfpqBuild[cuvs.Float16, cuvs.Float16](uid, u.idxcfg, u.tblcfg, nthread, devices) + case isF16Base && qt == metric.Quantization_INT8: + u.builder, err = ivfpqPkg.NewIvfpqBuild[cuvs.Float16, int8](uid, u.idxcfg, u.tblcfg, nthread, devices) + case isF16Base && qt == metric.Quantization_UINT8: + u.builder, err = ivfpqPkg.NewIvfpqBuild[cuvs.Float16, uint8](uid, u.idxcfg, u.tblcfg, nthread, devices) + case qt == metric.Quantization_F16: + u.builder, err = ivfpqPkg.NewIvfpqBuild[float32, cuvs.Float16](uid, u.idxcfg, u.tblcfg, nthread, devices) + case qt == metric.Quantization_INT8: + u.builder, err = ivfpqPkg.NewIvfpqBuild[float32, int8](uid, u.idxcfg, u.tblcfg, nthread, devices) + case qt == metric.Quantization_UINT8: + u.builder, err = ivfpqPkg.NewIvfpqBuild[float32, uint8](uid, u.idxcfg, u.tblcfg, nthread, devices) default: - u.buildf32, err = ivfpqPkg.NewIvfpqBuild[float32](uid, u.idxcfg, u.tblcfg, nthread, devices) + u.builder, err = ivfpqPkg.NewIvfpqBuild[float32, float32](uid, u.idxcfg, u.tblcfg, nthread, devices) } if err != nil { return err @@ -390,7 +430,7 @@ func (u *ivfpqCreateState) start(tf *TableFunction, proc *process.Process, nthRo if len(u.filterCols) > 0 { logutil.Infof("IVFPQ create: INCLUDE columns = %v (from %d arg vectors)", u.filterCols, len(tf.ctr.argVecs)-3) - if err = initFilterColumns(u.activeBuilder(), u.filterCols); err != nil { + if err = initFilterColumns(u.builder, u.filterCols); err != nil { return err } } @@ -422,17 +462,33 @@ func (u *ivfpqCreateState) start(tf *TableFunction, proc *process.Process, nthRo u.rowsSeen++ id := vector.GetFixedAtNoTypeCheck[int64](tf.ctr.argVecs[1], nthRow) - fa := types.BytesToArray[float32](faVec.GetBytesAt(nthRow)) - if uint(len(fa)) != u.idxcfg.CuvsIvfpq.Dimensions { - return moerr.NewInternalError(proc.Ctx, "vector dimension mismatch") + // Decode the base vector to its native type. f32 base -> []float32 (used by + // the f32 path and the CDC tail). f16 base -> native []cuvs.Float16 for the + // direct (half-storage) add; the CDC tail still transports f32 (exact widen) + // until the CDC pipeline is made native (step 5). + var fa []float32 + var hf []cuvs.Float16 + if u.baseOid == types.T_array_float16 { + h := types.BytesToArray[types.Float16](faVec.GetBytesAt(nthRow)) + if uint(len(h)) != u.idxcfg.CuvsIvfpq.Dimensions { + return moerr.NewInternalError(proc.Ctx, "vector dimension mismatch") + } + hf = f16ToCuvs(h) + if srcPos >= u.cdcCutoff { + fa = types.Float16ToFloat32Slice(h) + } + } else { + fa = types.BytesToArray[float32](faVec.GetBytesAt(nthRow)) + if uint(len(fa)) != u.idxcfg.CuvsIvfpq.Dimensions { + return moerr.NewInternalError(proc.Ctx, "vector dimension mismatch") + } } // Trailing rows below the cuvs k-means threshold (lists) route to // the CDC tail (search-side brute-force replay) instead of the // cuvs builder. if srcPos >= u.cdcCutoff { - vecCopy := append([]float32(nil), fa...) var incBytes []byte if len(u.filterCols) > 0 { incBytes, err = encodeIncludeRowFromArgVecs(u.filterCols, tf.ctr.argVecs, 3, nthRow) @@ -440,48 +496,37 @@ func (u *ivfpqCreateState) start(tf *TableFunction, proc *process.Process, nthRo return err } } + // Buffer the tail row as raw native base-type bytes so a vecf16 base is + // stored as half (2 bytes/elem) in the CDC record — no f32 detour. + var vecBytes []byte + if u.baseOid == types.T_array_float16 { + vecBytes = append([]byte(nil), util.UnsafeSliceToBytes(hf)...) + } else { + vecBytes = append([]byte(nil), util.UnsafeSliceToBytes(fa)...) + } u.cdcTail = append(u.cdcTail, cuvscdc.PendingRecord{ Pkid: id, - Vec: vecCopy, + Vec: vecBytes, Include: incBytes, }) return nil } - switch { - case u.buildf32 != nil: - err = u.buildf32.AddFloat(id, fa) - case u.buildf16 != nil: - err = u.buildf16.AddFloat(id, fa) - case u.buildi8 != nil: - err = u.buildi8.AddFloat(id, fa) - case u.buildui8 != nil: - err = u.buildui8.AddFloat(id, fa) + // Pass the vector as raw base-type bytes (f32 base -> fa, f16 base -> hf), + // reinterpreted with UnsafeSliceToBytes (zero-copy); the concrete + // IvfpqBuild[B,Q] casts them back to its own []B/[]Q. No per-row alloc. + vecBytes := util.UnsafeSliceToBytes(fa) + if u.baseOid == types.T_array_float16 { + vecBytes = util.UnsafeSliceToBytes(hf) } - if err != nil { + if err = u.builder.AddRow(id, vecBytes); err != nil { return err } if len(u.filterCols) > 0 { - if err = appendFilterRow(u.activeBuilder(), u.filterCols, tf.ctr.argVecs, 3, nthRow); err != nil { + if err = appendFilterRow(u.builder, u.filterCols, tf.ctr.argVecs, 3, nthRow); err != nil { return err } } return nil } - -// activeBuilder returns the live quantization-specialised builder through the -// filterColumnBuilder interface. See cagraCreateState.activeBuilder. -func (u *ivfpqCreateState) activeBuilder() filterColumnBuilder { - switch { - case u.buildf32 != nil: - return u.buildf32 - case u.buildf16 != nil: - return u.buildf16 - case u.buildi8 != nil: - return u.buildi8 - case u.buildui8 != nil: - return u.buildui8 - } - return nil -} diff --git a/pkg/sql/colexec/table_function/ivfpq_search_gpu.go b/pkg/sql/colexec/table_function/ivfpq_search_gpu.go index 4e74441cb12ea..25c99b654b420 100644 --- a/pkg/sql/colexec/table_function/ivfpq_search_gpu.go +++ b/pkg/sql/colexec/table_function/ivfpq_search_gpu.go @@ -60,15 +60,28 @@ func newIvfpqAlgoFn(idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTabl // test-only: mirror the build-side device simulation so search loads the same // SHARDED / REPLICATED topology. No-op when gpu_multi_simulation < 2. devices = vectorindex.SimulateDevices(devices, tblcfg.GpuMultiSimulation) - switch metric.QuantizationType(idxcfg.CuvsIvfpq.Quantization) { + // Dispatch on (base type B, storage type Q): the main indices store Q (the + // quantization), the CDC/overflow brute force is the base type B (f32/f16). + q := metric.QuantizationType(idxcfg.CuvsIvfpq.Quantization) + if types.T(tblcfg.KeyPartType) == types.T_array_float16 { + switch q { + case metric.Quantization_INT8: + return ivfpqPkg.NewIvfpqSearch[cuvs.Float16, int8](idxcfg, tblcfg, devices) + case metric.Quantization_UINT8: + return ivfpqPkg.NewIvfpqSearch[cuvs.Float16, uint8](idxcfg, tblcfg, devices) + default: // F16 (direct) + return ivfpqPkg.NewIvfpqSearch[cuvs.Float16, cuvs.Float16](idxcfg, tblcfg, devices) + } + } + switch q { case metric.Quantization_F16: - return ivfpqPkg.NewIvfpqSearch[cuvs.Float16](idxcfg, tblcfg, devices) + return ivfpqPkg.NewIvfpqSearch[float32, cuvs.Float16](idxcfg, tblcfg, devices) case metric.Quantization_INT8: - return ivfpqPkg.NewIvfpqSearch[int8](idxcfg, tblcfg, devices) + return ivfpqPkg.NewIvfpqSearch[float32, int8](idxcfg, tblcfg, devices) case metric.Quantization_UINT8: - return ivfpqPkg.NewIvfpqSearch[uint8](idxcfg, tblcfg, devices) + return ivfpqPkg.NewIvfpqSearch[float32, uint8](idxcfg, tblcfg, devices) default: - return ivfpqPkg.NewIvfpqSearch[float32](idxcfg, tblcfg, devices) + return ivfpqPkg.NewIvfpqSearch[float32, float32](idxcfg, tblcfg, devices) } } @@ -216,6 +229,24 @@ func (u *ivfpqSearchState) start(tf *TableFunction, proc *process.Process, nthRo u.idxcfg.CuvsIvfpq.Dimensions = uint(faVec.GetType().Width) u.idxcfg.Type = vectorindex.IVFPQ + // The query vector type must equal the index's base column type. The + // planner pushdown normally forces this, but the table function has no + // other guard: without it a mismatched query (e.g. a vecf16 query against + // an f32-base/f32-storage index) would drive the f32->f16 storage override + // below off the QUERY type and deserialize the on-disk index with the wrong + // storage type. Mirrors the CPU ivf_search guard. + if int32(faVec.GetType().Oid) != u.tblcfg.KeyPartType { + return moerr.NewInvalidInput(proc.Ctx, "query vector type does not match the index base column type") + } + + // A vecf16 base with no QUANTIZATION stores natively as half: derive the + // storage qtype from the (f16) base type so newIvfpqAlgo dispatches + // NewIvfpqSearch[cuvs.Float16]. (vecf16 + QUANTIZATION keeps int8/uint8.) + if types.T(u.tblcfg.KeyPartType) == types.T_array_float16 && + metric.QuantizationType(u.idxcfg.CuvsIvfpq.Quantization) == metric.Quantization_F32 { + u.idxcfg.CuvsIvfpq.Quantization = uint16(metric.Quantization_F16) + } + u.batch = tf.createResultBatch() u.inited = true } @@ -244,6 +275,13 @@ func (u *ivfpqSearchState) start(tf *TableFunction, proc *process.Process, nthRo veccache.Cache.Once() + // A vecf16 query is decoded natively to half. IvfpqSearch.Search dispatches: + // f16-direct (T==Float16) searches the half index natively; a quantized + // f16->int8/uint8 index quantizes the half query to T via the half quantizer. + if faVec.GetType().Oid == types.T_array_float16 { + return runIvfpqSearchHalf(proc, u, faVec, nthRow) + } + return runIvfpqSearch[float32](proc, u, faVec, nthRow) } @@ -252,7 +290,21 @@ func runIvfpqSearch[T types.RealNumbers](proc *process.Process, u *ivfpqSearchSt if uint(len(fa)) != u.idxcfg.CuvsIvfpq.Dimensions { return moerr.NewInvalidInput(proc.Ctx, fmt.Sprintf("vector ops between different dimensions (%d, %d) is not permitted.", u.idxcfg.CuvsIvfpq.Dimensions, len(fa))) } + return ivfpqRunSearchQuery(proc, u, fa) +} + +// runIvfpqSearchHalf decodes a vecf16 query natively to []cuvs.Float16 (no f32 +// detour) for a half-storage index. IvfpqSearch.Search dispatches the native +// half path; a filtered query is half-cast to f32 there (exact). +func runIvfpqSearchHalf(proc *process.Process, u *ivfpqSearchState, faVec *vector.Vector, nthRow int) (err error) { + h := types.BytesToArray[types.Float16](faVec.GetBytesAt(nthRow)) + if uint(len(h)) != u.idxcfg.CuvsIvfpq.Dimensions { + return moerr.NewInvalidInput(proc.Ctx, fmt.Sprintf("vector ops between different dimensions (%d, %d) is not permitted.", u.idxcfg.CuvsIvfpq.Dimensions, len(h))) + } + return ivfpqRunSearchQuery(proc, u, f16ToCuvs(h)) +} +func ivfpqRunSearchQuery(proc *process.Process, u *ivfpqSearchState, fa any) (err error) { algo := newIvfpqAlgo(u.idxcfg, u.tblcfg) rt := vectorindex.RuntimeConfig{ diff --git a/pkg/sql/colexec/table_function/table_function.go b/pkg/sql/colexec/table_function/table_function.go index a8bbca663d34c..0e7537d914a90 100644 --- a/pkg/sql/colexec/table_function/table_function.go +++ b/pkg/sql/colexec/table_function/table_function.go @@ -182,6 +182,12 @@ func (tableFunction *TableFunction) Prepare(proc *process.Process) error { tblArg.ctr.state, err = ivfCreatePrepare(proc, tblArg) case "ivf_search": tblArg.ctr.state, err = ivfSearchPrepare(proc, tblArg) + case "bm25_create": + tblArg.ctr.state, err = bm25CreatePrepare(proc, tblArg) + case "bm25_search": + tblArg.ctr.state, err = bm25SearchPrepare(proc, tblArg) + case "bm25_compact": + tblArg.ctr.state, err = bm25CompactPrepare(proc, tblArg) case "parse_jsonl_data": tblArg.ctr.state, err = parseJsonlDataPrepare(proc, tblArg) case "parse_jsonl_file": diff --git a/pkg/sql/compile/ddl.go b/pkg/sql/compile/ddl.go index 61a4fa7df8664..0c0cc07476374 100644 --- a/pkg/sql/compile/ddl.go +++ b/pkg/sql/compile/ddl.go @@ -517,9 +517,13 @@ func reindexSpecifiedParams(stmt tree.Statement, indexName string) map[string]st addInt(catalog.IndexAlgoParamKmeansTrainPercent, opt.KmeansTrainPercent) addInt(catalog.IndexAlgoParamKmeansMaxIteration, opt.KmeansMaxIteration) addInt(catalog.IndexAlgoParamMaxIndexCapacity, opt.MaxIndexCapacity) - // NOTE: quantization is intentionally NOT handled by reindex. The vecf16 - // branch owns the quantization work (per-backend validity, BF16, ...), so - // reindex neither merges nor rejects it here — revisit once that lands. + // quantization is normalized to lowercase here (matching the CREATE INDEX + // path) so case-sensitive consumers (GPU build switch / quantizer) behave + // identically; the per-backend VALUE check (which names a given algorithm + // accepts) is done in each plugin's ValidateReindexParams. + if opt.Quantization != "" { + m[catalog.Quantization] = catalog.ToLower(opt.Quantization) + } return m } @@ -1026,7 +1030,7 @@ func (s *Scope) AlterTableInplace(c *Compile) error { alterIndex = indexDef indexAlgo := catalog.ToLower(alterIndex.IndexAlgo) - if !indexplugin.IsVectorIndexAlgo(indexAlgo) { + if !indexplugin.IsVectorIndexAlgo(indexAlgo) && indexAlgo != catalog.MoIndexBm25Algo.ToString() { return moerr.NewInternalError(c.proc.Ctx, "invalid index algo type for alter reindex") } // Each algorithm's plugin owns parameter-update @@ -1099,7 +1103,7 @@ func (s *Scope) AlterTableInplace(c *Compile) error { if cctx == nil { cctx = newPluginCompileCtx(s, c, tblId, extra, dbSource, qry.Database, oTableDef, nil) } - err = p.Compile().HandleReindex(cctx, multiTableIndex.IndexDefs, tableAlterIndex.ForceSync) + err = p.Compile().HandleReindex(cctx, multiTableIndex.IndexDefs, tableAlterIndex.ForceSync, tableAlterIndex.Merge) } if err != nil { @@ -2558,6 +2562,38 @@ func (s *Scope) DropIndex(c *Compile) error { return err } + //6. Plugin-mediated drop hook — mirrors the HandleCreateIndex dispatch in + // CreateIndex. Vector-index plugins use it to evict their in-process search + // cache for the dropped index, so GPU/host resources are freed NOW instead of + // lingering until the 5-min VectorIndexCacheTTL housekeeping. Without this the + // hook (pkg/vectorindex/*/plugin/compile HandleDropIndex) was never invoked. + dropPluginIndexes := make(map[string]*MultiTableIndex) + for _, idef := range oldTableDef.Indexes { + if idef.IndexName != qry.IndexName || idef.Unique || !indexplugin.IsPluginAlgo(idef.IndexAlgo) { + continue + } + algo := catalog.ToLower(idef.IndexAlgo) + mti, ok := dropPluginIndexes[algo] + if !ok { + mti = &MultiTableIndex{IndexAlgo: algo, IndexDefs: make(map[string]*plan.IndexDef)} + dropPluginIndexes[algo] = mti + } + mti.IndexDefs[catalog.ToLower(idef.IndexAlgoTableType)] = idef + } + if len(dropPluginIndexes) > 0 { + dctx := newPluginCompileCtx(s, c, oldTableDef.TblId, nil, d, qry.Database, oldTableDef, nil) + for _, mti := range dropPluginIndexes { + if p, ok := indexplugin.Get(mti.IndexAlgo); ok { + // Best-effort cleanup: the 5-min TTL is the backstop, so don't + // fail the DROP if cache eviction errors — just log. + if e := p.Compile().HandleDropIndex(dctx, mti.IndexDefs); e != nil { + logutil.Warnf("[plugin] %s HandleDropIndex %s.%s/%s: %v", + mti.IndexAlgo, qry.Database, qry.Table, qry.IndexName, e) + } + } + } + } + return nil } diff --git a/pkg/sql/compile/plugin_context.go b/pkg/sql/compile/plugin_context.go index 729f6c3649876..2f7cab66b55a5 100644 --- a/pkg/sql/compile/plugin_context.go +++ b/pkg/sql/compile/plugin_context.go @@ -15,6 +15,7 @@ package compile import ( + "github.com/matrixorigin/matrixone/pkg/fileservice" compileplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/compile" "github.com/matrixorigin/matrixone/pkg/pb/api" "github.com/matrixorigin/matrixone/pkg/pb/plan" @@ -85,6 +86,27 @@ func (p *pluginCompileCtx) MainTableID() uint64 { return p.mainTabl func (p *pluginCompileCtx) MainExtra() *api.SchemaExtra { return p.mainExtra } func (p *pluginCompileCtx) RunSql(sql string) error { return p.c.runSql(sql) } +// RunWithSourceReadCacheSkip runs fn with SkipMemoryCacheWrites attached to the +// compile context, so block reads performed inside fn — notably an index build's +// source-table scans (kmeans sample + entry assignment) — do NOT populate the +// fileservice memory cache. The build reads the source once; queries never re-read +// it (re-rank fetches only a handful of rows), so caching it would just evict the +// index-entry blocks the queries actually hit. Mirrors the SkipAllCache policy +// compaction (mergeobjects) and LOAD DATA (external) already use for one-shot bulk +// reads. runSqlWithResultAndOptions reads c.proc.Ctx for the sub-execution, so +// attaching the policy here propagates to every read in the build; it is restored +// afterward. The build runs synchronously within this compile, so the temporary +// swap is single-threaded. +func (p *pluginCompileCtx) RunWithSourceReadCacheSkip(fn func() error) error { + if p.c == nil || p.c.proc == nil { + return fn() + } + prev := p.c.proc.Ctx + p.c.proc.Ctx = fileservice.WithFileServicePolicy(prev, fileservice.SkipMemoryCacheWrites) + defer func() { p.c.proc.Ctx = prev }() + return fn() +} + func (p *pluginCompileCtx) BuildIndexTable(def *plan.TableDef) error { return indexTableBuild(p.c, p.mainTableID, p.mainExtra, def, p.dbSource) } diff --git a/pkg/sql/compile/reindex_params_test.go b/pkg/sql/compile/reindex_params_test.go index 77df271471eb0..3ade308193f6c 100644 --- a/pkg/sql/compile/reindex_params_test.go +++ b/pkg/sql/compile/reindex_params_test.go @@ -38,6 +38,7 @@ func TestReindexSpecifiedParams(t *testing.T) { IntermediateGraphDegree: 128, GraphDegree: 64, ITopkSize: 256, + Quantization: "Float16", // mixed case -> normalized to lowercase KmeansTrainPercent: 5, KmeansMaxIteration: 30, MaxIndexCapacity: 2000, @@ -56,6 +57,7 @@ func TestReindexSpecifiedParams(t *testing.T) { catalog.IntermediateGraphDegree: "128", catalog.GraphDegree: "64", catalog.ITopkSize: "256", + catalog.Quantization: "float16", // normalized from "Float16" catalog.IndexAlgoParamKmeansTrainPercent: "5", catalog.IndexAlgoParamKmeansMaxIteration: "30", catalog.IndexAlgoParamMaxIndexCapacity: "2000", diff --git a/pkg/sql/parsers/dialect/mysql/keywords.go b/pkg/sql/parsers/dialect/mysql/keywords.go index 7db298cbb1a6c..b19bfb349f018 100644 --- a/pkg/sql/parsers/dialect/mysql/keywords.go +++ b/pkg/sql/parsers/dialect/mysql/keywords.go @@ -64,6 +64,7 @@ func init() { "ivfflat": IVFFLAT, "ivfpq": IVFPQ, "hnsw": HNSW, + "bm25": BM25, "m": M, "ef_construction": EF_CONSTRUCTION, "ef_search": EF_SEARCH, @@ -690,6 +691,10 @@ func init() { "array": ARRAY, "vecf32": VECF32, "vecf64": VECF64, + "vecbf16": VECBF16, + "vecf16": VECF16, + "vecint8": VECINT8, + "vecuint8": VECUINT8, "backup": BACKUP, "filesystem": FILESYSTEM, "handler": HANDLER, diff --git a/pkg/sql/parsers/dialect/mysql/mysql_sql.go b/pkg/sql/parsers/dialect/mysql/mysql_sql.go index 2339f301c332b..fa710d25d5c46 100644 --- a/pkg/sql/parsers/dialect/mysql/mysql_sql.go +++ b/pkg/sql/parsers/dialect/mysql/mysql_sql.go @@ -272,519 +272,524 @@ const ENUM = 57549 const UUID = 57550 const VECF32 = 57551 const VECF64 = 57552 -const GEOMETRY = 57553 -const POINT = 57554 -const LINESTRING = 57555 -const POLYGON = 57556 -const GEOMETRYCOLLECTION = 57557 -const MULTIPOINT = 57558 -const MULTILINESTRING = 57559 -const MULTIPOLYGON = 57560 -const GEOMETRY32 = 57561 -const GEOGRAPHY = 57562 -const GEOGRAPHY32 = 57563 -const POINT32 = 57564 -const LINESTRING32 = 57565 -const POLYGON32 = 57566 -const GEOMETRYCOLLECTION32 = 57567 -const MULTIPOINT32 = 57568 -const MULTILINESTRING32 = 57569 -const MULTIPOLYGON32 = 57570 -const INT1 = 57571 -const INT2 = 57572 -const INT3 = 57573 -const INT4 = 57574 -const INT8 = 57575 -const S3OPTION = 57576 -const STAGEOPTION = 57577 -const SQL_SMALL_RESULT = 57578 -const SQL_BIG_RESULT = 57579 -const SQL_BUFFER_RESULT = 57580 -const SQL_CALC_FOUND_ROWS = 57581 -const LOW_PRIORITY = 57582 -const HIGH_PRIORITY = 57583 -const DELAYED = 57584 -const CREATE = 57585 -const ALTER = 57586 -const DROP = 57587 -const RENAME = 57588 -const REMOVE = 57589 -const ANALYZE = 57590 -const PHYPLAN = 57591 -const ADD = 57592 -const RETURNS = 57593 -const SCHEMA = 57594 -const TABLE = 57595 -const SEQUENCE = 57596 -const INDEX = 57597 -const VIEW = 57598 -const TO = 57599 -const IGNORE = 57600 -const IF = 57601 -const PRIMARY = 57602 -const COLUMN = 57603 -const CONSTRAINT = 57604 -const SPATIAL = 57605 -const FULLTEXT = 57606 -const FOREIGN = 57607 -const KEY_BLOCK_SIZE = 57608 -const SHOW = 57609 -const DESCRIBE = 57610 -const EXPLAIN = 57611 -const DATE = 57612 -const ESCAPE = 57613 -const REPAIR = 57614 -const OPTIMIZE = 57615 -const TRUNCATE = 57616 -const MAXVALUE = 57617 -const PARTITION = 57618 -const REORGANIZE = 57619 -const LESS = 57620 -const THAN = 57621 -const PROCEDURE = 57622 -const TRIGGER = 57623 -const STATUS = 57624 -const VARIABLES = 57625 -const ROLE = 57626 -const PROXY = 57627 -const AVG_ROW_LENGTH = 57628 -const STORAGE = 57629 -const DISK = 57630 -const MEMORY = 57631 -const CHECKSUM = 57632 -const COMPRESSION = 57633 -const DATA = 57634 -const DIRECTORY = 57635 -const DELAY_KEY_WRITE = 57636 -const ENCRYPTION = 57637 -const ENGINE = 57638 -const MAX_ROWS = 57639 -const MIN_ROWS = 57640 -const PACK_KEYS = 57641 -const ROW_FORMAT = 57642 -const STATS_AUTO_RECALC = 57643 -const STATS_PERSISTENT = 57644 -const STATS_SAMPLE_PAGES = 57645 -const DYNAMIC = 57646 -const COMPRESSED = 57647 -const REDUNDANT = 57648 -const COMPACT = 57649 -const FIXED = 57650 -const COLUMN_FORMAT = 57651 -const AUTO_RANDOM = 57652 -const ENGINE_ATTRIBUTE = 57653 -const SECONDARY_ENGINE_ATTRIBUTE = 57654 -const INSERT_METHOD = 57655 -const RESTRICT = 57656 -const CASCADE = 57657 -const ACTION = 57658 -const PARTIAL = 57659 -const SIMPLE = 57660 -const CHECK = 57661 -const ENFORCED = 57662 -const RANGE = 57663 -const LIST = 57664 -const ALGORITHM = 57665 -const LINEAR = 57666 -const PARTITIONS = 57667 -const SUBPARTITION = 57668 -const SUBPARTITIONS = 57669 -const CLUSTER = 57670 -const TYPE = 57671 -const ANY = 57672 -const SOME = 57673 -const EXTERNAL = 57674 -const LOCALFILE = 57675 -const URL = 57676 -const PREPARE = 57677 -const DEALLOCATE = 57678 -const RESET = 57679 -const EXTENSION = 57680 -const RETENTION = 57681 -const PERIOD = 57682 -const CLONE = 57683 -const BRANCH = 57684 -const LOG = 57685 -const REVERT = 57686 -const REBASE = 57687 -const DIFF = 57688 -const PICK = 57689 -const CONFLICT = 57690 -const CONFLICT_FAIL = 57691 -const CONFLICT_SKIP = 57692 -const CONFLICT_ACCEPT = 57693 -const OUTPUT = 57694 -const SUMMARY = 57695 -const INCREMENT = 57696 -const CYCLE = 57697 -const MINVALUE = 57698 -const PUBLICATION = 57699 -const SUBSCRIPTION = 57700 -const SUBSCRIPTIONS = 57701 -const PUBLICATIONS = 57702 -const SYNC_INTERVAL = 57703 -const SYNC = 57704 -const COVERAGE = 57705 -const CCPR = 57706 -const PROPERTIES = 57707 -const PARSER = 57708 -const VISIBLE = 57709 -const INVISIBLE = 57710 -const BTREE = 57711 -const HASH = 57712 -const RTREE = 57713 -const BSI = 57714 -const IVFFLAT = 57715 -const MASTER = 57716 -const HNSW = 57717 -const CAGRA = 57718 -const IVFPQ = 57719 -const ZONEMAP = 57720 -const LEADING = 57721 -const BOTH = 57722 -const TRAILING = 57723 -const UNKNOWN = 57724 -const LISTS = 57725 -const OP_TYPE = 57726 -const REINDEX = 57727 -const EF_SEARCH = 57728 -const EF_CONSTRUCTION = 57729 -const M = 57730 -const ASYNC = 57731 -const FORCE_SYNC = 57732 -const AUTO_UPDATE = 57733 -const INTERMEDIATE_GRAPH_DEGREE = 57734 -const GRAPH_DEGREE = 57735 -const QUANTIZATION = 57736 -const BITS_PER_CODE = 57737 -const DISTRIBUTION_MODE = 57738 -const ITOPK_SIZE = 57739 -const INCLUDE = 57740 -const KMEANS_TRAIN_PERCENT = 57741 -const KMEANS_MAX_ITERATION = 57742 -const MAX_INDEX_CAPACITY = 57743 -const EXPIRE = 57744 -const ACCOUNT = 57745 -const ACCOUNTS = 57746 -const UNLOCK = 57747 -const DAY = 57748 -const NEVER = 57749 -const PUMP = 57750 -const MYSQL_COMPATIBILITY_MODE = 57751 -const UNIQUE_CHECK_ON_AUTOINCR = 57752 -const MODIFY = 57753 -const CHANGE = 57754 -const SECOND = 57755 -const ASCII = 57756 -const COALESCE = 57757 -const COLLATION = 57758 -const HOUR = 57759 -const MICROSECOND = 57760 -const MINUTE = 57761 -const MONTH = 57762 -const QUARTER = 57763 -const REPEAT = 57764 -const REVERSE = 57765 -const ROW_COUNT = 57766 -const WEEK = 57767 -const REVOKE = 57768 -const FUNCTION = 57769 -const PRIVILEGES = 57770 -const TABLESPACE = 57771 -const EXECUTE = 57772 -const SUPER = 57773 -const GRANT = 57774 -const OPTION = 57775 -const REFERENCES = 57776 -const REPLICATION = 57777 -const SLAVE = 57778 -const CLIENT = 57779 -const USAGE = 57780 -const RELOAD = 57781 -const FILE = 57782 -const FILES = 57783 -const TEMPORARY = 57784 -const ROUTINE = 57785 -const EVENT = 57786 -const SHUTDOWN = 57787 -const NULLX = 57788 -const AUTO_INCREMENT = 57789 -const APPROXNUM = 57790 -const ENGINES = 57791 -const LOW_CARDINALITY = 57792 -const AUTOEXTEND_SIZE = 57793 -const ADMIN_NAME = 57794 -const RANDOM = 57795 -const SUSPEND = 57796 -const ATTRIBUTE = 57797 -const HISTORY = 57798 -const REUSE = 57799 -const CURRENT = 57800 -const OPTIONAL = 57801 -const FAILED_LOGIN_ATTEMPTS = 57802 -const PASSWORD_LOCK_TIME = 57803 -const UNBOUNDED = 57804 -const SECONDARY = 57805 -const RESTRICTED = 57806 -const USER = 57807 -const IDENTIFIED = 57808 -const CIPHER = 57809 -const ISSUER = 57810 -const X509 = 57811 -const SUBJECT = 57812 -const SAN = 57813 -const REQUIRE = 57814 -const SSL = 57815 -const NONE = 57816 -const PASSWORD = 57817 -const SHARED = 57818 -const EXCLUSIVE = 57819 -const MAX_QUERIES_PER_HOUR = 57820 -const MAX_UPDATES_PER_HOUR = 57821 -const MAX_CONNECTIONS_PER_HOUR = 57822 -const MAX_USER_CONNECTIONS = 57823 -const FORMAT = 57824 -const VERBOSE = 57825 -const CONNECTION = 57826 -const TRIGGERS = 57827 -const PROFILES = 57828 -const LOAD = 57829 -const INLINE = 57830 -const INFILE = 57831 -const TERMINATED = 57832 -const OPTIONALLY = 57833 -const ENCLOSED = 57834 -const ESCAPED = 57835 -const STARTING = 57836 -const LINES = 57837 -const ROWS = 57838 -const IMPORT = 57839 -const DISCARD = 57840 -const JSONTYPE = 57841 -const MODUMP = 57842 -const OVER = 57843 -const PRECEDING = 57844 -const FOLLOWING = 57845 -const GROUPS = 57846 -const DATABASES = 57847 -const TABLES = 57848 -const SEQUENCES = 57849 -const EXTENDED = 57850 -const FULL = 57851 -const PROCESSLIST = 57852 -const FIELDS = 57853 -const COLUMNS = 57854 -const OPEN = 57855 -const ERRORS = 57856 -const WARNINGS = 57857 -const INDEXES = 57858 -const SCHEMAS = 57859 -const NODE = 57860 -const LOCKS = 57861 -const ROLES = 57862 -const RULE = 57863 -const RULES = 57864 -const TABLE_NUMBER = 57865 -const COLUMN_NUMBER = 57866 -const TABLE_VALUES = 57867 -const TABLE_SIZE = 57868 -const TASKS = 57869 -const RUNS = 57870 -const NAMES = 57871 -const GLOBAL = 57872 -const PERSIST = 57873 -const SESSION = 57874 -const ISOLATION = 57875 -const LEVEL = 57876 -const READ = 57877 -const WRITE = 57878 -const ONLY = 57879 -const REPEATABLE = 57880 -const COMMITTED = 57881 -const UNCOMMITTED = 57882 -const SERIALIZABLE = 57883 -const LOCAL = 57884 -const EVENTS = 57885 -const PLUGINS = 57886 -const CURRENT_TIMESTAMP = 57887 -const DATABASE = 57888 -const CURRENT_TIME = 57889 -const LOCALTIME = 57890 -const LOCALTIMESTAMP = 57891 -const UTC_DATE = 57892 -const UTC_TIME = 57893 -const UTC_TIMESTAMP = 57894 -const REPLACE = 57895 -const CONVERT = 57896 -const SEPARATOR = 57897 -const TIMESTAMPDIFF = 57898 -const TIMESTAMPADD = 57899 -const CURRENT_DATE = 57900 -const CURRENT_USER = 57901 -const CURRENT_ROLE = 57902 -const SECOND_MICROSECOND = 57903 -const MINUTE_MICROSECOND = 57904 -const MINUTE_SECOND = 57905 -const HOUR_MICROSECOND = 57906 -const HOUR_SECOND = 57907 -const HOUR_MINUTE = 57908 -const DAY_MICROSECOND = 57909 -const DAY_SECOND = 57910 -const DAY_MINUTE = 57911 -const DAY_HOUR = 57912 -const YEAR_MONTH = 57913 -const SQL_TSI_HOUR = 57914 -const SQL_TSI_DAY = 57915 -const SQL_TSI_WEEK = 57916 -const SQL_TSI_MONTH = 57917 -const SQL_TSI_QUARTER = 57918 -const SQL_TSI_YEAR = 57919 -const SQL_TSI_SECOND = 57920 -const SQL_TSI_MINUTE = 57921 -const RECURSIVE = 57922 -const CONFIG = 57923 -const DRAINER = 57924 -const SOURCE = 57925 -const STREAM = 57926 -const HEADERS = 57927 -const CONNECTOR = 57928 -const CONNECTORS = 57929 -const DAEMON = 57930 -const PAUSE = 57931 -const CANCEL = 57932 -const RESUME = 57933 -const SCHEDULE = 57934 -const TIMEZONE = 57935 -const TIMEOUT = 57936 -const TASK = 57937 -const MATCH = 57938 -const AGAINST = 57939 -const BOOLEAN = 57940 -const LANGUAGE = 57941 -const QUERY = 57942 -const EXPANSION = 57943 -const WITHOUT = 57944 -const VALIDATION = 57945 -const UPGRADE = 57946 -const RETRY = 57947 -const ADDDATE = 57948 -const BIT_AND = 57949 -const BIT_OR = 57950 -const BIT_XOR = 57951 -const CAST = 57952 -const COUNT = 57953 -const APPROX_COUNT = 57954 -const APPROX_COUNT_DISTINCT = 57955 -const SERIAL_EXTRACT = 57956 -const APPROX_PERCENTILE = 57957 -const CURDATE = 57958 -const CURTIME = 57959 -const DATE_ADD = 57960 -const DATE_SUB = 57961 -const EXTRACT = 57962 -const GROUP_CONCAT = 57963 -const MAX = 57964 -const MID = 57965 -const MIN = 57966 -const NOW = 57967 -const POSITION = 57968 -const SESSION_USER = 57969 -const STD = 57970 -const STDDEV = 57971 -const MEDIAN = 57972 -const CLUSTER_CENTERS = 57973 -const KMEANS = 57974 -const STDDEV_POP = 57975 -const STDDEV_SAMP = 57976 -const SUBDATE = 57977 -const SUBSTR = 57978 -const SUBSTRING = 57979 -const SUM = 57980 -const SYSDATE = 57981 -const SYSTEM_USER = 57982 -const TRANSLATE = 57983 -const TRIM = 57984 -const VARIANCE = 57985 -const VAR_POP = 57986 -const VAR_SAMP = 57987 -const AVG = 57988 -const RANK = 57989 -const ROW_NUMBER = 57990 -const DENSE_RANK = 57991 -const CUME_DIST = 57992 -const BIT_CAST = 57993 -const LAG = 57994 -const LEAD = 57995 -const FIRST_VALUE = 57996 -const LAST_VALUE = 57997 -const NTH_VALUE = 57998 -const NTILE = 57999 -const PERCENT_RANK = 58000 -const BITMAP_BIT_POSITION = 58001 -const BITMAP_BUCKET_NUMBER = 58002 -const BITMAP_COUNT = 58003 -const BITMAP_CONSTRUCT_AGG = 58004 -const BITMAP_OR_AGG = 58005 -const JSON_ARRAYAGG = 58006 -const JSON_OBJECTAGG = 58007 -const GET_FORMAT = 58008 -const SRID = 58009 -const NEXTVAL = 58010 -const SETVAL = 58011 -const CURRVAL = 58012 -const LASTVAL = 58013 -const ROW = 58014 -const OUTFILE = 58015 -const HEADER = 58016 -const MAX_FILE_SIZE = 58017 -const FORCE_QUOTE = 58018 -const PARALLEL = 58019 -const STRICT = 58020 -const SPLITSIZE = 58021 -const UNUSED = 58022 -const BINDINGS = 58023 -const GENERATED = 58024 -const ALWAYS = 58025 -const STORED = 58026 -const VIRTUAL = 58027 -const DO = 58028 -const DECLARE = 58029 -const LOOP = 58030 -const WHILE = 58031 -const LEAVE = 58032 -const ITERATE = 58033 -const UNTIL = 58034 -const CALL = 58035 -const PREV = 58036 -const SLIDING = 58037 -const FILL = 58038 -const SPBEGIN = 58039 -const BACKEND = 58040 -const SERVERS = 58041 -const HANDLER = 58042 -const PERCENT = 58043 -const SAMPLE = 58044 -const MO_TS = 58045 -const PITR = 58046 -const RECOVERY_WINDOW = 58047 -const INTERNAL = 58048 -const CDC_TASK_NAME = 58049 -const CDC = 58050 -const GROUPING = 58051 -const SETS = 58052 -const CUBE = 58053 -const ROLLUP = 58054 -const LOGSERVICE = 58055 -const REPLICAS = 58056 -const STORES = 58057 -const SETTINGS = 58058 -const KILL = 58059 -const BACKUP = 58060 -const FILESYSTEM = 58061 -const PARALLELISM = 58062 -const RESTORE = 58063 -const QUERY_RESULT = 58064 -const ARRAY = 58065 +const VECBF16 = 57553 +const VECF16 = 57554 +const VECINT8 = 57555 +const VECUINT8 = 57556 +const GEOMETRY = 57557 +const POINT = 57558 +const LINESTRING = 57559 +const POLYGON = 57560 +const GEOMETRYCOLLECTION = 57561 +const MULTIPOINT = 57562 +const MULTILINESTRING = 57563 +const MULTIPOLYGON = 57564 +const GEOMETRY32 = 57565 +const GEOGRAPHY = 57566 +const GEOGRAPHY32 = 57567 +const POINT32 = 57568 +const LINESTRING32 = 57569 +const POLYGON32 = 57570 +const GEOMETRYCOLLECTION32 = 57571 +const MULTIPOINT32 = 57572 +const MULTILINESTRING32 = 57573 +const MULTIPOLYGON32 = 57574 +const INT1 = 57575 +const INT2 = 57576 +const INT3 = 57577 +const INT4 = 57578 +const INT8 = 57579 +const S3OPTION = 57580 +const STAGEOPTION = 57581 +const SQL_SMALL_RESULT = 57582 +const SQL_BIG_RESULT = 57583 +const SQL_BUFFER_RESULT = 57584 +const SQL_CALC_FOUND_ROWS = 57585 +const LOW_PRIORITY = 57586 +const HIGH_PRIORITY = 57587 +const DELAYED = 57588 +const CREATE = 57589 +const ALTER = 57590 +const DROP = 57591 +const RENAME = 57592 +const REMOVE = 57593 +const ANALYZE = 57594 +const PHYPLAN = 57595 +const ADD = 57596 +const RETURNS = 57597 +const SCHEMA = 57598 +const TABLE = 57599 +const SEQUENCE = 57600 +const INDEX = 57601 +const VIEW = 57602 +const TO = 57603 +const IGNORE = 57604 +const IF = 57605 +const PRIMARY = 57606 +const COLUMN = 57607 +const CONSTRAINT = 57608 +const SPATIAL = 57609 +const FULLTEXT = 57610 +const FOREIGN = 57611 +const KEY_BLOCK_SIZE = 57612 +const SHOW = 57613 +const DESCRIBE = 57614 +const EXPLAIN = 57615 +const DATE = 57616 +const ESCAPE = 57617 +const REPAIR = 57618 +const OPTIMIZE = 57619 +const TRUNCATE = 57620 +const MAXVALUE = 57621 +const PARTITION = 57622 +const REORGANIZE = 57623 +const LESS = 57624 +const THAN = 57625 +const PROCEDURE = 57626 +const TRIGGER = 57627 +const STATUS = 57628 +const VARIABLES = 57629 +const ROLE = 57630 +const PROXY = 57631 +const AVG_ROW_LENGTH = 57632 +const STORAGE = 57633 +const DISK = 57634 +const MEMORY = 57635 +const CHECKSUM = 57636 +const COMPRESSION = 57637 +const DATA = 57638 +const DIRECTORY = 57639 +const DELAY_KEY_WRITE = 57640 +const ENCRYPTION = 57641 +const ENGINE = 57642 +const MAX_ROWS = 57643 +const MIN_ROWS = 57644 +const PACK_KEYS = 57645 +const ROW_FORMAT = 57646 +const STATS_AUTO_RECALC = 57647 +const STATS_PERSISTENT = 57648 +const STATS_SAMPLE_PAGES = 57649 +const DYNAMIC = 57650 +const COMPRESSED = 57651 +const REDUNDANT = 57652 +const COMPACT = 57653 +const FIXED = 57654 +const COLUMN_FORMAT = 57655 +const AUTO_RANDOM = 57656 +const ENGINE_ATTRIBUTE = 57657 +const SECONDARY_ENGINE_ATTRIBUTE = 57658 +const INSERT_METHOD = 57659 +const RESTRICT = 57660 +const CASCADE = 57661 +const ACTION = 57662 +const PARTIAL = 57663 +const SIMPLE = 57664 +const CHECK = 57665 +const ENFORCED = 57666 +const RANGE = 57667 +const LIST = 57668 +const ALGORITHM = 57669 +const LINEAR = 57670 +const PARTITIONS = 57671 +const SUBPARTITION = 57672 +const SUBPARTITIONS = 57673 +const CLUSTER = 57674 +const TYPE = 57675 +const ANY = 57676 +const SOME = 57677 +const EXTERNAL = 57678 +const LOCALFILE = 57679 +const URL = 57680 +const PREPARE = 57681 +const DEALLOCATE = 57682 +const RESET = 57683 +const EXTENSION = 57684 +const RETENTION = 57685 +const PERIOD = 57686 +const CLONE = 57687 +const BRANCH = 57688 +const LOG = 57689 +const REVERT = 57690 +const REBASE = 57691 +const DIFF = 57692 +const PICK = 57693 +const CONFLICT = 57694 +const CONFLICT_FAIL = 57695 +const CONFLICT_SKIP = 57696 +const CONFLICT_ACCEPT = 57697 +const OUTPUT = 57698 +const SUMMARY = 57699 +const INCREMENT = 57700 +const CYCLE = 57701 +const MINVALUE = 57702 +const PUBLICATION = 57703 +const SUBSCRIPTION = 57704 +const SUBSCRIPTIONS = 57705 +const PUBLICATIONS = 57706 +const SYNC_INTERVAL = 57707 +const SYNC = 57708 +const COVERAGE = 57709 +const CCPR = 57710 +const PROPERTIES = 57711 +const PARSER = 57712 +const VISIBLE = 57713 +const INVISIBLE = 57714 +const BTREE = 57715 +const HASH = 57716 +const RTREE = 57717 +const BSI = 57718 +const IVFFLAT = 57719 +const MASTER = 57720 +const HNSW = 57721 +const CAGRA = 57722 +const IVFPQ = 57723 +const BM25 = 57724 +const ZONEMAP = 57725 +const LEADING = 57726 +const BOTH = 57727 +const TRAILING = 57728 +const UNKNOWN = 57729 +const LISTS = 57730 +const OP_TYPE = 57731 +const REINDEX = 57732 +const EF_SEARCH = 57733 +const EF_CONSTRUCTION = 57734 +const M = 57735 +const ASYNC = 57736 +const FORCE_SYNC = 57737 +const AUTO_UPDATE = 57738 +const INTERMEDIATE_GRAPH_DEGREE = 57739 +const GRAPH_DEGREE = 57740 +const QUANTIZATION = 57741 +const BITS_PER_CODE = 57742 +const DISTRIBUTION_MODE = 57743 +const ITOPK_SIZE = 57744 +const INCLUDE = 57745 +const KMEANS_TRAIN_PERCENT = 57746 +const KMEANS_MAX_ITERATION = 57747 +const MAX_INDEX_CAPACITY = 57748 +const EXPIRE = 57749 +const ACCOUNT = 57750 +const ACCOUNTS = 57751 +const UNLOCK = 57752 +const DAY = 57753 +const NEVER = 57754 +const PUMP = 57755 +const MYSQL_COMPATIBILITY_MODE = 57756 +const UNIQUE_CHECK_ON_AUTOINCR = 57757 +const MODIFY = 57758 +const CHANGE = 57759 +const SECOND = 57760 +const ASCII = 57761 +const COALESCE = 57762 +const COLLATION = 57763 +const HOUR = 57764 +const MICROSECOND = 57765 +const MINUTE = 57766 +const MONTH = 57767 +const QUARTER = 57768 +const REPEAT = 57769 +const REVERSE = 57770 +const ROW_COUNT = 57771 +const WEEK = 57772 +const REVOKE = 57773 +const FUNCTION = 57774 +const PRIVILEGES = 57775 +const TABLESPACE = 57776 +const EXECUTE = 57777 +const SUPER = 57778 +const GRANT = 57779 +const OPTION = 57780 +const REFERENCES = 57781 +const REPLICATION = 57782 +const SLAVE = 57783 +const CLIENT = 57784 +const USAGE = 57785 +const RELOAD = 57786 +const FILE = 57787 +const FILES = 57788 +const TEMPORARY = 57789 +const ROUTINE = 57790 +const EVENT = 57791 +const SHUTDOWN = 57792 +const NULLX = 57793 +const AUTO_INCREMENT = 57794 +const APPROXNUM = 57795 +const ENGINES = 57796 +const LOW_CARDINALITY = 57797 +const AUTOEXTEND_SIZE = 57798 +const ADMIN_NAME = 57799 +const RANDOM = 57800 +const SUSPEND = 57801 +const ATTRIBUTE = 57802 +const HISTORY = 57803 +const REUSE = 57804 +const CURRENT = 57805 +const OPTIONAL = 57806 +const FAILED_LOGIN_ATTEMPTS = 57807 +const PASSWORD_LOCK_TIME = 57808 +const UNBOUNDED = 57809 +const SECONDARY = 57810 +const RESTRICTED = 57811 +const USER = 57812 +const IDENTIFIED = 57813 +const CIPHER = 57814 +const ISSUER = 57815 +const X509 = 57816 +const SUBJECT = 57817 +const SAN = 57818 +const REQUIRE = 57819 +const SSL = 57820 +const NONE = 57821 +const PASSWORD = 57822 +const SHARED = 57823 +const EXCLUSIVE = 57824 +const MAX_QUERIES_PER_HOUR = 57825 +const MAX_UPDATES_PER_HOUR = 57826 +const MAX_CONNECTIONS_PER_HOUR = 57827 +const MAX_USER_CONNECTIONS = 57828 +const FORMAT = 57829 +const VERBOSE = 57830 +const CONNECTION = 57831 +const TRIGGERS = 57832 +const PROFILES = 57833 +const LOAD = 57834 +const INLINE = 57835 +const INFILE = 57836 +const TERMINATED = 57837 +const OPTIONALLY = 57838 +const ENCLOSED = 57839 +const ESCAPED = 57840 +const STARTING = 57841 +const LINES = 57842 +const ROWS = 57843 +const IMPORT = 57844 +const DISCARD = 57845 +const JSONTYPE = 57846 +const MODUMP = 57847 +const OVER = 57848 +const PRECEDING = 57849 +const FOLLOWING = 57850 +const GROUPS = 57851 +const DATABASES = 57852 +const TABLES = 57853 +const SEQUENCES = 57854 +const EXTENDED = 57855 +const FULL = 57856 +const PROCESSLIST = 57857 +const FIELDS = 57858 +const COLUMNS = 57859 +const OPEN = 57860 +const ERRORS = 57861 +const WARNINGS = 57862 +const INDEXES = 57863 +const SCHEMAS = 57864 +const NODE = 57865 +const LOCKS = 57866 +const ROLES = 57867 +const RULE = 57868 +const RULES = 57869 +const TABLE_NUMBER = 57870 +const COLUMN_NUMBER = 57871 +const TABLE_VALUES = 57872 +const TABLE_SIZE = 57873 +const TASKS = 57874 +const RUNS = 57875 +const NAMES = 57876 +const GLOBAL = 57877 +const PERSIST = 57878 +const SESSION = 57879 +const ISOLATION = 57880 +const LEVEL = 57881 +const READ = 57882 +const WRITE = 57883 +const ONLY = 57884 +const REPEATABLE = 57885 +const COMMITTED = 57886 +const UNCOMMITTED = 57887 +const SERIALIZABLE = 57888 +const LOCAL = 57889 +const EVENTS = 57890 +const PLUGINS = 57891 +const CURRENT_TIMESTAMP = 57892 +const DATABASE = 57893 +const CURRENT_TIME = 57894 +const LOCALTIME = 57895 +const LOCALTIMESTAMP = 57896 +const UTC_DATE = 57897 +const UTC_TIME = 57898 +const UTC_TIMESTAMP = 57899 +const REPLACE = 57900 +const CONVERT = 57901 +const SEPARATOR = 57902 +const TIMESTAMPDIFF = 57903 +const TIMESTAMPADD = 57904 +const CURRENT_DATE = 57905 +const CURRENT_USER = 57906 +const CURRENT_ROLE = 57907 +const SECOND_MICROSECOND = 57908 +const MINUTE_MICROSECOND = 57909 +const MINUTE_SECOND = 57910 +const HOUR_MICROSECOND = 57911 +const HOUR_SECOND = 57912 +const HOUR_MINUTE = 57913 +const DAY_MICROSECOND = 57914 +const DAY_SECOND = 57915 +const DAY_MINUTE = 57916 +const DAY_HOUR = 57917 +const YEAR_MONTH = 57918 +const SQL_TSI_HOUR = 57919 +const SQL_TSI_DAY = 57920 +const SQL_TSI_WEEK = 57921 +const SQL_TSI_MONTH = 57922 +const SQL_TSI_QUARTER = 57923 +const SQL_TSI_YEAR = 57924 +const SQL_TSI_SECOND = 57925 +const SQL_TSI_MINUTE = 57926 +const RECURSIVE = 57927 +const CONFIG = 57928 +const DRAINER = 57929 +const SOURCE = 57930 +const STREAM = 57931 +const HEADERS = 57932 +const CONNECTOR = 57933 +const CONNECTORS = 57934 +const DAEMON = 57935 +const PAUSE = 57936 +const CANCEL = 57937 +const RESUME = 57938 +const SCHEDULE = 57939 +const TIMEZONE = 57940 +const TIMEOUT = 57941 +const TASK = 57942 +const MATCH = 57943 +const AGAINST = 57944 +const BOOLEAN = 57945 +const LANGUAGE = 57946 +const QUERY = 57947 +const EXPANSION = 57948 +const WITHOUT = 57949 +const VALIDATION = 57950 +const UPGRADE = 57951 +const RETRY = 57952 +const ADDDATE = 57953 +const BIT_AND = 57954 +const BIT_OR = 57955 +const BIT_XOR = 57956 +const CAST = 57957 +const COUNT = 57958 +const APPROX_COUNT = 57959 +const APPROX_COUNT_DISTINCT = 57960 +const SERIAL_EXTRACT = 57961 +const APPROX_PERCENTILE = 57962 +const CURDATE = 57963 +const CURTIME = 57964 +const DATE_ADD = 57965 +const DATE_SUB = 57966 +const EXTRACT = 57967 +const GROUP_CONCAT = 57968 +const MAX = 57969 +const MID = 57970 +const MIN = 57971 +const NOW = 57972 +const POSITION = 57973 +const SESSION_USER = 57974 +const STD = 57975 +const STDDEV = 57976 +const MEDIAN = 57977 +const CLUSTER_CENTERS = 57978 +const KMEANS = 57979 +const STDDEV_POP = 57980 +const STDDEV_SAMP = 57981 +const SUBDATE = 57982 +const SUBSTR = 57983 +const SUBSTRING = 57984 +const SUM = 57985 +const SYSDATE = 57986 +const SYSTEM_USER = 57987 +const TRANSLATE = 57988 +const TRIM = 57989 +const VARIANCE = 57990 +const VAR_POP = 57991 +const VAR_SAMP = 57992 +const AVG = 57993 +const RANK = 57994 +const ROW_NUMBER = 57995 +const DENSE_RANK = 57996 +const CUME_DIST = 57997 +const BIT_CAST = 57998 +const LAG = 57999 +const LEAD = 58000 +const FIRST_VALUE = 58001 +const LAST_VALUE = 58002 +const NTH_VALUE = 58003 +const NTILE = 58004 +const PERCENT_RANK = 58005 +const BITMAP_BIT_POSITION = 58006 +const BITMAP_BUCKET_NUMBER = 58007 +const BITMAP_COUNT = 58008 +const BITMAP_CONSTRUCT_AGG = 58009 +const BITMAP_OR_AGG = 58010 +const JSON_ARRAYAGG = 58011 +const JSON_OBJECTAGG = 58012 +const GET_FORMAT = 58013 +const SRID = 58014 +const NEXTVAL = 58015 +const SETVAL = 58016 +const CURRVAL = 58017 +const LASTVAL = 58018 +const ROW = 58019 +const OUTFILE = 58020 +const HEADER = 58021 +const MAX_FILE_SIZE = 58022 +const FORCE_QUOTE = 58023 +const PARALLEL = 58024 +const STRICT = 58025 +const SPLITSIZE = 58026 +const UNUSED = 58027 +const BINDINGS = 58028 +const GENERATED = 58029 +const ALWAYS = 58030 +const STORED = 58031 +const VIRTUAL = 58032 +const DO = 58033 +const DECLARE = 58034 +const LOOP = 58035 +const WHILE = 58036 +const LEAVE = 58037 +const ITERATE = 58038 +const UNTIL = 58039 +const CALL = 58040 +const PREV = 58041 +const SLIDING = 58042 +const FILL = 58043 +const SPBEGIN = 58044 +const BACKEND = 58045 +const SERVERS = 58046 +const HANDLER = 58047 +const PERCENT = 58048 +const SAMPLE = 58049 +const MO_TS = 58050 +const PITR = 58051 +const RECOVERY_WINDOW = 58052 +const INTERNAL = 58053 +const CDC_TASK_NAME = 58054 +const CDC = 58055 +const GROUPING = 58056 +const SETS = 58057 +const CUBE = 58058 +const ROLLUP = 58059 +const LOGSERVICE = 58060 +const REPLICAS = 58061 +const STORES = 58062 +const SETTINGS = 58063 +const KILL = 58064 +const BACKUP = 58065 +const FILESYSTEM = 58066 +const PARALLELISM = 58067 +const RESTORE = 58068 +const QUERY_RESULT = 58069 +const ARRAY = 58070 var yyToknames = [...]string{ "$end", @@ -1014,6 +1019,10 @@ var yyToknames = [...]string{ "UUID", "VECF32", "VECF64", + "VECBF16", + "VECF16", + "VECINT8", + "VECUINT8", "GEOMETRY", "POINT", "LINESTRING", @@ -1181,6 +1190,7 @@ var yyToknames = [...]string{ "HNSW", "CAGRA", "IVFPQ", + "BM25", "ZONEMAP", "LEADING", "BOTH", @@ -1540,7 +1550,7 @@ const yyEofCode = 1 const yyErrCode = 2 const yyInitialStackSize = 16 -//line mysql_sql.y:14585 +//line mysql_sql.y:14692 //line yacctab:1 var yyExca = [...]int{ @@ -1548,7372 +1558,7469 @@ var yyExca = [...]int{ 1, -1, -2, 0, -1, 155, - 11, 885, - 24, 885, - -2, 878, + 11, 886, + 24, 886, + -2, 879, -1, 181, - 270, 1405, - 272, 1247, - -2, 1320, + 274, 1409, + 276, 1248, + -2, 1324, -1, 211, - 46, 690, - 272, 690, - 299, 697, - 300, 697, - 533, 690, - -2, 728, + 46, 691, + 276, 691, + 303, 698, + 304, 698, + 538, 691, + -2, 729, -1, 251, - 744, 2270, - -2, 577, - -1, 608, - 744, 2397, + 749, 2285, + -2, 578, + -1, 613, + 749, 2412, -2, 437, - -1, 666, - 744, 2456, + -1, 671, + 749, 2471, -2, 435, - -1, 667, - 744, 2457, + -1, 672, + 749, 2472, -2, 436, - -1, 668, - 744, 2458, + -1, 673, + 749, 2473, -2, 438, - -1, 826, - 351, 201, - 505, 201, - 506, 201, - -2, 2141, - -1, 894, - 88, 1897, - -2, 2333, - -1, 895, - 88, 1915, - -2, 2302, - -1, 899, - 88, 1916, - -2, 2332, - -1, 945, - 88, 1818, - -2, 2548, - -1, 946, - 88, 1819, - -2, 2547, - -1, 947, - 88, 1820, - -2, 2537, - -1, 948, - 88, 2508, - -2, 2528, - -1, 949, - 88, 2509, - -2, 2529, - -1, 950, - 88, 2510, - -2, 2539, + -1, 831, + 355, 201, + 510, 201, + 511, 201, + -2, 2151, + -1, 900, + 88, 1903, + -2, 2348, + -1, 901, + 88, 1921, + -2, 2317, + -1, 905, + 88, 1922, + -2, 2347, -1, 951, - 88, 2511, - -2, 2517, + 88, 1824, + -2, 2563, -1, 952, - 88, 2512, - -2, 2526, + 88, 1825, + -2, 2562, -1, 953, - 88, 2513, - -2, 2541, + 88, 1826, + -2, 2552, -1, 954, - 88, 2514, - -2, 2546, + 88, 2523, + -2, 2543, -1, 955, - 88, 2515, - -2, 2551, + 88, 2524, + -2, 2544, -1, 956, - 88, 2516, - -2, 2552, + 88, 2525, + -2, 2554, -1, 957, - 88, 1893, - -2, 2371, + 88, 2526, + -2, 2532, -1, 958, - 88, 1894, - -2, 2121, + 88, 2527, + -2, 2541, -1, 959, - 88, 1895, - -2, 2380, + 88, 2528, + -2, 2556, -1, 960, - 88, 1896, - -2, 2134, + 88, 2529, + -2, 2561, + -1, 961, + 88, 2530, + -2, 2566, -1, 962, + 88, 2531, + -2, 2567, + -1, 963, 88, 1899, - -2, 2143, + -2, 2386, -1, 964, + 88, 1900, + -2, 2131, + -1, 965, 88, 1901, - -2, 2405, + -2, 2395, -1, 966, - 88, 1903, - -2, 2165, + 88, 1902, + -2, 2144, -1, 968, 88, 1905, - -2, 2417, - -1, 969, - 88, 1906, - -2, 2416, + -2, 2153, -1, 970, 88, 1907, - -2, 2231, - -1, 971, - 88, 1908, - -2, 2328, + -2, 2420, + -1, 972, + 88, 1909, + -2, 2175, -1, 974, 88, 1911, - -2, 2428, + -2, 2432, + -1, 975, + 88, 1912, + -2, 2431, -1, 976, 88, 1913, - -2, 2431, + -2, 2246, -1, 977, 88, 1914, - -2, 2433, - -1, 978, - 88, 1917, - -2, 2440, - -1, 979, - 88, 1918, - -2, 2311, + -2, 2343, -1, 980, + 88, 1917, + -2, 2443, + -1, 982, 88, 1919, - -2, 2358, - -1, 981, + -2, 2446, + -1, 983, 88, 1920, - -2, 2322, - -1, 982, - 88, 1921, - -2, 2348, - -1, 993, - 88, 1794, - -2, 2542, - -1, 994, - 88, 1795, - -2, 2543, - -1, 995, - 88, 1796, - -2, 2544, - -1, 1111, - 528, 728, - 529, 728, - -2, 691, - -1, 1166, - 130, 2121, - 141, 2121, - 173, 2121, - -2, 2089, - -1, 1300, - 24, 914, - -2, 857, - -1, 1420, - 11, 885, - 24, 885, - -2, 1654, - -1, 1518, - 24, 914, - -2, 857, - -1, 1901, - 88, 1968, - -2, 2330, - -1, 1902, - 88, 1969, - -2, 2331, - -1, 2600, - 89, 1103, - -2, 1109, + -2, 2448, + -1, 984, + 88, 1923, + -2, 2455, + -1, 985, + 88, 1924, + -2, 2326, + -1, 986, + 88, 1925, + -2, 2373, + -1, 987, + 88, 1926, + -2, 2337, + -1, 988, + 88, 1927, + -2, 2363, + -1, 999, + 88, 1800, + -2, 2557, + -1, 1000, + 88, 1801, + -2, 2558, + -1, 1001, + 88, 1802, + -2, 2559, + -1, 1117, + 533, 729, + 534, 729, + -2, 692, + -1, 1172, + 130, 2131, + 141, 2131, + 173, 2131, + -2, 2099, + -1, 1310, + 24, 915, + -2, 858, + -1, 1430, + 11, 886, + 24, 886, + -2, 1659, + -1, 1529, + 24, 915, + -2, 858, + -1, 1916, + 88, 1974, + -2, 2345, + -1, 1917, + 88, 1975, + -2, 2346, -1, 2617, - 113, 1312, - 160, 1312, - 208, 1312, - 211, 1312, - 312, 1312, - -2, 1305, - -1, 2810, - 11, 885, - 24, 885, - -2, 1030, - -1, 2847, - 89, 2075, - 174, 2075, - -2, 2313, - -1, 2848, - 89, 2075, - 174, 2075, - -2, 2312, - -1, 2849, - 89, 2033, - 174, 2033, - -2, 2299, - -1, 2850, - 89, 2034, - 174, 2034, - -2, 2304, - -1, 2851, - 89, 2035, - 174, 2035, - -2, 2219, - -1, 2852, - 89, 2036, - 174, 2036, - -2, 2212, - -1, 2853, - 89, 2037, - 174, 2037, - -2, 2108, - -1, 2854, - 89, 2038, - 174, 2038, - -2, 2301, - -1, 2855, + 89, 1104, + -2, 1110, + -1, 2634, + 113, 1316, + 160, 1316, + 208, 1316, + 211, 1316, + 316, 1316, + -2, 1309, + -1, 2827, + 11, 886, + 24, 886, + -2, 1031, + -1, 2864, + 89, 2085, + 174, 2085, + -2, 2328, + -1, 2865, + 89, 2085, + 174, 2085, + -2, 2327, + -1, 2866, 89, 2039, 174, 2039, - -2, 2217, - -1, 2856, + -2, 2314, + -1, 2867, 89, 2040, 174, 2040, - -2, 2211, - -1, 2857, + -2, 2319, + -1, 2868, 89, 2041, 174, 2041, - -2, 2196, - -1, 2858, - 89, 2075, - 174, 2075, - -2, 2197, - -1, 2859, - 89, 2075, - 174, 2075, - -2, 2198, - -1, 2861, - 89, 2046, - 174, 2046, - -2, 2348, - -1, 2862, - 89, 2023, - 174, 2023, - -2, 2333, - -1, 2863, - 89, 2073, - 174, 2073, - -2, 2302, - -1, 2864, - 89, 2073, - 174, 2073, - -2, 2332, - -1, 2865, - 89, 2073, - 174, 2073, - -2, 2144, - -1, 2866, - 89, 2071, - 174, 2071, - -2, 2322, - -1, 2867, - 88, 2003, - 89, 2003, - 163, 2003, - 164, 2003, - 166, 2003, - 174, 2003, - -2, 2107, - -1, 2868, - 88, 2004, - 89, 2004, - 163, 2004, - 164, 2004, - 166, 2004, - 174, 2004, - -2, 2109, + -2, 2234, -1, 2869, - 88, 2005, - 89, 2005, - 163, 2005, - 164, 2005, - 166, 2005, - 174, 2005, - -2, 2376, + 89, 2042, + 174, 2042, + -2, 2227, -1, 2870, - 88, 2007, - 89, 2007, - 163, 2007, - 164, 2007, - 166, 2007, - 174, 2007, - -2, 2303, + 89, 2043, + 174, 2043, + -2, 2118, -1, 2871, - 88, 2009, - 89, 2009, - 163, 2009, - 164, 2009, - 166, 2009, - 174, 2009, - -2, 2280, + 89, 2044, + 174, 2044, + -2, 2316, -1, 2872, - 88, 2011, - 89, 2011, - 163, 2011, - 164, 2011, - 166, 2011, - 174, 2011, - -2, 2218, + 89, 2045, + 174, 2045, + -2, 2232, -1, 2873, - 88, 2013, - 89, 2013, - 163, 2013, - 164, 2013, - 166, 2013, - 174, 2013, - -2, 2190, + 89, 2046, + 174, 2046, + -2, 2226, -1, 2874, - 88, 2014, - 89, 2014, - 163, 2014, - 164, 2014, - 166, 2014, - 174, 2014, - -2, 2191, + 89, 2047, + 174, 2047, + -2, 2207, -1, 2875, - 88, 2016, - 89, 2016, - 163, 2016, - 164, 2016, - 166, 2016, - 174, 2016, - -2, 2106, + 89, 2085, + 174, 2085, + -2, 2208, -1, 2876, - 89, 2078, - 163, 2078, - 164, 2078, - 166, 2078, - 174, 2078, - -2, 2149, + 89, 2085, + 174, 2085, + -2, 2209, -1, 2877, - 89, 2078, - 163, 2078, - 164, 2078, - 166, 2078, - 174, 2078, - -2, 2166, + 89, 2085, + 174, 2085, + -2, 2210, -1, 2878, - 89, 2081, - 163, 2081, - 164, 2081, - 166, 2081, - 174, 2081, - -2, 2145, + 89, 2085, + 174, 2085, + -2, 2211, -1, 2879, - 89, 2081, - 163, 2081, - 164, 2081, - 166, 2081, - 174, 2081, - -2, 2234, + 89, 2085, + 174, 2085, + -2, 2212, -1, 2880, - 89, 2078, - 163, 2078, - 164, 2078, - 166, 2078, - 174, 2078, - -2, 2262, - -1, 2881, - 89, 2051, - 174, 2051, - -2, 2170, + 89, 2085, + 174, 2085, + -2, 2213, -1, 2882, - 89, 2052, - 174, 2052, - -2, 2248, + 89, 2056, + 174, 2056, + -2, 2363, -1, 2883, - 89, 2053, - 174, 2053, - -2, 2209, + 89, 2029, + 174, 2029, + -2, 2348, -1, 2884, - 89, 2054, - 174, 2054, - -2, 2249, + 89, 2083, + 174, 2083, + -2, 2317, -1, 2885, - 89, 2055, - 174, 2055, - -2, 2171, + 89, 2083, + 174, 2083, + -2, 2347, -1, 2886, - 89, 2056, - 174, 2056, - -2, 2223, + 89, 2083, + 174, 2083, + -2, 2154, -1, 2887, - 89, 2057, - 174, 2057, - -2, 2222, + 89, 2081, + 174, 2081, + -2, 2337, -1, 2888, - 89, 2058, - 174, 2058, - -2, 2224, + 88, 2009, + 89, 2009, + 163, 2009, + 164, 2009, + 166, 2009, + 174, 2009, + -2, 2117, -1, 2889, - 89, 2059, - 174, 2059, - -2, 2173, + 88, 2010, + 89, 2010, + 163, 2010, + 164, 2010, + 166, 2010, + 174, 2010, + -2, 2119, -1, 2890, - 89, 2060, - 174, 2060, - -2, 2172, + 88, 2011, + 89, 2011, + 163, 2011, + 164, 2011, + 166, 2011, + 174, 2011, + -2, 2391, -1, 2891, + 88, 2013, + 89, 2013, + 163, 2013, + 164, 2013, + 166, 2013, + 174, 2013, + -2, 2318, + -1, 2892, + 88, 2015, + 89, 2015, + 163, 2015, + 164, 2015, + 166, 2015, + 174, 2015, + -2, 2295, + -1, 2893, + 88, 2017, + 89, 2017, + 163, 2017, + 164, 2017, + 166, 2017, + 174, 2017, + -2, 2233, + -1, 2894, + 88, 2019, + 89, 2019, + 163, 2019, + 164, 2019, + 166, 2019, + 174, 2019, + -2, 2201, + -1, 2895, + 88, 2020, + 89, 2020, + 163, 2020, + 164, 2020, + 166, 2020, + 174, 2020, + -2, 2202, + -1, 2896, + 88, 2022, + 89, 2022, + 163, 2022, + 164, 2022, + 166, 2022, + 174, 2022, + -2, 2116, + -1, 2897, + 89, 2088, + 163, 2088, + 164, 2088, + 166, 2088, + 174, 2088, + -2, 2159, + -1, 2898, + 89, 2088, + 163, 2088, + 164, 2088, + 166, 2088, + 174, 2088, + -2, 2176, + -1, 2899, + 89, 2091, + 163, 2091, + 164, 2091, + 166, 2091, + 174, 2091, + -2, 2155, + -1, 2900, + 89, 2091, + 163, 2091, + 164, 2091, + 166, 2091, + 174, 2091, + -2, 2249, + -1, 2901, + 89, 2088, + 163, 2088, + 164, 2088, + 166, 2088, + 174, 2088, + -2, 2277, + -1, 2902, 89, 2061, 174, 2061, - -2, 2174, - -1, 2892, + -2, 2180, + -1, 2903, 89, 2062, 174, 2062, - -2, 2175, - -1, 2893, + -2, 2263, + -1, 2904, 89, 2063, 174, 2063, - -2, 2176, - -1, 2894, + -2, 2224, + -1, 2905, 89, 2064, 174, 2064, - -2, 2177, - -1, 2895, + -2, 2264, + -1, 2906, 89, 2065, 174, 2065, - -2, 2178, - -1, 2896, + -2, 2181, + -1, 2907, 89, 2066, 174, 2066, - -2, 2179, - -1, 2897, + -2, 2238, + -1, 2908, 89, 2067, 174, 2067, - -2, 2180, - -1, 2898, + -2, 2237, + -1, 2909, 89, 2068, 174, 2068, - -2, 2181, - -1, 3151, - 113, 1312, - 160, 1312, - 208, 1312, - 211, 1312, - 312, 1312, - -2, 1306, - -1, 3185, - 86, 793, - 174, 793, - -2, 1520, - -1, 3657, - 211, 1312, - 336, 1617, - -2, 1583, - -1, 3702, - 11, 885, - 24, 885, - -2, 1654, - -1, 3897, - 113, 1312, - 160, 1312, - 208, 1312, - 211, 1312, - -2, 1461, - -1, 3902, - 113, 1312, - 160, 1312, - 208, 1312, - 211, 1312, - -2, 1461, - -1, 3918, - 86, 793, - 174, 793, - -2, 1520, - -1, 3939, - 211, 1312, - 336, 1617, - -2, 1584, - -1, 4139, - 113, 1312, - 160, 1312, - 208, 1312, - 211, 1312, - -2, 1462, - -1, 4169, - 89, 1423, - 174, 1423, - -2, 1312, - -1, 4370, - 89, 1423, - 174, 1423, - -2, 1312, - -1, 4590, + -2, 2239, + -1, 2910, + 89, 2069, + 174, 2069, + -2, 2183, + -1, 2911, + 89, 2070, + 174, 2070, + -2, 2182, + -1, 2912, + 89, 2071, + 174, 2071, + -2, 2184, + -1, 2913, + 89, 2072, + 174, 2072, + -2, 2185, + -1, 2914, + 89, 2073, + 174, 2073, + -2, 2186, + -1, 2915, + 89, 2074, + 174, 2074, + -2, 2187, + -1, 2916, + 89, 2075, + 174, 2075, + -2, 2188, + -1, 2917, + 89, 2076, + 174, 2076, + -2, 2189, + -1, 2918, + 89, 2077, + 174, 2077, + -2, 2190, + -1, 2919, + 89, 2078, + 174, 2078, + -2, 2191, + -1, 3174, + 113, 1316, + 160, 1316, + 208, 1316, + 211, 1316, + 316, 1316, + -2, 1310, + -1, 3208, + 86, 794, + 174, 794, + -2, 1524, + -1, 3681, + 211, 1316, + 340, 1622, + -2, 1588, + -1, 3727, + 11, 886, + 24, 886, + -2, 1659, + -1, 3923, + 113, 1316, + 160, 1316, + 208, 1316, + 211, 1316, + -2, 1465, + -1, 3928, + 113, 1316, + 160, 1316, + 208, 1316, + 211, 1316, + -2, 1465, + -1, 3944, + 86, 794, + 174, 794, + -2, 1524, + -1, 3965, + 211, 1316, + 340, 1622, + -2, 1589, + -1, 4167, + 113, 1316, + 160, 1316, + 208, 1316, + 211, 1316, + -2, 1466, + -1, 4197, + 89, 1427, + 174, 1427, + -2, 1316, + -1, 4401, 89, 1427, 174, 1427, - -2, 1312, - -1, 4645, - 89, 1428, - 174, 1428, - -2, 1312, + -2, 1316, + -1, 4623, + 89, 1431, + 174, 1431, + -2, 1316, + -1, 4678, + 89, 1432, + 174, 1432, + -2, 1316, } const yyPrivate = 57344 -const yyLast = 68212 +const yyLast = 69019 var yyAct = [...]int{ - 860, 836, 4694, 862, 4668, 3215, 240, 4686, 1810, 4600, - 4594, 3924, 2217, 4038, 1881, 4604, 3680, 4593, 4605, 3985, - 4370, 845, 4494, 3643, 4551, 2835, 4443, 3768, 4266, 3209, - 3953, 4348, 3532, 1718, 4434, 4308, 3530, 1877, 4200, 838, - 3769, 4033, 1458, 4369, 4471, 1947, 4126, 3766, 3868, 3212, - 719, 3101, 891, 4444, 4338, 1301, 4044, 1165, 227, 3, - 4446, 38, 3876, 1650, 1644, 3406, 2156, 4149, 738, 2687, - 3940, 1934, 3882, 1306, 3188, 3652, 4136, 752, 762, 771, - 1884, 3830, 771, 3601, 3903, 4107, 4141, 3584, 3559, 2629, - 2935, 3026, 3336, 1931, 2322, 3335, 3334, 2340, 225, 2284, - 3866, 789, 3304, 3588, 3238, 3672, 3654, 3025, 3661, 3905, - 3109, 154, 3699, 2319, 3331, 2364, 1953, 1930, 3822, 2804, - 3750, 784, 2430, 2405, 2842, 2690, 3366, 3728, 3137, 3322, - 3564, 1711, 3566, 2175, 3549, 3660, 3612, 3560, 3562, 768, - 780, 3561, 1949, 2647, 2638, 2637, 2630, 2063, 3557, 2942, - 3152, 833, 828, 2564, 37, 1598, 2916, 2563, 3512, 2426, - 2464, 2401, 2369, 1803, 2315, 1794, 1815, 1034, 1799, 2425, - 2805, 1604, 1798, 2787, 2288, 3125, 3119, 2688, 1787, 3240, - 2207, 2285, 3168, 752, 1074, 6, 2646, 2617, 2782, 2126, - 236, 8, 235, 7, 2840, 1948, 2460, 2427, 1229, 2636, - 1880, 1875, 3220, 837, 2633, 1760, 1727, 737, 1567, 875, - 155, 1696, 1159, 2608, 1690, 155, 719, 2147, 2683, 2174, - 827, 2398, 1633, 2611, 2566, 1941, 846, 1917, 1866, 15, - 1322, 2386, 1767, 1158, 1874, 1545, 1695, 2812, 2783, 777, - 240, 835, 240, 753, 1219, 1220, 1692, 1073, 1954, 2125, - 2121, 752, 1750, 997, 787, 786, 25, 1645, 26, 24, - 1654, 226, 718, 1629, 1051, 218, 1071, 1199, 1057, 1122, - 1540, 17, 222, 10, 1516, 1067, 1459, 783, 1106, 28, - 745, 1385, 1386, 1387, 1384, 2434, 999, 155, 4456, 770, - 1385, 1386, 1387, 1384, 1385, 1386, 1387, 1384, 1174, 70, - 4334, 3073, 1000, 2814, 70, 3073, 3073, 766, 16, 1216, - 3785, 2087, 3522, 3521, 3921, 3631, 1619, 3422, 3421, 1541, - 14, 1307, 4090, 2444, 3885, 1308, 3761, 2976, 2922, 2920, - 2919, 1171, 2917, 1542, 2076, 1774, 744, 1212, 1215, 1770, - 1217, 1211, 224, 739, 34, 2562, 1535, 1611, 1612, 1613, - 1694, 4421, 1499, 2836, 1615, 1811, 4074, 767, 1825, 3523, - 3519, 756, 1212, 2577, 2569, 2083, 1544, 1212, 3507, 3504, - 1653, 775, 4680, 1670, 1173, 5, 70, 1021, 1018, 2070, - 1531, 4031, 3065, 3063, 1247, 3402, 763, 3400, 1307, 1385, - 1386, 1387, 1384, 1385, 1386, 1387, 1384, 2374, 765, 3505, - 3502, 4602, 4601, 4193, 3775, 4429, 4273, 4267, 4034, 3767, - 1210, 2397, 4448, 2632, 1453, 8, 998, 7, 2940, 3474, - 3547, 2393, 764, 2728, 4700, 818, 3067, 2762, 820, 1172, - 4442, 4677, 4281, 819, 155, 4440, 1009, 4320, 3857, 3003, - 4079, 1772, 4279, 2584, 1144, 818, 4507, 1735, 820, 155, - 1552, 155, 1550, 819, 4077, 1549, 3852, 3550, 2598, 1022, - 1019, 1175, 1576, 782, 3472, 1594, 1546, 3329, 2612, 2832, - 2442, 2310, 2267, 988, 1382, 987, 989, 990, 3100, 991, - 992, 4322, 2819, 2097, 2833, 2818, 1574, 1697, 2820, 1699, - 1247, 3373, 3374, 3375, 2168, 2299, 2300, 2298, 2095, 2332, - 2102, 2103, 1610, 1666, 1823, 1169, 1667, 1559, 829, 2768, - 2767, 1651, 1652, 1932, 1933, 1641, 3529, 834, 1265, 1266, - 1232, 1170, 3647, 70, 2189, 1822, 2936, 818, 1649, 1131, - 820, 1883, 1648, 1651, 1652, 819, 4062, 1380, 70, 1168, - 70, 1255, 1259, 1261, 1263, 1268, 1167, 1273, 1269, 1270, - 1271, 1272, 1010, 3098, 1250, 1251, 1252, 1253, 1230, 1231, - 1256, 1989, 1233, 4451, 1235, 1236, 1237, 1238, 1234, 1239, - 1240, 1241, 1242, 1243, 1246, 1248, 1244, 1245, 1274, 1275, - 1276, 1277, 1278, 1279, 1280, 1281, 1283, 1282, 1284, 1285, - 1286, 1287, 1288, 1289, 1290, 1291, 1258, 1260, 1262, 1264, - 1267, 1022, 1019, 1375, 3506, 3503, 4450, 1669, 3121, 829, - 1680, 4451, 4565, 4449, 1137, 1135, 2541, 1136, 3122, 4450, - 4564, 4449, 4563, 4634, 1265, 1266, 1232, 4432, 3097, 3407, - 1221, 4608, 4609, 1575, 4577, 3645, 4553, 1249, 1362, 4556, - 3096, 1363, 2720, 4270, 2166, 1140, 3770, 1255, 1259, 1261, - 1263, 1268, 2957, 1273, 1269, 1270, 1271, 1272, 3068, 1311, - 1250, 1251, 1252, 1253, 1230, 1231, 1256, 3120, 1233, 1365, - 1235, 1236, 1237, 1238, 1234, 1239, 1240, 1241, 1242, 1243, - 1246, 1248, 1244, 1245, 1274, 1275, 1276, 1277, 1278, 1279, - 1280, 1281, 1283, 1282, 1284, 1285, 1286, 1287, 1288, 1289, - 1290, 1291, 1258, 1260, 1262, 1264, 1267, 1773, 1771, 1194, - 3412, 4061, 4672, 4673, 3355, 3093, 2796, 2797, 1145, 4063, - 2446, 1020, 1017, 3408, 3259, 3409, 183, 223, 182, 214, - 184, 4467, 3788, 1887, 1016, 2316, 183, 223, 182, 214, - 184, 1639, 3770, 1249, 3867, 2438, 4553, 752, 4435, 4436, - 4437, 4438, 752, 3874, 1310, 2306, 2770, 4118, 3435, 183, - 223, 182, 214, 184, 1141, 2443, 3580, 1862, 3128, 2777, - 3578, 3099, 1063, 771, 771, 1336, 2098, 752, 4324, 4325, - 183, 223, 182, 214, 184, 1683, 4081, 2167, 1360, 1577, - 3094, 2096, 2606, 1195, 2330, 2331, 1668, 749, 3106, 3968, - 219, 3323, 1831, 4579, 944, 3777, 1846, 3433, 3066, 1377, - 219, 1222, 183, 223, 182, 214, 184, 2726, 2967, 768, - 768, 768, 210, 4607, 3575, 3576, 1143, 1378, 1379, 1350, - 4032, 1534, 3401, 219, 1867, 2773, 2774, 1871, 3574, 3317, - 3577, 2772, 183, 223, 4330, 1428, 1317, 4115, 3585, 2837, - 1361, 3984, 1373, 1374, 219, 2086, 4397, 4455, 1661, 4075, - 3586, 1870, 3075, 183, 223, 182, 214, 184, 2780, 4333, - 3791, 1372, 1171, 4078, 3439, 3072, 3649, 736, 1188, 1183, - 1178, 1182, 1186, 1308, 1308, 3869, 219, 1886, 1885, 3980, - 1308, 1755, 153, 1310, 3674, 3675, 1309, 2449, 2451, 2452, - 3673, 1974, 1551, 1342, 1664, 1665, 1191, 1142, 3423, 2265, - 1181, 1548, 2761, 4459, 2764, 1173, 219, 3420, 4311, 4144, - 3676, 2469, 3677, 3679, 3678, 2763, 4091, 2309, 3891, 1314, - 3754, 1364, 1462, 3095, 3599, 3124, 3613, 219, 2433, 4487, - 1212, 4360, 1212, 1212, 4482, 1212, 3169, 1367, 1325, 1328, - 1368, 1171, 3834, 1212, 4352, 1212, 1139, 1308, 4083, 4084, - 4085, 1189, 1651, 1652, 3836, 1257, 3572, 155, 155, 155, - 1172, 2445, 181, 212, 221, 213, 2918, 4323, 1370, 4280, - 1775, 1023, 2619, 1192, 1872, 1013, 2706, 766, 766, 766, - 1193, 4261, 2686, 2709, 1173, 3327, 211, 773, 3586, 772, - 2614, 3973, 1640, 821, 822, 823, 824, 825, 1869, 3513, - 1320, 4472, 4489, 1537, 1539, 998, 1543, 3925, 3644, 1329, - 1300, 1330, 1303, 821, 822, 823, 824, 825, 3064, 1179, - 4495, 1299, 1563, 1339, 1542, 1463, 1566, 767, 767, 767, - 1824, 1573, 1542, 1547, 4080, 1558, 2266, 1170, 3214, 1420, - 2708, 3932, 1514, 1190, 3848, 1519, 70, 70, 70, 1341, - 1014, 4466, 1334, 1335, 2595, 1138, 763, 763, 763, 752, - 752, 1257, 1628, 1074, 1429, 1651, 1652, 4317, 765, 765, - 765, 4099, 1424, 1425, 1426, 1427, 1065, 4289, 1066, 4290, - 1227, 1180, 3682, 1325, 1328, 3586, 3845, 1366, 3210, 3211, - 3542, 3214, 764, 764, 764, 821, 822, 823, 824, 825, - 4578, 2760, 1893, 1896, 1897, 3989, 2707, 1970, 4188, 1313, - 1315, 1318, 3847, 1894, 1967, 4706, 4361, 1868, 1969, 1966, - 1968, 1972, 1973, 2317, 3436, 1015, 1971, 1371, 752, 4353, - 1679, 3581, 2738, 1685, 2693, 3127, 2776, 752, 3650, 2737, - 4047, 719, 719, 1355, 4326, 4292, 1357, 1332, 3134, 1369, - 2161, 719, 719, 1707, 1329, 1722, 1722, 2837, 752, 2798, - 1647, 3260, 1187, 3261, 3262, 3324, 2758, 2759, 1706, 1340, - 1474, 1475, 1327, 1326, 1358, 4291, 1626, 2450, 1625, 4119, - 771, 1751, 738, 1624, 2784, 1422, 1227, 4496, 1763, 1724, - 3131, 3132, 1720, 1720, 1643, 1642, 4592, 2438, 4183, 1184, - 1521, 2307, 1185, 240, 4339, 3130, 4689, 4374, 3573, 3653, - 1599, 1177, 719, 1863, 3496, 1729, 769, 2729, 3906, 3674, - 3675, 2791, 2795, 2796, 2797, 2792, 2801, 2793, 2799, 2686, - 4029, 2794, 1681, 2800, 3288, 1319, 1569, 1570, 1571, 769, - 4177, 1568, 1580, 1582, 1583, 1584, 1585, 782, 1587, 1581, - 4550, 3912, 1693, 3831, 1593, 1554, 1316, 1586, 1419, 1418, - 769, 3703, 3669, 1520, 1518, 2963, 1684, 1347, 1012, 2020, - 2022, 2021, 71, 1977, 1978, 1979, 1980, 1981, 1982, 1975, - 1976, 2824, 71, 1807, 1716, 1717, 2766, 2724, 1812, 2567, - 1578, 3860, 769, 1351, 1556, 2435, 2305, 2692, 1821, 3681, - 2282, 1579, 2694, 3368, 3370, 71, 1609, 1565, 1857, 220, - 1858, 2618, 1635, 1636, 3384, 3385, 3999, 1327, 1326, 1353, - 1196, 3082, 1844, 2703, 1176, 3718, 71, 1847, 3705, 1603, - 3438, 1592, 1356, 1359, 1600, 1075, 1591, 1722, 1590, 1722, - 1310, 1589, 1146, 769, 776, 1814, 3311, 4191, 1630, 1634, - 1634, 1634, 2019, 1701, 1703, 1352, 2695, 1064, 71, 4690, - 183, 223, 4373, 1714, 1715, 3164, 2596, 1346, 768, 3257, - 1731, 768, 768, 2696, 745, 1630, 1630, 1655, 1671, 1672, - 1658, 3823, 2271, 2269, 3160, 1856, 2461, 2270, 3670, 1752, - 2079, 1782, 1895, 2955, 747, 1621, 748, 1607, 1705, 1785, - 4591, 1788, 1789, 1077, 1078, 1079, 3090, 1796, 1797, 71, - 1722, 2588, 155, 1790, 1791, 3141, 3147, 3148, 3149, 3142, - 3146, 3143, 3145, 3144, 1776, 2105, 1805, 1310, 1951, 1802, - 744, 1618, 1806, 1730, 3158, 1562, 1354, 1801, 1743, 1627, - 2106, 1983, 1984, 3838, 2002, 1988, 1637, 1935, 4285, 1819, - 183, 223, 4445, 2003, 1656, 1657, 1764, 1659, 1660, 4184, - 4185, 1662, 2447, 2448, 1765, 1035, 2010, 2587, 2012, 1749, - 2013, 2014, 2015, 2791, 2795, 2796, 2797, 2792, 2801, 2793, - 2799, 2084, 1882, 2794, 3161, 2800, 4201, 4202, 4203, 4207, - 4205, 4206, 4208, 4209, 4210, 4204, 2104, 1553, 1879, 3369, - 3781, 70, 1024, 4687, 4688, 2750, 1568, 1025, 4289, 3913, - 4290, 3289, 3291, 3292, 3293, 3290, 155, 4179, 4150, 155, - 155, 4178, 4702, 1310, 219, 4708, 4284, 4260, 2693, 2696, - 2697, 1860, 1898, 155, 1620, 2088, 766, 1876, 2089, 766, - 766, 2092, 1171, 4696, 2078, 3186, 752, 752, 752, 1817, - 1147, 1986, 1800, 1851, 2363, 2107, 2109, 4560, 2110, 2702, - 2112, 2113, 2114, 2700, 1829, 738, 1751, 1832, 2610, 2061, - 1028, 2122, 3083, 1722, 2128, 2129, 4292, 2131, 1685, 752, - 1854, 3725, 1849, 1383, 752, 1173, 767, 1722, 2001, 767, - 767, 1873, 1878, 1074, 1841, 1853, 2157, 1848, 2590, 2589, - 1555, 1557, 2064, 1855, 3172, 70, 4291, 2440, 70, 70, - 1838, 1839, 3279, 3280, 1722, 763, 3671, 4683, 763, 763, - 1685, 1028, 70, 1132, 1132, 762, 2837, 765, 4697, 1919, - 765, 765, 1852, 1032, 1560, 1561, 3720, 1865, 1030, 1029, - 1420, 2080, 1915, 1916, 1850, 2188, 1926, 1927, 2693, 2696, - 2149, 764, 1685, 2505, 764, 764, 2504, 2197, 2197, 1347, - 1685, 2556, 1685, 1685, 2432, 2072, 752, 752, 1830, 2264, - 3595, 1833, 1834, 2122, 2275, 2067, 2798, 1722, 2279, 2280, - 3724, 1620, 1864, 2295, 1027, 719, 2697, 1383, 2803, 1030, - 1029, 2692, 2686, 2691, 4647, 2689, 2694, 2130, 2609, 719, - 3187, 1722, 4648, 1302, 3163, 1132, 2409, 2681, 3112, 2361, - 3863, 2132, 2192, 2432, 1843, 2016, 2017, 4620, 1134, 1134, - 4617, 1133, 1133, 1842, 4616, 1031, 1620, 2337, 2339, 752, - 2122, 1722, 2802, 2345, 3790, 752, 752, 752, 780, 780, - 3630, 1383, 2432, 3113, 3114, 2355, 2153, 2357, 2358, 2359, - 2695, 3278, 2723, 2365, 2219, 1002, 1003, 1004, 1005, 2062, - 240, 2962, 4610, 240, 240, 4588, 240, 2068, 3686, 4543, - 2273, 1345, 2116, 2127, 3684, 2118, 2119, 2120, 1302, 4648, - 1385, 1386, 1387, 1384, 2333, 2193, 1422, 2143, 2134, 2135, - 2136, 2137, 1347, 2200, 1992, 1993, 1994, 2077, 155, 2081, - 1134, 4542, 4621, 1133, 2085, 4618, 2697, 2008, 4517, 2440, - 2009, 2692, 2686, 2691, 2169, 2689, 2694, 2416, 2117, 2311, - 2325, 2326, 3596, 2302, 1347, 2304, 183, 223, 3187, 2028, - 2029, 3553, 2347, 2348, 2349, 3725, 2323, 2324, 3469, 3511, - 2154, 3509, 2666, 2158, 3177, 4490, 2408, 2479, 2163, 2164, - 4589, 2181, 2157, 2344, 1383, 2803, 1722, 2429, 2373, 2060, - 2318, 2376, 2377, 2186, 2379, 2383, 1383, 2199, 2177, 2396, - 2695, 2407, 2803, 2962, 3387, 1630, 2404, 2278, 2171, 4285, - 3725, 2201, 2202, 4286, 3069, 2411, 1383, 768, 2941, 1634, - 2431, 2297, 2294, 2479, 4478, 2196, 2198, 2296, 4419, 4418, - 2406, 1634, 2431, 1344, 2679, 2561, 1903, 1904, 1905, 1906, - 1907, 1908, 1909, 1910, 1911, 1912, 1913, 1914, 2423, 2478, - 2283, 1007, 2162, 2272, 1928, 1929, 2277, 2301, 2798, 2303, - 2440, 1876, 2312, 1385, 1386, 1387, 1384, 1385, 1386, 1387, - 1384, 1515, 2555, 2554, 2180, 2514, 2513, 2512, 1952, 1385, - 1386, 1387, 1384, 1987, 2422, 2328, 1207, 1208, 1209, 2390, - 2187, 2281, 2336, 2190, 2191, 2343, 1602, 1938, 1708, 155, - 2172, 2173, 155, 155, 3499, 155, 2011, 2350, 2351, 4479, - 2342, 4389, 4388, 4420, 2644, 4387, 4715, 2182, 2183, 2370, - 1206, 2665, 1345, 1203, 2176, 3468, 2178, 2179, 4698, 4416, - 4386, 1385, 1386, 1387, 1384, 4249, 4364, 2477, 2194, 3921, - 2185, 3392, 3189, 863, 873, 2388, 4363, 3078, 4336, 155, - 2965, 2964, 1171, 864, 2956, 865, 869, 872, 868, 866, - 867, 2023, 2024, 2025, 2026, 155, 2157, 2030, 2031, 2032, - 2033, 2035, 2036, 2037, 2038, 2039, 2040, 2041, 2042, 2043, - 2044, 2045, 2046, 2047, 2673, 766, 2500, 2420, 70, 3500, - 4305, 70, 70, 4302, 70, 1173, 2479, 2479, 3497, 2568, - 2479, 2570, 2483, 2572, 2573, 3994, 2421, 2576, 3934, 2368, - 2353, 2418, 2409, 2424, 2467, 2479, 752, 1685, 752, 1685, - 870, 2440, 4114, 3893, 2437, 1385, 1386, 1387, 1384, 2591, - 3815, 2440, 2537, 2479, 2082, 767, 828, 1826, 1437, 752, - 752, 752, 2481, 3811, 2462, 2607, 2453, 1331, 1297, 3694, - 1420, 871, 1292, 3873, 70, 752, 752, 752, 752, 3458, - 2540, 2542, 2543, 2544, 763, 2546, 3417, 4247, 1919, 2455, - 3363, 2002, 2002, 2640, 2382, 1383, 765, 3179, 2644, 2648, - 1026, 2651, 3992, 3498, 2456, 2457, 2471, 2653, 2654, 2655, - 2837, 2658, 1685, 3935, 2549, 2999, 3000, 2466, 2465, 2327, - 764, 1400, 2993, 3635, 1200, 1201, 1202, 1205, 3894, 1204, - 3430, 1002, 1003, 1004, 1005, 3816, 3174, 2419, 2917, 2547, - 1685, 1385, 1386, 1387, 1384, 1780, 1779, 3759, 3812, 1273, - 1269, 1270, 1271, 1272, 3695, 2149, 2998, 2715, 2997, 2996, - 2994, 1413, 3044, 1417, 1383, 4051, 1385, 1386, 1387, 1384, - 2581, 3175, 2583, 3032, 3024, 2803, 2978, 1419, 1418, 1414, - 1416, 1412, 3180, 1415, 1399, 1398, 1408, 1409, 1410, 1411, - 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1400, 4483, 2550, - 2960, 2932, 1171, 2652, 2670, 1385, 1386, 1387, 1384, 1616, - 2672, 2930, 2674, 1617, 1733, 2484, 2454, 2722, 1631, 1991, - 1990, 3175, 752, 2197, 2548, 1385, 1386, 1387, 1384, 2928, - 2558, 2807, 2807, 2295, 2807, 2635, 2926, 3614, 2643, 2995, - 1385, 1386, 1387, 1384, 4484, 1173, 2571, 2644, 2557, 2521, - 2575, 2520, 2503, 2494, 719, 719, 2493, 2492, 1383, 1383, - 2480, 1383, 1310, 3046, 1991, 1990, 1033, 2439, 1722, 752, - 4354, 1712, 1835, 2675, 4709, 1710, 2599, 2985, 4151, 2089, - 3909, 3907, 1713, 2553, 4676, 2644, 2933, 752, 2721, 2685, - 3531, 4050, 2684, 1310, 2899, 738, 2931, 1007, 2475, 4457, - 1172, 1462, 1763, 155, 2295, 2830, 2911, 2907, 2371, 2909, - 2765, 2641, 240, 3534, 2927, 4411, 2678, 4335, 3615, 2515, - 2516, 2927, 2518, 2644, 4152, 2903, 3910, 3908, 4277, 2525, - 1663, 2659, 2811, 2556, 1383, 2034, 1383, 1383, 1383, 1632, - 4219, 1383, 1383, 4181, 1171, 2479, 4180, 2002, 4166, 2002, - 752, 2809, 2440, 2813, 2952, 2671, 2821, 1836, 2822, 3104, - 4355, 4122, 2958, 3884, 3616, 2429, 3726, 3716, 2815, 3708, - 2698, 2699, 1722, 2704, 1722, 1942, 1722, 2827, 2828, 1709, - 2027, 1310, 2458, 2459, 2839, 3696, 2844, 1173, 3590, 2977, - 3320, 3531, 3319, 1634, 1401, 1402, 1403, 1404, 1405, 1406, - 1407, 1400, 3178, 2906, 1463, 1925, 4356, 3139, 3074, 2975, - 2912, 2968, 1403, 1404, 1405, 1406, 1407, 1400, 2826, 1722, - 1310, 1922, 1924, 1921, 3006, 1923, 2574, 2414, 2413, 2660, - 2661, 2775, 2412, 1616, 2945, 2845, 2781, 1617, 1596, 2663, - 2664, 3015, 2810, 1595, 1312, 1942, 1722, 2472, 1768, 2816, - 2371, 3393, 3001, 1387, 1384, 4562, 1720, 3534, 1171, 1385, - 1386, 1387, 1384, 4196, 2111, 4304, 1701, 1703, 4303, 1384, - 3762, 1385, 1386, 1387, 1384, 2831, 4195, 3617, 3249, 3016, - 2667, 3247, 2921, 1720, 1385, 1386, 1387, 1384, 3226, 3224, - 4172, 2635, 2662, 3760, 830, 2834, 4705, 2668, 4532, 4533, - 2669, 1173, 2900, 4391, 4392, 2939, 4625, 2905, 4123, 4124, - 3076, 3021, 3022, 2294, 3056, 3080, 3057, 4587, 3084, 4586, - 4116, 155, 1439, 2972, 4535, 752, 752, 752, 1385, 1386, - 1387, 1384, 2988, 3533, 2990, 1438, 4534, 2987, 1385, 1386, - 1387, 1384, 1310, 2974, 4531, 2969, 4530, 2913, 2948, 4529, - 1722, 2946, 3871, 1685, 4528, 2983, 1172, 3138, 2937, 1685, - 2275, 4704, 2961, 4526, 2496, 3300, 2959, 2006, 3027, 3028, - 4525, 3004, 1876, 2966, 3033, 1820, 3298, 3048, 3296, 3049, - 4117, 3051, 2007, 3053, 3054, 3182, 3185, 1385, 1386, 1387, - 1384, 3060, 4524, 2979, 2980, 1768, 3191, 1398, 1408, 1409, - 1410, 1411, 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1400, - 70, 4523, 3872, 4522, 3201, 3002, 2992, 2904, 1385, 1386, - 1387, 1384, 3445, 3159, 1310, 3299, 4521, 1769, 1385, 1386, - 1387, 1384, 3223, 3285, 2495, 2844, 3297, 4519, 3295, 1310, - 1310, 1310, 2197, 2982, 3153, 1310, 4518, 3233, 3234, 3235, - 3236, 1310, 3243, 4485, 3244, 3245, 3156, 3246, 3102, 3248, - 3170, 1385, 1386, 1387, 1384, 3061, 4377, 4367, 4357, 4329, - 3243, 1391, 1392, 1393, 1394, 1395, 1396, 1397, 1389, 4301, - 4650, 4268, 2807, 4190, 2845, 4154, 3135, 4153, 4707, 3202, - 3926, 3014, 3154, 3284, 3911, 3877, 3301, 3870, 3853, 3579, - 1385, 1386, 1387, 1384, 3460, 3426, 2219, 1385, 1386, 1387, - 1384, 3204, 3405, 719, 3116, 3404, 3118, 3309, 3283, 3192, - 3282, 2275, 3281, 3273, 3267, 1310, 2295, 2295, 2295, 2295, - 2295, 2295, 1213, 1214, 3266, 3265, 3115, 1218, 3264, 3133, - 3190, 3070, 2934, 1310, 2295, 2823, 2560, 2807, 2392, 3162, - 2127, 2391, 3306, 2389, 2385, 3221, 2384, 2334, 2487, 3221, - 2094, 3217, 2091, 3371, 1827, 1722, 3218, 3459, 3181, 1533, - 4701, 155, 3184, 8, 4597, 7, 3228, 3883, 752, 752, - 3007, 3218, 3229, 3230, 2943, 2944, 3565, 3232, 1385, 1386, - 1387, 1384, 155, 3239, 1385, 1386, 1387, 1384, 4699, 3203, - 3206, 1385, 1386, 1387, 1384, 3219, 4039, 2727, 3312, 2476, - 2730, 2731, 2732, 2733, 2734, 2735, 2736, 4327, 4328, 2739, - 2740, 2741, 2742, 2743, 2744, 2745, 2746, 2747, 2748, 2749, - 3231, 2751, 2752, 2753, 2754, 2755, 3225, 2756, 3359, 3222, - 4674, 1295, 4640, 3389, 4504, 4574, 4572, 3325, 4069, 3263, - 4309, 4548, 3275, 1704, 240, 1385, 1386, 1387, 1384, 240, - 1385, 1386, 1387, 1384, 4469, 4066, 4127, 3337, 4463, 3372, - 4662, 1385, 1386, 1387, 1384, 1385, 1386, 1387, 1384, 3388, - 4454, 70, 4452, 4439, 4430, 3337, 3315, 1385, 1386, 1387, - 1384, 3321, 1385, 1386, 1387, 1384, 4406, 3425, 4405, 4065, - 1294, 4396, 4395, 1722, 4502, 4381, 3432, 3338, 3339, 3340, - 3341, 3342, 3343, 3356, 4376, 4375, 3361, 3362, 3360, 3194, - 4332, 4054, 4316, 4314, 3197, 3318, 1385, 1386, 1387, 1384, - 4300, 3379, 4053, 4269, 3376, 2294, 2294, 2294, 2294, 2294, - 2294, 3380, 4174, 4131, 4120, 3419, 4052, 3010, 1385, 1386, - 1387, 1384, 3394, 2294, 3193, 4104, 4103, 3398, 4101, 1385, - 1386, 1387, 1384, 3198, 3199, 3017, 4096, 1789, 4094, 4073, - 1796, 1797, 4072, 1385, 1386, 1387, 1384, 1790, 1791, 3977, - 4071, 4068, 3200, 1805, 4498, 4067, 1802, 4041, 4037, 1806, - 1399, 1398, 1408, 1409, 1410, 1411, 1401, 1402, 1403, 1404, - 1405, 1406, 1407, 1400, 4035, 4005, 1385, 1386, 1387, 1384, - 4002, 3996, 3396, 3305, 3865, 3855, 3517, 3395, 3803, 3520, - 3840, 2507, 3824, 4654, 3524, 3429, 752, 1685, 3802, 3800, - 3794, 3776, 3434, 3737, 3714, 3536, 3538, 3539, 3541, 3713, - 3543, 3544, 3711, 3414, 3410, 1385, 1386, 1387, 1384, 3710, - 3697, 3692, 1310, 3691, 3591, 3551, 3545, 3535, 1310, 3525, - 3518, 3516, 3428, 155, 3568, 3570, 2565, 3440, 155, 3437, - 3424, 3475, 3476, 3442, 3441, 3583, 3403, 3477, 3478, 3479, - 3480, 752, 3481, 3482, 3483, 3484, 3485, 3486, 3487, 3488, - 3489, 3490, 3491, 3492, 3378, 3457, 3598, 3313, 3602, 1310, - 3453, 3454, 752, 3451, 752, 2275, 1310, 1310, 2981, 3450, - 3310, 3452, 3448, 3449, 3307, 3294, 2002, 3286, 2002, 3276, - 3274, 3627, 3270, 3269, 3268, 3105, 3510, 2295, 2648, 3091, - 3634, 3079, 1399, 1398, 1408, 1409, 1410, 1411, 1401, 1402, - 1403, 1404, 1405, 1406, 1407, 1400, 3071, 944, 943, 2715, - 2950, 3594, 70, 3796, 2938, 2901, 2592, 70, 4306, 2579, - 3527, 3659, 3605, 3662, 3587, 3662, 3662, 3515, 3597, 3611, - 1310, 3514, 2578, 2395, 2387, 3501, 3622, 2195, 3153, 2124, - 1385, 1386, 1387, 1384, 2093, 2090, 2075, 2074, 3687, 3123, - 3470, 1828, 1470, 3683, 3554, 3571, 1722, 1722, 1466, 1465, - 3218, 3156, 1385, 1386, 1387, 1384, 3624, 3642, 2064, 3464, - 1171, 3637, 1298, 3626, 3646, 3648, 1011, 1385, 1386, 1387, - 1384, 4296, 4295, 4282, 4278, 4102, 3632, 4070, 4048, 3688, - 3689, 4016, 3997, 1720, 1720, 2474, 1385, 1386, 1387, 1384, - 3914, 3218, 3593, 752, 3902, 183, 223, 3604, 3218, 3218, - 3901, 3897, 3657, 1173, 3609, 3610, 3862, 3568, 3820, 3618, - 3623, 3620, 3818, 3625, 3463, 3817, 3814, 3813, 3461, 3658, - 1685, 3633, 3801, 2275, 2275, 3667, 3799, 3765, 3764, 3629, - 3641, 3749, 183, 223, 3748, 3628, 2685, 3555, 3552, 2684, - 3508, 1385, 1386, 1387, 1384, 1385, 1386, 1387, 1384, 3466, - 3663, 3664, 2151, 3455, 3447, 3255, 3256, 3668, 1172, 3446, - 155, 3444, 3218, 1385, 1386, 1387, 1384, 155, 3045, 219, - 3271, 3272, 3043, 3386, 155, 2929, 1388, 2925, 1310, 3685, - 2924, 2923, 2148, 3006, 1421, 2293, 2294, 3042, 2526, 2519, - 3693, 3763, 2511, 1431, 3041, 1385, 1386, 1387, 1384, 1385, - 1386, 1387, 1384, 3316, 2510, 155, 2150, 4516, 2509, 2508, - 3701, 2506, 2502, 2501, 1385, 1386, 1387, 1384, 2499, 1441, - 3040, 1385, 1386, 1387, 1384, 3665, 2490, 3039, 2486, 752, - 2485, 2394, 3698, 183, 223, 2053, 3706, 3721, 3722, 3707, - 3709, 3712, 2051, 2050, 2049, 3715, 2048, 1385, 1386, 1387, - 1384, 3038, 2005, 3719, 1385, 1386, 1387, 1384, 3733, 2004, - 3734, 3037, 750, 1995, 1734, 3036, 1732, 183, 223, 3782, - 3216, 2844, 4661, 4624, 3784, 4541, 4503, 223, 1385, 1386, - 1387, 1384, 3742, 3621, 3745, 3746, 3747, 1460, 1385, 1386, - 1387, 1384, 1385, 1386, 1387, 1384, 3035, 4497, 3783, 4425, - 3636, 4422, 3752, 4404, 4385, 3638, 3639, 219, 3702, 4378, - 3826, 4263, 4262, 4214, 3827, 4194, 2365, 153, 4514, 3034, - 2845, 3773, 4192, 1385, 1386, 1387, 1384, 3031, 3841, 4187, - 3843, 3780, 4165, 4148, 4017, 3849, 3030, 781, 4014, 3786, - 3804, 219, 3787, 3029, 3975, 3640, 1385, 1386, 1387, 1384, - 219, 3974, 3971, 3792, 1385, 1386, 1387, 1384, 3023, 3837, - 3970, 3850, 3933, 1385, 1386, 1387, 1384, 3011, 1070, 3930, - 1385, 1386, 1387, 1384, 3005, 3928, 752, 2275, 3886, 3806, - 3839, 3808, 3844, 3810, 3846, 1385, 1386, 1387, 1384, 3835, - 3892, 3832, 3548, 3456, 1385, 1386, 1387, 1384, 2984, 3900, - 1784, 1385, 1386, 1387, 1384, 2552, 1795, 1786, 3861, 1801, - 1804, 1792, 1781, 1605, 3348, 3864, 3308, 155, 3821, 3302, - 2807, 2295, 3918, 3825, 3227, 1385, 1386, 1387, 1384, 3173, - 3166, 3829, 1385, 1386, 1387, 1384, 750, 3165, 3859, 3157, - 3881, 3117, 3723, 3047, 3936, 2825, 2757, 1310, 2642, 2601, - 3701, 3854, 2600, 2559, 3858, 2551, 3659, 1920, 219, 2352, - 1310, 2152, 2071, 1861, 4512, 1793, 3741, 1532, 1517, 3890, - 1513, 1512, 1511, 3878, 1510, 1310, 2545, 3991, 1509, 1508, - 1507, 1722, 1385, 1386, 1387, 1384, 1506, 3986, 3987, 3988, - 1505, 3880, 1504, 1503, 1502, 1501, 4000, 1500, 1499, 3920, - 1498, 3915, 1497, 1385, 1386, 1387, 1384, 4510, 1937, 752, - 1496, 2275, 1495, 1494, 3993, 2295, 1310, 3969, 1720, 1493, - 3917, 1408, 1409, 1410, 1411, 1401, 1402, 1403, 1404, 1405, - 1406, 1407, 1400, 1492, 3960, 1385, 1386, 1387, 1384, 3916, - 1491, 3923, 1762, 1490, 1489, 4023, 1488, 1487, 3702, 1486, - 1485, 240, 223, 182, 214, 184, 155, 1484, 1483, 1482, - 1481, 1480, 1479, 155, 3976, 3978, 3981, 1478, 1477, 4009, - 4006, 1476, 1473, 1472, 1471, 1469, 1468, 1467, 1464, 3990, - 1457, 1456, 1454, 4022, 1453, 1452, 1451, 1450, 3995, 3937, - 1449, 1448, 1447, 1446, 1445, 3998, 1444, 1443, 4001, 1442, - 1436, 1435, 3979, 1434, 1433, 1432, 4007, 4008, 4011, 4004, - 1349, 4012, 1296, 4003, 4010, 3729, 3730, 3239, 3972, 2657, - 2616, 1337, 3358, 4652, 4606, 219, 2157, 3732, 3704, 4086, - 2294, 3314, 3140, 4092, 2838, 2628, 1614, 4046, 1348, 4098, - 3346, 3353, 3351, 3349, 3740, 3739, 3354, 3352, 3350, 4030, - 4019, 3345, 3738, 3735, 1310, 4040, 3357, 3344, 3337, 2489, - 4020, 4043, 1399, 1398, 1408, 1409, 1410, 1411, 1401, 1402, - 1403, 1404, 1405, 1406, 1407, 1400, 4561, 1310, 1722, 1722, - 138, 73, 4132, 72, 69, 3602, 4441, 4095, 4170, 4097, - 3418, 3176, 4082, 1597, 2145, 2146, 3589, 4055, 3899, 4056, - 4140, 4057, 3416, 1310, 4076, 4140, 2140, 2141, 2142, 2725, - 4018, 4129, 3655, 3171, 3656, 1720, 1935, 3778, 3779, 1310, - 4159, 1310, 4128, 3982, 2294, 4134, 4135, 3753, 1888, 1889, - 1890, 1891, 1892, 4162, 2256, 4164, 1777, 4109, 1722, 3919, - 4111, 4089, 4110, 1816, 4106, 2943, 2944, 3922, 2973, 4137, - 4130, 740, 741, 2586, 742, 743, 4121, 2585, 1813, 752, - 155, 1310, 1310, 2593, 2354, 1310, 1310, 2268, 4133, 4147, - 4142, 1343, 4146, 1939, 4382, 1935, 4100, 1943, 1944, 1945, - 1946, 4158, 2411, 3920, 4216, 1421, 3563, 3556, 1985, 3205, - 4248, 3167, 2677, 4218, 2626, 2155, 3218, 1996, 4155, 3969, - 4171, 4168, 4211, 2115, 4665, 2157, 4198, 4199, 4255, 4175, - 4212, 4213, 3251, 1991, 1990, 4380, 3960, 1528, 1529, 3252, - 3253, 3254, 4264, 4265, 1526, 1527, 1524, 1525, 1522, 1523, - 3690, 2778, 2771, 2276, 1674, 1673, 4143, 1722, 1376, 3702, - 2415, 3751, 3744, 2594, 2417, 3337, 2160, 1623, 1622, 70, - 1588, 2052, 1646, 2054, 2055, 2056, 2057, 2058, 2650, 4631, - 4113, 1882, 2065, 1882, 4251, 4297, 4298, 4250, 752, 4112, - 4276, 4253, 4288, 4629, 1720, 4580, 2971, 4558, 4557, 4555, - 4473, 4426, 4310, 4258, 4312, 2970, 746, 4257, 4160, 4036, - 3805, 3772, 3771, 3757, 2399, 2710, 4271, 155, 2680, 4275, - 1818, 3756, 3391, 4025, 4283, 1620, 4656, 4655, 4656, 4093, - 4313, 4287, 4315, 3842, 3828, 3427, 3086, 3085, 3077, 2902, - 2488, 1333, 1304, 4042, 4655, 4189, 4021, 4635, 1002, 1003, - 1004, 1005, 1070, 1302, 4108, 4344, 1302, 1305, 3904, 4349, - 3413, 4318, 4342, 2620, 1809, 1638, 81, 4293, 4294, 4319, - 2, 4064, 4368, 3927, 4678, 3929, 1310, 4679, 1, 3062, - 2069, 1530, 1338, 1006, 1001, 2165, 1698, 2817, 2329, 1726, - 4366, 4337, 4372, 2073, 4331, 1008, 3364, 3365, 3743, 3367, - 2335, 1608, 3092, 2436, 3326, 4088, 2769, 2605, 3582, 4343, - 4340, 2184, 4346, 4345, 4046, 1606, 1076, 1997, 1840, 1324, - 4362, 1837, 4358, 1323, 1321, 1310, 1399, 1398, 1408, 1409, - 1410, 1411, 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1400, - 1940, 2018, 877, 4156, 4157, 2631, 3303, 3887, 3888, 3889, - 3277, 4254, 4379, 4664, 4693, 3895, 3896, 1722, 4623, 4667, - 4417, 1859, 861, 4549, 3774, 3411, 4431, 4627, 4433, 4274, - 4390, 2441, 1381, 1413, 3619, 1417, 2065, 1102, 921, 889, - 1455, 2065, 2065, 2403, 3473, 3471, 888, 3875, 3129, 4259, - 4414, 1414, 1416, 1412, 1720, 1415, 1399, 1398, 1408, 1409, - 1410, 1411, 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1400, - 3383, 4351, 1103, 2381, 4453, 4428, 4447, 4272, 1882, 1778, - 1783, 2676, 4458, 4359, 4493, 4169, 3651, 3213, 1808, 4427, - 4488, 4465, 3931, 2372, 4060, 4058, 2375, 4252, 4059, 2378, - 788, 2308, 2380, 717, 1156, 4215, 2627, 2656, 4220, 4384, - 4460, 1048, 4461, 3856, 2615, 1049, 1041, 3151, 155, 3150, - 1899, 1390, 1918, 3494, 4474, 3495, 4470, 1430, 832, 2470, - 3126, 3954, 3377, 80, 79, 78, 77, 248, 880, 4462, - 247, 4307, 4125, 2402, 4544, 4669, 858, 857, 856, 855, - 854, 4468, 853, 4492, 2789, 2790, 1310, 2788, 2786, 2785, - 2290, 4477, 2289, 4476, 3390, 3755, 2360, 2362, 4520, 3600, - 3242, 3983, 3237, 2208, 2206, 1310, 4509, 4511, 4513, 4515, - 1689, 2705, 2712, 2205, 4491, 4603, 1722, 4537, 3793, 4527, - 4049, 4538, 4500, 4486, 3964, 4505, 4545, 4506, 4186, 3287, - 3943, 4508, 4045, 2139, 2701, 2225, 3258, 2222, 2221, 3250, - 4546, 4182, 4176, 2253, 4347, 4139, 3938, 3939, 3945, 4536, - 1254, 4167, 2625, 1720, 1228, 1223, 1225, 1226, 1224, 2991, - 4573, 4173, 3717, 2682, 1070, 1601, 3558, 3111, 4554, 4547, - 3110, 3955, 3108, 4552, 3107, 1572, 1722, 4464, 4576, 4105, - 4349, 4570, 2843, 2841, 3946, 1293, 4567, 4569, 3731, 4575, - 4566, 4568, 4571, 2468, 3727, 3941, 4590, 2473, 4217, 3528, - 3966, 3967, 4598, 4581, 1538, 2482, 3942, 1536, 2639, 4582, - 3736, 4583, 3347, 1720, 2400, 3415, 2291, 2287, 2286, 1198, - 4584, 4585, 1197, 1759, 3833, 3898, 48, 3328, 2779, 4321, - 2144, 1042, 2613, 1677, 117, 42, 133, 116, 201, 63, - 200, 62, 1691, 18, 2491, 131, 3947, 1882, 4615, 4611, - 4619, 4612, 2498, 4613, 198, 4614, 61, 47, 46, 196, - 111, 110, 109, 1728, 108, 130, 195, 60, 232, 4630, - 231, 4632, 4633, 234, 233, 230, 4622, 4628, 2914, 4626, - 2517, 1310, 2915, 229, 4447, 2522, 2523, 2524, 1766, 4636, - 2527, 2528, 2529, 2530, 2531, 2532, 2533, 2534, 2535, 2536, - 4639, 2538, 2539, 4372, 4637, 4643, 4638, 228, 4559, 4145, - 4646, 4645, 4644, 4540, 4649, 996, 1247, 45, 4423, 4424, - 44, 4653, 4663, 4651, 202, 4671, 43, 118, 4670, 64, - 41, 4657, 4658, 4659, 4660, 4246, 40, 2649, 3546, 2159, - 3851, 3103, 2597, 1310, 39, 4675, 35, 13, 12, 36, - 23, 22, 1845, 21, 27, 4682, 33, 4681, 4492, 4684, - 4685, 32, 148, 3965, 4691, 2691, 147, 4695, 31, 146, - 4692, 145, 144, 143, 142, 141, 140, 30, 20, 55, - 54, 53, 52, 1675, 1676, 51, 1678, 50, 4703, 1682, - 3951, 1686, 1687, 1688, 9, 136, 134, 129, 4671, 4711, - 4163, 4670, 4710, 127, 29, 128, 125, 126, 121, 120, - 4695, 4712, 3948, 3952, 3950, 3949, 4716, 119, 114, 112, - 92, 91, 90, 4641, 1736, 1737, 1738, 1739, 1740, 1741, - 1742, 105, 1744, 1745, 1746, 1747, 1748, 104, 103, 102, - 1754, 101, 1756, 1757, 1758, 100, 98, 99, 1101, 89, - 1265, 1266, 1232, 88, 1399, 1398, 1408, 1409, 1410, 1411, - 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1400, 87, 86, - 3958, 3959, 85, 1255, 1259, 1261, 1263, 1268, 122, 1273, - 1269, 1270, 1271, 1272, 107, 1882, 1250, 1251, 1252, 1253, - 1230, 1231, 1256, 115, 1233, 113, 1235, 1236, 1237, 1238, - 1234, 1239, 1240, 1241, 1242, 1243, 1246, 1248, 1244, 1245, - 1274, 1275, 1276, 1277, 1278, 1279, 1280, 1281, 1283, 1282, - 1284, 1285, 1286, 1287, 1288, 1289, 1290, 1291, 1258, 1260, - 1262, 1264, 1267, 96, 106, 97, 95, 3968, 4161, 94, - 93, 84, 83, 82, 124, 123, 135, 203, 65, 180, - 3944, 179, 178, 3957, 177, 176, 174, 2065, 175, 2065, - 173, 172, 4225, 171, 4393, 4394, 170, 169, 168, 1249, - 56, 4398, 4399, 4400, 4401, 4402, 4403, 57, 2065, 2065, - 4407, 4408, 4409, 4410, 58, 59, 191, 4412, 4413, 190, - 4415, 192, 1399, 1398, 1408, 1409, 1410, 1411, 1401, 1402, - 1403, 1404, 1405, 1406, 1407, 1400, 194, 197, 193, 1385, - 1386, 1387, 1384, 199, 188, 186, 1762, 189, 187, 185, - 74, 11, 132, 19, 4, 0, 0, 0, 0, 0, - 183, 223, 182, 214, 184, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 4224, 0, 0, 0, - 215, 2099, 2100, 2101, 3467, 0, 0, 206, 0, 0, - 0, 216, 0, 0, 0, 0, 0, 2951, 0, 2954, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 153, 0, 0, 0, 2133, 0, 4475, 0, 3962, 2138, - 0, 0, 4480, 4481, 0, 139, 0, 0, 0, 3462, - 1974, 0, 0, 0, 219, 0, 0, 0, 1399, 1398, - 1408, 1409, 1410, 1411, 1401, 1402, 1403, 1404, 1405, 1406, - 1407, 1400, 0, 4501, 0, 0, 0, 2986, 0, 0, - 2989, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 3008, 3009, 0, 0, 0, 0, 0, 0, - 0, 3012, 3013, 1399, 1398, 1408, 1409, 1410, 1411, 1401, - 1402, 1403, 1404, 1405, 1406, 1407, 1400, 3018, 3019, 3020, - 3956, 2203, 2204, 0, 0, 0, 0, 3961, 0, 0, - 0, 0, 0, 0, 0, 3963, 0, 0, 2463, 0, - 0, 0, 0, 162, 163, 0, 164, 165, 0, 0, - 0, 166, 0, 3050, 167, 3052, 0, 0, 3055, 0, - 1888, 2065, 1399, 1398, 1408, 1409, 1410, 1411, 1401, 1402, - 1403, 1404, 1405, 1406, 1407, 1400, 0, 0, 0, 0, - 0, 4221, 0, 0, 2341, 0, 0, 0, 0, 0, - 2341, 2341, 2341, 0, 183, 223, 182, 214, 184, 1399, - 1398, 1408, 1409, 1410, 1411, 1401, 1402, 1403, 1404, 1405, - 1406, 1407, 1400, 0, 215, 0, 0, 0, 0, 0, - 0, 206, 0, 0, 0, 216, 181, 212, 221, 213, - 75, 137, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 153, 0, 0, 1257, 0, 0, - 211, 205, 204, 0, 0, 0, 0, 76, 0, 139, - 0, 0, 0, 0, 0, 0, 1970, 0, 219, 0, - 3195, 3196, 0, 1967, 0, 161, 0, 1969, 1966, 1968, - 1972, 1973, 4226, 4227, 0, 1971, 0, 0, 0, 1065, - 0, 1066, 0, 0, 0, 0, 0, 0, 4222, 4223, - 0, 4230, 4229, 4228, 4241, 4242, 4243, 4231, 4232, 4235, - 4237, 4236, 4233, 4234, 4238, 4239, 4240, 1090, 207, 208, - 209, 4244, 0, 0, 0, 0, 0, 0, 0, 0, - 1046, 0, 4245, 0, 0, 0, 0, 0, 0, 0, - 0, 2346, 0, 0, 1060, 0, 1056, 0, 0, 0, - 0, 0, 0, 2356, 0, 0, 0, 162, 163, 0, - 164, 165, 0, 0, 0, 166, 0, 0, 167, 0, - 0, 0, 1227, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 217, 1086, - 1087, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1132, 0, 0, 0, 0, 0, 0, 0, 0, 149, - 0, 0, 0, 210, 1037, 150, 0, 0, 2410, 0, - 2065, 1955, 1956, 1957, 1958, 1959, 1960, 1961, 1962, 1963, - 1964, 1965, 1977, 1978, 1979, 1980, 1981, 1982, 1975, 1976, - 181, 212, 221, 213, 75, 137, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 211, 205, 204, 0, 0, 0, - 151, 76, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 68, 0, 0, 0, 0, 0, 161, - 0, 0, 0, 0, 0, 1134, 0, 0, 1133, 1062, - 0, 1055, 0, 0, 0, 0, 0, 0, 0, 0, - 1059, 1058, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 3397, 0, 3399, 0, 0, 0, 0, 0, - 0, 1047, 207, 208, 209, 0, 71, 0, 0, 0, - 0, 2580, 0, 2582, 0, 0, 2402, 1118, 0, 0, - 0, 1054, 0, 0, 0, 0, 0, 1091, 0, 0, - 0, 0, 0, 0, 2602, 2603, 2604, 0, 0, 0, - 1064, 0, 159, 220, 160, 1053, 0, 0, 0, 1052, - 2621, 2622, 2623, 2624, 1093, 1040, 0, 66, 0, 0, - 0, 0, 3443, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 217, 0, 1045, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 3465, - 0, 0, 0, 149, 0, 0, 0, 210, 0, 150, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1043, 3493, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1114, 0, 1116, 1113, 0, 0, 0, 1117, 152, - 49, 0, 0, 0, 0, 0, 67, 0, 0, 0, - 5, 0, 0, 0, 151, 0, 0, 0, 0, 1063, - 0, 0, 0, 800, 799, 806, 796, 68, 0, 0, - 156, 157, 0, 0, 158, 0, 803, 804, 1112, 805, - 809, 0, 1044, 790, 0, 0, 0, 0, 0, 0, - 1085, 0, 0, 814, 0, 0, 0, 1691, 0, 0, - 0, 1092, 1127, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 71, 0, 0, 1123, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, - 0, 0, 820, 0, 1728, 0, 0, 819, 2065, 0, - 0, 0, 0, 2065, 0, 0, 159, 220, 160, 1124, - 1128, 0, 2341, 1061, 0, 0, 0, 0, 0, 0, - 0, 66, 0, 0, 0, 0, 0, 0, 0, 1109, - 0, 1107, 1111, 1131, 0, 0, 0, 1108, 1105, 1104, - 0, 1110, 1095, 1096, 1094, 0, 1084, 1097, 1098, 1099, - 1100, 1081, 3666, 1050, 1129, 0, 1130, 0, 0, 0, - 0, 0, 1039, 0, 0, 0, 0, 1125, 1126, 0, - 0, 0, 0, 0, 0, 2949, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 152, 49, 1121, 0, 0, 0, 0, - 67, 1120, 0, 0, 0, 0, 0, 0, 0, 1082, - 0, 0, 0, 0, 0, 0, 0, 0, 1115, 0, - 0, 0, 0, 3700, 156, 157, 0, 0, 158, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 791, 793, - 792, 0, 0, 800, 799, 806, 796, 0, 0, 0, - 798, 1038, 0, 0, 0, 1036, 803, 804, 0, 805, - 809, 0, 802, 790, 0, 0, 0, 0, 0, 817, - 0, 0, 0, 814, 0, 0, 795, 0, 0, 0, - 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 2254, 0, 0, 0, 0, - 2215, 0, 1119, 2262, 0, 0, 0, 0, 1088, 1089, - 0, 0, 1080, 0, 0, 0, 0, 1083, 0, 818, - 0, 0, 820, 0, 0, 0, 0, 819, 0, 0, - 3087, 3088, 3089, 2256, 2224, 0, 0, 0, 0, 0, - 0, 0, 0, 2257, 2258, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3795, 0, 0, 0, 2223, - 0, 0, 0, 3797, 3798, 0, 800, 799, 806, 796, - 0, 0, 0, 0, 0, 0, 0, 2231, 0, 803, - 804, 3183, 805, 809, 0, 0, 790, 0, 0, 0, - 0, 0, 0, 3807, 0, 3809, 814, 0, 0, 0, - 0, 0, 0, 0, 3819, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 797, 801, 807, 0, 808, - 810, 0, 0, 811, 812, 813, 0, 0, 0, 815, - 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 3700, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 2247, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 791, 793, - 792, 0, 1974, 0, 0, 0, 0, 0, 0, 0, - 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 802, 0, 0, 0, 0, 0, 0, 817, - 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2214, 2216, 2213, 0, 0, 0, 2210, 0, 0, 0, - 0, 2235, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 2241, 0, 0, 0, 0, 0, 0, 0, - 2226, 0, 2209, 3381, 3382, 0, 0, 0, 0, 0, - 0, 0, 2229, 2263, 0, 0, 2230, 2232, 2234, 794, - 2236, 2237, 2238, 2242, 2243, 2244, 2246, 2249, 2250, 2251, - 0, 0, 0, 0, 2065, 0, 0, 2239, 2248, 2240, - 0, 791, 793, 792, 0, 0, 0, 0, 0, 2218, - 0, 2065, 0, 798, 4013, 0, 0, 4015, 0, 0, - 0, 0, 0, 0, 0, 802, 0, 821, 822, 823, - 824, 825, 817, 0, 0, 2254, 0, 0, 0, 795, - 2215, 4024, 0, 2262, 0, 0, 0, 0, 0, 0, - 0, 2255, 0, 0, 0, 797, 801, 807, 0, 808, - 810, 0, 0, 811, 812, 813, 0, 0, 0, 815, - 816, 0, 0, 2256, 2224, 0, 0, 0, 0, 0, - 0, 0, 0, 2257, 2258, 0, 0, 0, 1970, 0, - 0, 0, 0, 0, 0, 1967, 0, 2211, 2212, 1969, - 1966, 1968, 1972, 1973, 0, 0, 0, 1971, 0, 2223, - 0, 0, 2254, 0, 0, 2252, 0, 0, 0, 0, - 183, 223, 0, 0, 0, 0, 0, 2231, 0, 0, - 0, 0, 0, 2228, 0, 0, 0, 2227, 0, 0, - 0, 0, 0, 0, 4138, 0, 0, 0, 0, 0, - 2256, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 2245, 0, 0, 0, 0, 0, 0, 0, 0, - 2233, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 2260, 2259, 0, 0, 0, 797, 801, - 807, 0, 808, 810, 219, 0, 811, 812, 813, 0, - 0, 0, 815, 816, 2231, 0, 0, 2247, 0, 0, - 0, 3526, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 2254, 0, 0, 794, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2220, 0, 0, 1955, 1956, 1957, 1958, 1959, 1960, 1961, - 1962, 1963, 1964, 1965, 1977, 1978, 1979, 1980, 1981, 1982, - 1975, 1976, 0, 0, 2256, 0, 3592, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 821, 822, 823, - 824, 825, 0, 0, 2247, 0, 0, 3606, 2261, 3607, - 2214, 3208, 2213, 0, 0, 0, 3207, 0, 0, 0, - 0, 2235, 0, 0, 0, 0, 0, 0, 4371, 0, - 0, 0, 2241, 2254, 0, 0, 0, 0, 2231, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 2229, 2263, 0, 0, 2230, 2232, 2234, 0, - 2236, 2237, 2238, 2242, 2243, 2244, 2246, 2249, 2250, 2251, - 0, 2256, 0, 0, 0, 0, 0, 2239, 2248, 2240, - 0, 0, 794, 0, 0, 0, 0, 0, 0, 2218, - 0, 0, 0, 0, 0, 0, 0, 0, 2235, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 2241, - 0, 0, 0, 0, 2254, 0, 0, 0, 2247, 0, - 0, 0, 0, 0, 0, 2231, 0, 0, 0, 2229, - 2263, 2255, 0, 2230, 2232, 2234, 0, 2236, 2237, 2238, - 2242, 2243, 2244, 2246, 2249, 2250, 2251, 0, 2341, 0, - 0, 0, 2256, 0, 2239, 2248, 2240, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 2211, 2212, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 2252, 0, 0, 0, 4383, - 0, 4341, 0, 0, 0, 2247, 2231, 0, 2255, 0, - 0, 0, 2235, 2228, 0, 0, 0, 2227, 0, 0, - 0, 0, 0, 2241, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 2245, 0, 2229, 2263, 0, 0, 2230, 2232, 2234, - 2233, 2236, 2237, 2238, 2242, 2243, 2244, 2246, 2249, 2250, - 2251, 0, 0, 2260, 2259, 0, 0, 0, 2239, 2248, - 2240, 0, 2252, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 3789, 0, 2247, 0, 0, 0, - 2228, 0, 0, 0, 2227, 0, 0, 1441, 0, 2235, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2241, 0, 0, 0, 0, 0, 0, 0, 2245, 0, - 2220, 0, 2255, 0, 0, 0, 0, 2233, 0, 0, - 2229, 2263, 0, 0, 2230, 2232, 2234, 0, 2236, 2237, - 2238, 2242, 2243, 2244, 2246, 2249, 2250, 2251, 0, 0, - 0, 0, 0, 0, 0, 2239, 2248, 2240, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 2261, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2235, 0, 0, 4499, 0, 0, 2252, 0, 0, 0, - 0, 2241, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 2228, 0, 0, 0, 2227, 2255, - 0, 2229, 2263, 0, 0, 2230, 2232, 2234, 0, 2236, - 2237, 2238, 2242, 2243, 2244, 2246, 2249, 2250, 2251, 0, - 0, 2341, 2245, 0, 0, 0, 2239, 2248, 2240, 0, - 0, 2233, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 866, 841, 4727, 868, 4701, 3238, 240, 4719, 1825, 4633, + 4627, 2233, 3950, 4637, 3705, 4638, 4626, 4401, 1896, 4011, + 4065, 4526, 2852, 3667, 850, 4475, 4228, 4584, 3554, 4297, + 3793, 3979, 4379, 1729, 3556, 4339, 3232, 1892, 4466, 843, + 3794, 4060, 4400, 4503, 1469, 4154, 3894, 3791, 3123, 897, + 724, 1311, 3235, 4369, 4476, 1962, 3902, 4071, 1171, 227, + 3, 4478, 1661, 3429, 3908, 1949, 2704, 2172, 743, 3966, + 4177, 1655, 38, 3358, 3676, 4164, 3211, 757, 767, 776, + 3048, 1899, 776, 3856, 3625, 4135, 3608, 3583, 2956, 1316, + 3929, 2646, 2300, 3359, 1964, 3612, 2356, 794, 839, 3892, + 4169, 3327, 2338, 2335, 3357, 3261, 3696, 3685, 3931, 3354, + 225, 3131, 154, 3678, 3724, 2380, 1969, 3818, 3848, 1946, + 1945, 2446, 3389, 2821, 2421, 3775, 789, 2859, 2707, 3753, + 2191, 3159, 3590, 3573, 2963, 3588, 3636, 2664, 3586, 3585, + 785, 2654, 3584, 1609, 2655, 2647, 3581, 3345, 2079, 37, + 773, 3684, 833, 2480, 3175, 1180, 70, 3536, 2581, 838, + 1722, 70, 2580, 2417, 2385, 1809, 1830, 1040, 2937, 2804, + 2442, 1814, 2331, 1813, 1615, 1818, 3147, 2441, 1165, 3141, + 1798, 3263, 2799, 757, 1080, 2705, 3243, 2304, 2223, 3191, + 2822, 2663, 236, 8, 2634, 1235, 2142, 235, 7, 6, + 2857, 1963, 2650, 1578, 1890, 2443, 2476, 2414, 2653, 842, + 1771, 742, 1707, 1738, 2163, 1701, 724, 2625, 2700, 832, + 1630, 2301, 840, 2583, 1956, 2190, 1932, 851, 723, 1881, + 2628, 1332, 2402, 70, 1778, 2137, 1889, 1556, 1703, 1164, + 240, 1706, 240, 34, 1225, 1226, 2829, 2141, 1079, 1761, + 1003, 757, 758, 1970, 782, 2800, 791, 1640, 1664, 1656, + 1665, 226, 218, 24, 1895, 1551, 1644, 25, 1205, 26, + 1128, 1057, 17, 222, 1077, 1073, 15, 1626, 792, 10, + 775, 1527, 1063, 1112, 1470, 1257, 788, 2450, 28, 1395, + 1396, 1397, 1394, 1395, 1396, 1397, 1394, 1395, 1396, 1397, + 1394, 4488, 4365, 3095, 2831, 1005, 1006, 3095, 3095, 2103, + 1222, 3947, 3810, 3655, 3546, 3545, 3445, 3444, 2460, 1317, + 1552, 769, 761, 4118, 3911, 3047, 1318, 16, 3786, 2997, + 2943, 2941, 1177, 749, 2938, 2940, 1553, 2092, 1785, 1781, + 1217, 1218, 224, 744, 2579, 14, 1546, 1622, 1623, 1624, + 1221, 1705, 1223, 4453, 771, 1510, 2853, 4102, 3547, 3543, + 2594, 2586, 780, 1840, 2099, 1826, 772, 1218, 1555, 1179, + 1218, 3531, 3528, 4713, 3087, 3085, 1681, 5, 2086, 1542, + 70, 1783, 1395, 1396, 1397, 1394, 4058, 1257, 1027, 1317, + 1024, 1395, 1396, 1397, 1394, 70, 3425, 70, 3423, 2390, + 4635, 4634, 4221, 3529, 3526, 768, 3800, 4461, 4304, 4298, + 4061, 3792, 2413, 4480, 1947, 1948, 2649, 8, 3089, 1275, + 1276, 1238, 7, 770, 1216, 1464, 1004, 823, 2961, 3498, + 825, 3571, 2409, 2745, 2779, 824, 4733, 4474, 4710, 4312, + 1015, 4472, 1265, 1269, 1271, 1273, 1278, 4351, 1283, 1279, + 1280, 1281, 1282, 4310, 3883, 1260, 1261, 1262, 1263, 1236, + 1237, 1266, 3027, 1239, 3024, 1241, 1242, 1243, 1244, 1240, + 1245, 1246, 1247, 1248, 1249, 1256, 1258, 1250, 1251, 1252, + 1253, 1254, 1255, 1284, 1285, 1286, 1287, 1288, 1289, 1290, + 1291, 1293, 1292, 1294, 1295, 1296, 1297, 1298, 1299, 1300, + 1301, 1268, 1270, 1272, 1274, 1277, 2601, 1257, 4539, 1838, + 1746, 1563, 1557, 1561, 1560, 4107, 3878, 3574, 834, 2615, + 2283, 1275, 1276, 1238, 1175, 1176, 1028, 1227, 823, 4105, + 1837, 825, 1025, 1181, 787, 3496, 824, 1605, 2458, 3352, + 2629, 2849, 1259, 4353, 1265, 1269, 1271, 1273, 1278, 1392, + 1283, 1279, 1280, 1281, 1282, 1200, 1016, 1260, 1261, 1262, + 1263, 1236, 1237, 1266, 2348, 1239, 2005, 1241, 1242, 1243, + 1244, 1240, 1245, 1246, 1247, 1248, 1249, 1256, 1258, 1250, + 1251, 1252, 1253, 1254, 1255, 1284, 1285, 1286, 1287, 1288, + 1289, 1290, 1291, 1293, 1292, 1294, 1295, 1296, 1297, 1298, + 1299, 1300, 1301, 1268, 1270, 1272, 1274, 1277, 3530, 3527, + 1150, 1587, 2850, 1028, 2113, 1025, 1691, 2111, 994, 834, + 993, 995, 996, 2836, 997, 998, 2835, 1570, 3396, 2837, + 3397, 3398, 1708, 1677, 1710, 1585, 1678, 1022, 2314, 1201, + 1621, 1275, 1276, 1238, 1259, 2785, 2184, 1784, 1782, 2784, + 2315, 2316, 2118, 2119, 3090, 3671, 1652, 3669, 1662, 1663, + 3553, 3122, 4641, 4642, 1265, 1269, 1271, 1273, 1278, 2957, + 1283, 1279, 1280, 1281, 1282, 1137, 2205, 1260, 1261, 1262, + 1263, 1236, 1237, 1266, 1898, 1239, 1390, 1241, 1242, 1243, + 1244, 1240, 1245, 1246, 1247, 1248, 1249, 1256, 1258, 1250, + 1251, 1252, 1253, 1254, 1255, 1284, 1285, 1286, 1287, 1288, + 1289, 1290, 1291, 1293, 1292, 1294, 1295, 1296, 1297, 1298, + 1299, 1300, 1301, 1268, 1270, 1272, 1274, 1277, 1194, 1189, + 1184, 1188, 1192, 1026, 1385, 1023, 3120, 1680, 4090, 3118, + 2326, 3143, 1174, 1173, 2737, 4483, 183, 223, 182, 214, + 184, 3144, 757, 4483, 4598, 1372, 1197, 757, 1373, 1320, + 1187, 4482, 4597, 4482, 1259, 183, 223, 182, 214, 184, + 4481, 4596, 183, 223, 182, 214, 184, 4481, 776, 776, + 1346, 2558, 757, 1586, 1143, 1141, 1375, 1142, 183, 223, + 182, 214, 184, 4667, 4610, 1882, 823, 2182, 1886, 825, + 3142, 4705, 4706, 4464, 824, 3088, 786, 4467, 4468, 4469, + 4470, 1195, 3119, 1365, 3115, 1146, 1367, 3795, 4586, 1660, + 219, 4586, 1885, 1659, 1662, 1663, 183, 223, 182, 214, + 184, 1846, 1545, 1198, 2459, 773, 773, 773, 1327, 219, + 1199, 4589, 1228, 3430, 1368, 3431, 219, 3432, 3435, 3378, + 1438, 2813, 2814, 1319, 1324, 4640, 4301, 1694, 2102, 3795, + 2346, 2347, 219, 2978, 1321, 4109, 2462, 2332, 3282, 3893, + 4499, 1267, 2454, 1902, 3813, 4487, 4364, 3816, 1177, 4146, + 1185, 3462, 3094, 1335, 1338, 1318, 1650, 1318, 1151, 2794, + 3116, 2322, 1019, 1318, 1877, 3900, 3604, 3458, 3346, 1320, + 219, 1990, 3150, 2787, 1196, 2623, 1370, 1069, 2114, 754, + 3128, 2112, 3994, 1352, 4089, 1179, 1388, 1389, 70, 70, + 70, 3446, 4091, 4612, 2778, 3443, 2781, 1679, 2988, 2485, + 1377, 3802, 3602, 1378, 3456, 1147, 210, 2780, 1473, 1588, + 2183, 2743, 1186, 1387, 1360, 2449, 1887, 1218, 4059, 1218, + 1218, 3424, 1218, 4106, 1339, 3121, 3598, 1177, 1218, 1218, + 3340, 1380, 2281, 1318, 1361, 2790, 2791, 1020, 1371, 2789, + 1884, 2461, 4361, 1267, 2939, 4355, 4356, 4143, 2854, 4103, + 3609, 3610, 3097, 1786, 1383, 1384, 3599, 3600, 2797, 1382, + 1363, 741, 4010, 1474, 1179, 4311, 1233, 1149, 3673, 4292, + 1672, 1330, 3601, 1366, 1369, 1766, 769, 769, 769, 4428, + 826, 827, 828, 829, 830, 2465, 2467, 2468, 4006, 1548, + 1550, 3895, 1554, 1193, 1562, 3086, 1362, 1313, 1901, 1900, + 1340, 1004, 1310, 3117, 1675, 1676, 1559, 1553, 1574, 771, + 771, 771, 1577, 1021, 4354, 4491, 4342, 1584, 1553, 1374, + 1839, 772, 772, 772, 1525, 1309, 1176, 1530, 1344, 1345, + 1190, 4172, 1349, 1191, 4320, 4119, 4321, 3917, 1434, 1435, + 1436, 1437, 1183, 1351, 3779, 757, 757, 3623, 1148, 1080, + 1439, 1376, 4315, 4391, 3596, 1323, 1325, 1328, 3146, 1883, + 768, 768, 768, 1267, 1558, 4320, 4383, 4321, 1233, 2282, + 1662, 1663, 1662, 1663, 3637, 4519, 4514, 1364, 770, 770, + 770, 826, 827, 828, 829, 830, 3192, 3860, 1337, 1336, + 1029, 1381, 1986, 3862, 4108, 2636, 778, 1145, 777, 1983, + 3350, 3610, 4323, 1985, 1982, 1984, 1988, 1989, 4111, 4112, + 4113, 1987, 2631, 1379, 757, 3999, 1690, 3537, 1651, 1696, + 1908, 1911, 1912, 757, 3698, 3699, 4504, 724, 724, 4521, + 3697, 1909, 4322, 4323, 3951, 4527, 3668, 724, 724, 2723, + 1569, 1733, 1733, 3237, 757, 2703, 2726, 3958, 1565, 3233, + 3234, 1202, 3237, 1658, 3874, 1182, 1018, 3700, 2612, 3701, + 3703, 3702, 3704, 4322, 1485, 1486, 776, 1762, 743, 4016, + 3707, 2325, 4498, 1639, 1774, 1735, 4348, 4127, 1731, 1731, + 1432, 1071, 3871, 1072, 3566, 2777, 4216, 1567, 1233, 240, + 4205, 4739, 4722, 183, 223, 2755, 4074, 1740, 724, 3610, + 2754, 1342, 3156, 2725, 2775, 2776, 1144, 1329, 2710, 1138, + 2177, 774, 1718, 1717, 1692, 183, 223, 1580, 1581, 1582, + 3187, 2854, 3873, 1591, 1593, 1594, 1595, 1596, 1350, 1598, + 774, 1654, 1653, 1326, 1637, 1604, 4211, 774, 1636, 3183, + 2333, 2793, 4392, 153, 3674, 4611, 3605, 3347, 3459, 1695, + 1531, 1635, 4528, 774, 3149, 4384, 3619, 1529, 1993, 1994, + 1995, 1996, 1997, 1998, 1991, 1992, 4370, 219, 4625, 3677, + 2724, 4405, 1610, 1822, 3520, 2746, 3932, 71, 1827, 2815, + 2466, 2703, 4056, 1579, 1592, 1632, 4147, 787, 1836, 3181, + 3283, 774, 3284, 3285, 1620, 4583, 71, 1429, 1428, 1727, + 1728, 1590, 2454, 71, 1140, 1712, 1714, 1139, 1704, 3153, + 3154, 3597, 1859, 3311, 3857, 1725, 1726, 1862, 3938, 71, + 1611, 3728, 2323, 1614, 3152, 1878, 4357, 1733, 2713, 1733, + 1320, 1829, 3698, 3699, 1641, 1645, 1645, 1645, 1357, 3184, + 1629, 3693, 1646, 1647, 70, 4723, 2984, 2841, 1638, 826, + 827, 828, 829, 830, 4026, 1648, 2783, 71, 1682, 1683, + 2720, 1641, 1641, 1667, 1668, 2741, 1670, 1671, 1666, 773, + 1673, 1669, 773, 773, 2095, 2709, 1787, 2584, 1793, 1763, + 2711, 2451, 3407, 3408, 2321, 2298, 1871, 1796, 1576, 1799, + 1800, 3706, 3391, 3393, 1564, 1597, 3886, 1811, 1812, 1589, + 1733, 1801, 1802, 1803, 1804, 1805, 1806, 3104, 1716, 3743, + 4316, 3334, 3620, 749, 4317, 1910, 3730, 1320, 1320, 1967, + 1741, 3461, 1817, 1603, 1897, 1821, 1602, 1834, 1820, 2477, + 4404, 1754, 1999, 2000, 2712, 2018, 2004, 1950, 2635, 1601, + 1600, 4316, 1356, 1775, 2019, 4477, 1760, 1152, 1335, 1338, + 1776, 781, 70, 2287, 2285, 70, 70, 2026, 2286, 2028, + 1081, 2029, 2030, 2031, 3694, 752, 1070, 753, 4219, 70, + 3280, 4207, 3849, 1966, 2613, 4206, 4624, 1918, 1919, 1920, + 1921, 1922, 1923, 1924, 1925, 1926, 1927, 1928, 1929, 2714, + 2976, 1034, 1618, 4720, 4721, 1943, 1944, 1566, 1568, 1894, + 3112, 4212, 4213, 1083, 1084, 1085, 2808, 2812, 2813, 2814, + 2809, 2818, 2810, 2816, 1320, 2605, 2811, 1138, 2817, 1339, + 1968, 1573, 2036, 2038, 2037, 2003, 2104, 1875, 2094, 2105, + 1913, 2121, 2108, 2122, 1177, 2463, 2464, 757, 757, 757, + 769, 1832, 2002, 769, 769, 1579, 2123, 2125, 2027, 2126, + 1041, 2128, 2129, 2130, 1038, 3864, 743, 1762, 1844, 1036, + 1035, 1847, 2138, 3186, 1733, 2144, 2145, 2077, 2147, 1696, + 757, 1179, 1816, 771, 2604, 757, 771, 771, 1733, 2017, + 3939, 1869, 1888, 1864, 1080, 772, 1868, 2173, 772, 772, + 1866, 1891, 3392, 1863, 3312, 3314, 3315, 3316, 3313, 2100, + 2719, 2080, 1870, 2120, 2717, 1733, 1893, 1030, 2088, 2035, + 2767, 1696, 1140, 1930, 1931, 1139, 767, 1941, 1942, 1845, + 1934, 1856, 1848, 1849, 768, 1031, 1034, 768, 768, 2096, + 183, 223, 3302, 3303, 2607, 2606, 2204, 1853, 1854, 1138, + 4178, 1867, 770, 1696, 4291, 770, 770, 2448, 2213, 2213, + 1037, 1696, 2165, 1696, 1696, 1571, 1572, 757, 757, 1865, + 2280, 4735, 3105, 4729, 2138, 2291, 2083, 4716, 1733, 2295, + 2296, 4680, 3020, 3021, 2311, 1153, 724, 2448, 3654, 3014, + 3806, 3749, 1631, 1337, 1336, 4653, 2740, 1815, 2627, 1033, + 724, 1879, 1733, 2146, 1036, 1035, 3695, 3134, 2522, 2148, + 2819, 2521, 2208, 4650, 219, 4741, 1283, 1279, 1280, 1281, + 1282, 2032, 2033, 3019, 4649, 3018, 3017, 3015, 2353, 2355, + 757, 2138, 1733, 4593, 2361, 2169, 757, 757, 757, 785, + 785, 4643, 3135, 3136, 1140, 3750, 2371, 1139, 2373, 2374, + 2375, 1858, 1312, 1631, 2381, 1312, 2456, 2078, 4730, 1393, + 1857, 240, 4681, 1631, 240, 240, 4681, 240, 4748, 2235, + 2084, 2134, 2135, 2136, 183, 223, 3210, 2132, 3195, 2289, + 4654, 3301, 2854, 2209, 2150, 2151, 2152, 2153, 3745, 1357, + 1355, 2216, 2349, 2093, 2424, 2097, 1393, 1432, 4651, 4621, + 2101, 2573, 3209, 2448, 2379, 2008, 2009, 2010, 4576, 2456, + 3016, 2341, 2342, 4575, 1393, 3889, 2133, 1357, 2024, 3815, + 4549, 2025, 2432, 2983, 2420, 2327, 2495, 2178, 2626, 1880, + 2179, 2180, 2363, 2364, 2365, 3711, 2174, 4254, 2143, 3709, + 2044, 2045, 2170, 3577, 2710, 2713, 3750, 2983, 2422, 2196, + 2820, 1354, 2159, 2318, 3535, 2320, 2683, 2173, 4522, 2360, + 4510, 1733, 2445, 2215, 2312, 2203, 2339, 2340, 2206, 2207, + 2076, 2389, 2334, 2187, 2392, 2393, 1641, 2395, 2193, 2185, + 4272, 2412, 3533, 2192, 4622, 2194, 2195, 2423, 2217, 2218, + 1645, 3200, 2399, 1393, 2188, 2189, 2197, 4451, 1393, 2201, + 2425, 2820, 1645, 773, 4142, 2495, 2427, 2820, 2202, 2288, + 4450, 2198, 2199, 2212, 2214, 1393, 70, 3750, 1526, 70, + 70, 4253, 70, 4420, 2439, 2293, 3410, 3091, 1686, 1687, + 2299, 1689, 2210, 2317, 1693, 2319, 1697, 1698, 1699, 2328, + 1355, 2962, 2294, 2456, 4419, 4511, 4418, 1395, 1396, 1397, + 1394, 4417, 183, 223, 182, 214, 184, 3210, 2447, 2377, + 2801, 2398, 2313, 2447, 2696, 2352, 2358, 2406, 2359, 1747, + 1748, 1749, 1750, 1751, 1752, 1753, 2815, 1755, 1756, 1757, + 1758, 1759, 4452, 4395, 1357, 1765, 70, 1767, 1768, 1769, + 2578, 4394, 2386, 2366, 2367, 2661, 1891, 2808, 2812, 2813, + 2814, 2809, 2818, 2810, 2816, 2682, 2714, 2811, 2495, 2817, + 2572, 2709, 2703, 2708, 835, 2706, 2711, 2571, 2474, 2475, + 2531, 2404, 1008, 1009, 1010, 1011, 219, 2698, 1177, 2495, + 950, 2495, 1861, 2530, 2425, 4367, 2495, 2039, 2040, 2041, + 2042, 2529, 2173, 2046, 2047, 2048, 2049, 2051, 2052, 2053, + 2054, 2055, 2056, 2057, 2058, 2059, 2060, 2061, 2062, 2063, + 2438, 2344, 2297, 2436, 1613, 1179, 4336, 4333, 2456, 3523, + 2712, 2494, 1953, 1719, 769, 2585, 2456, 2587, 4731, 2589, + 2590, 4021, 2483, 2593, 4448, 4280, 3947, 3415, 3960, 2440, + 2434, 3212, 757, 1696, 757, 1696, 1395, 1396, 1397, 1394, + 3100, 1395, 1396, 1397, 1394, 2608, 2453, 771, 3521, 2554, + 4250, 3899, 833, 2986, 2985, 757, 757, 757, 2497, 772, + 2495, 2624, 2469, 2977, 2690, 2557, 2559, 2560, 2561, 2478, + 2563, 757, 757, 757, 757, 1395, 1396, 1397, 1394, 3919, + 2472, 2473, 2566, 2471, 1934, 2710, 2713, 2018, 2018, 2657, + 4278, 1393, 2661, 2435, 3524, 2665, 2517, 2668, 768, 2493, + 2487, 2500, 2564, 2670, 2671, 2672, 2854, 2675, 1696, 1395, + 1396, 1397, 1394, 3961, 2437, 3841, 770, 4229, 4230, 4231, + 4235, 4233, 4234, 4236, 4237, 4238, 4239, 4232, 1013, 1395, + 1396, 1397, 1394, 3522, 2384, 3837, 1696, 2369, 1395, 1396, + 1397, 1394, 181, 212, 221, 213, 1395, 1396, 1397, 1394, + 3719, 4255, 4256, 2732, 2098, 1395, 1396, 1397, 1394, 1841, + 1448, 3482, 2165, 2598, 3920, 2600, 211, 2567, 4251, 4252, + 3440, 4259, 4258, 4257, 4270, 4271, 4273, 4260, 4261, 4264, + 4266, 4265, 4262, 4263, 4267, 4268, 4269, 2565, 3386, 3202, + 1341, 4274, 1219, 1220, 1307, 2669, 3493, 1224, 4276, 1177, + 3842, 1302, 4275, 3197, 3492, 1395, 1396, 1397, 1394, 3066, + 4019, 3054, 2575, 3068, 2739, 2343, 2687, 3659, 757, 2213, + 3838, 2470, 2689, 1410, 2691, 3046, 3453, 2824, 2824, 2311, + 2824, 2652, 2999, 4742, 2981, 3720, 1179, 1395, 1396, 1397, + 1394, 2588, 1213, 1214, 1215, 2592, 1393, 2714, 1032, 2953, + 724, 724, 2709, 2703, 2708, 3198, 2706, 2711, 1320, 2482, + 2481, 4515, 2951, 2570, 1733, 757, 2692, 1791, 1790, 1429, + 1428, 2616, 4709, 2820, 3203, 2105, 1212, 2949, 1627, 1209, + 2947, 2738, 1628, 757, 4489, 2702, 2701, 4078, 3198, 1320, + 2920, 743, 3638, 1674, 2661, 2491, 1393, 1473, 1774, 1642, + 2311, 2847, 2782, 2928, 2658, 2930, 2660, 4516, 240, 2574, + 1393, 2712, 1723, 2538, 4385, 2695, 2537, 1393, 2924, 2661, + 2520, 2828, 4442, 1724, 2511, 2532, 2533, 2510, 2535, 2677, + 2678, 4366, 2509, 2676, 2954, 2542, 4179, 2499, 2496, 2680, + 2681, 1177, 2838, 2018, 2839, 2018, 757, 2952, 4308, 2455, + 2973, 2826, 1474, 2830, 1008, 1009, 1010, 1011, 2979, 1850, + 3935, 2445, 2948, 2844, 2845, 2948, 2715, 2716, 1733, 2721, + 1733, 4248, 1733, 3639, 4209, 1721, 2679, 1320, 1179, 2856, + 3933, 2685, 4180, 2862, 2686, 2998, 3784, 2007, 2006, 1645, + 2938, 2661, 2927, 2832, 2573, 2688, 1627, 2815, 1393, 2684, + 1628, 1393, 2007, 2006, 4386, 1393, 3936, 2989, 2861, 1393, + 3006, 4208, 1393, 4077, 1039, 1733, 1320, 1393, 2933, 3640, + 1643, 3028, 2495, 2495, 4194, 4150, 3934, 3910, 1712, 1714, + 2932, 2792, 2966, 2798, 2456, 4595, 3751, 1744, 3037, 3741, + 3733, 3721, 3614, 1733, 1851, 3343, 2833, 3342, 3022, 3201, + 4387, 3161, 1731, 71, 3096, 1177, 1413, 1414, 1415, 1416, + 1417, 1410, 2996, 70, 1206, 1207, 1208, 1211, 1940, 1210, + 2843, 2993, 1395, 1396, 1397, 1394, 3038, 2851, 2848, 1720, + 1731, 2362, 3558, 3787, 1937, 1939, 1936, 2652, 1938, 1872, + 220, 1873, 1179, 2372, 2591, 1395, 1396, 1397, 1394, 2430, + 2429, 2960, 2921, 2050, 3043, 3044, 2942, 3098, 2926, 3555, + 2428, 1607, 3102, 1606, 1322, 3106, 2387, 2925, 2043, 1957, + 1013, 2488, 757, 757, 757, 3032, 1395, 1396, 1397, 1394, + 1779, 3009, 2387, 3011, 3416, 3785, 3126, 1957, 2127, 1320, + 2958, 4335, 3558, 3039, 2995, 2967, 4334, 1733, 1394, 2990, + 1696, 4738, 3025, 2969, 4224, 4223, 1696, 2291, 3641, 3004, + 3555, 3272, 2426, 3070, 3270, 3071, 2982, 3073, 2980, 3075, + 3076, 3249, 2987, 1411, 1412, 1413, 1414, 1415, 1416, 1417, + 1410, 3247, 3205, 3208, 1401, 1402, 1403, 1404, 1405, 1406, + 1407, 1399, 4200, 3214, 1397, 1394, 3082, 4564, 4565, 3000, + 3001, 1395, 1396, 1397, 1394, 1395, 1396, 1397, 1394, 3013, + 3008, 3224, 4422, 4423, 2934, 3484, 4737, 3023, 3557, 4658, + 3003, 1320, 4151, 4152, 4620, 1395, 1396, 1397, 1394, 3246, + 1450, 3182, 1891, 2862, 1780, 4619, 1320, 1320, 1320, 2213, + 3160, 4568, 1320, 1449, 3256, 3257, 3258, 3259, 1320, 3266, + 2513, 3267, 3268, 4144, 3269, 3176, 3271, 3078, 2861, 3079, + 4567, 3179, 3083, 1395, 1396, 1397, 1394, 3266, 3163, 3170, + 3171, 3172, 3164, 3169, 3165, 3167, 3166, 3168, 3483, 2824, + 3157, 4566, 4563, 3193, 869, 879, 3177, 1395, 1396, 1397, + 1394, 2964, 2965, 3324, 870, 1779, 871, 875, 878, 874, + 872, 873, 3225, 4562, 3897, 1395, 1396, 1397, 1394, 4561, + 724, 3323, 4560, 4145, 3138, 3241, 3140, 2235, 2291, 4558, + 2512, 3215, 1320, 2311, 2311, 2311, 2311, 2311, 2311, 4557, + 3241, 3252, 3253, 4556, 3227, 70, 3255, 4555, 3321, 3137, + 1320, 2311, 3262, 3155, 2824, 4554, 3329, 1395, 1396, 1397, + 1394, 4553, 3319, 3185, 3308, 3244, 4551, 4550, 2022, 3244, + 3394, 876, 1733, 3240, 3898, 4517, 4683, 1395, 1396, 1397, + 1394, 3322, 8, 2023, 3207, 757, 757, 7, 3251, 3204, + 3124, 3903, 3036, 3049, 3050, 4408, 4398, 3029, 4388, 3055, + 4360, 1398, 877, 1395, 1396, 1397, 1394, 3468, 3320, 1431, + 4630, 3335, 4332, 3226, 4299, 3229, 4218, 4182, 1441, 4536, + 3242, 2143, 3318, 3382, 3307, 3909, 3360, 3248, 4181, 3952, + 3217, 3254, 4097, 3937, 1835, 3220, 3896, 1395, 1396, 1397, + 1394, 3245, 3879, 3603, 3360, 1452, 1395, 1396, 1397, 1394, + 3449, 3428, 3427, 4094, 3412, 3332, 3306, 3305, 3286, 1395, + 1396, 1397, 1394, 3304, 3296, 240, 3395, 3290, 1715, 3289, + 240, 3288, 3348, 4093, 3287, 3213, 3092, 3223, 2955, 3298, + 1395, 1396, 1397, 1394, 3216, 1395, 1396, 1397, 1394, 1395, + 1396, 1397, 1394, 3221, 3222, 2840, 3338, 2577, 3341, 3344, + 1395, 1396, 1397, 1394, 2408, 2407, 2405, 2401, 3448, 2400, + 2350, 2110, 3411, 2107, 1733, 2744, 3379, 3455, 2747, 2748, + 2749, 2750, 2751, 2752, 2753, 1842, 3384, 2756, 2757, 2758, + 2759, 2760, 2761, 2762, 2763, 2764, 2765, 2766, 3383, 2768, + 2769, 2770, 2771, 2772, 3385, 2773, 3402, 1544, 3403, 3399, + 3589, 4734, 3442, 4732, 3361, 3362, 3363, 3364, 3365, 3366, + 4066, 1409, 1408, 1418, 1419, 1420, 1421, 1411, 1412, 1413, + 1414, 1415, 1416, 1417, 1410, 3417, 1800, 4358, 4359, 4707, + 3421, 1811, 1812, 4673, 1305, 4607, 1801, 1802, 1803, 1804, + 1805, 1806, 2524, 4605, 4082, 881, 155, 4340, 4581, 4548, + 1817, 155, 4501, 1821, 4155, 4495, 1820, 1408, 1418, 1419, + 1420, 1421, 1411, 1412, 1413, 1414, 1415, 1416, 1417, 1410, + 70, 1395, 1396, 1397, 1394, 70, 4486, 4484, 3541, 4471, + 3418, 3544, 4081, 4462, 4437, 3419, 3548, 4740, 757, 1696, + 4436, 3452, 3457, 1304, 4427, 2309, 4426, 3560, 3562, 3563, + 3565, 4080, 3567, 3568, 4412, 3433, 3437, 4407, 4406, 1395, + 1396, 1397, 1394, 4363, 1320, 4347, 750, 4003, 4345, 4331, + 1320, 3829, 4695, 155, 4300, 4202, 3592, 3594, 1395, 1396, + 1397, 1394, 4159, 4148, 3451, 4132, 4131, 3607, 4129, 4124, + 3464, 3480, 3465, 757, 1395, 1396, 1397, 1394, 1395, 1396, + 1397, 1394, 4122, 4101, 3476, 3477, 4100, 4099, 4096, 3622, + 4095, 3626, 1320, 4068, 3474, 757, 4064, 757, 2291, 1320, + 1320, 3473, 755, 3475, 4062, 3471, 3472, 4032, 4029, 2018, + 4023, 2018, 3328, 3891, 3651, 3881, 3866, 3850, 3534, 3828, + 2311, 2665, 3826, 3658, 1409, 1408, 1418, 1419, 1420, 1421, + 1411, 1412, 1413, 1414, 1415, 1416, 1417, 1410, 3578, 3819, + 3822, 3801, 2732, 3762, 3241, 3618, 3739, 3738, 1773, 3551, + 3736, 3735, 3722, 3717, 3683, 3611, 3686, 3716, 3686, 3686, + 3615, 3575, 3538, 1320, 3539, 3569, 3629, 1395, 1396, 1397, + 1394, 3559, 3549, 3635, 3542, 3540, 2582, 3621, 3463, 3460, + 3646, 3712, 3447, 3426, 3708, 3176, 3241, 3401, 3336, 1733, + 1733, 3333, 3330, 3241, 3241, 1178, 3317, 3309, 3299, 3297, + 155, 3648, 3179, 3525, 1177, 3595, 3670, 3672, 1076, 2080, + 3293, 3666, 3656, 3292, 3650, 155, 3291, 155, 3127, 3113, + 3661, 3101, 3713, 3714, 3093, 2504, 1731, 1731, 950, 949, + 1395, 1396, 1397, 1394, 2492, 2971, 757, 2959, 3628, 3617, + 2922, 1179, 2609, 2596, 2595, 3633, 3634, 2411, 2403, 2211, + 3592, 3642, 3649, 3647, 2140, 3644, 3494, 3241, 3657, 2109, + 3653, 3682, 2106, 1696, 2091, 2090, 2291, 2291, 1843, 1481, + 1477, 3681, 1476, 3665, 1308, 3691, 755, 1017, 3145, 2702, + 2701, 3499, 3500, 1395, 1396, 1397, 1394, 3501, 3502, 3503, + 3504, 4534, 3505, 3506, 3507, 3508, 3509, 3510, 3511, 3512, + 3513, 3514, 3515, 3516, 4530, 3692, 4337, 3710, 4327, 3687, + 3688, 4326, 1395, 1396, 1397, 1394, 3488, 4313, 183, 223, + 4309, 1320, 1395, 1396, 1397, 1394, 3028, 4130, 3487, 4098, + 4079, 4075, 3718, 4043, 3788, 4024, 3940, 3485, 1903, 1904, + 1905, 1906, 1907, 1395, 1396, 1397, 1394, 3660, 3928, 3927, + 3067, 3923, 3662, 3663, 3726, 1395, 1396, 1397, 1394, 3888, + 3846, 3844, 3843, 3689, 1395, 1396, 1397, 1394, 223, 182, + 214, 184, 757, 3840, 3723, 3746, 3747, 1395, 1396, 1397, + 1394, 3839, 3732, 1954, 3731, 3827, 3825, 1958, 1959, 1960, + 1961, 3740, 219, 3744, 3278, 3279, 1431, 3790, 3065, 2001, + 3789, 3774, 3737, 3758, 3773, 3759, 3064, 3652, 2012, 3294, + 3295, 3664, 3734, 3579, 3576, 2862, 3532, 3063, 3490, 3478, + 3470, 3767, 3062, 3807, 3809, 1395, 1396, 1397, 1394, 3469, + 3770, 3771, 3772, 1395, 1396, 1397, 1394, 3467, 3409, 2950, + 2861, 219, 3339, 3808, 1395, 1396, 1397, 1394, 3777, 1395, + 1396, 1397, 1394, 2946, 3852, 2945, 2944, 2543, 3853, 2536, + 2381, 2490, 2068, 3061, 2070, 2071, 2072, 2073, 2074, 3798, + 2528, 2527, 3867, 2081, 3869, 3805, 3060, 2526, 2525, 3875, + 3830, 2523, 2519, 2518, 3812, 2516, 2507, 3811, 2503, 3748, + 1395, 1396, 1397, 1394, 2502, 3863, 2410, 2069, 2067, 3817, + 3059, 2066, 3876, 1395, 1396, 1397, 1394, 2065, 3058, 3820, + 2064, 2021, 3832, 3766, 3834, 2020, 3836, 3057, 183, 223, + 757, 2291, 2011, 3870, 1745, 3872, 3056, 1395, 1396, 1397, + 1394, 3053, 1743, 3239, 3918, 1395, 1396, 1397, 1394, 1395, + 1396, 1397, 1394, 3926, 1395, 1396, 1397, 1394, 4694, 4657, + 223, 3858, 3052, 1395, 1396, 1397, 1394, 3051, 1395, 1396, + 1397, 1394, 3847, 3887, 2824, 2311, 3944, 3851, 3645, 4574, + 3890, 4535, 1471, 4529, 4457, 3855, 2181, 4454, 3907, 1395, + 1396, 1397, 1394, 3045, 1395, 1396, 1397, 1394, 3962, 4435, + 4416, 1320, 219, 4409, 4294, 3726, 4293, 3880, 4243, 3884, + 3683, 4222, 2200, 4220, 1320, 183, 223, 4215, 3885, 3033, + 1395, 1396, 1397, 1394, 4193, 3916, 3904, 4176, 4044, 4041, + 1320, 3026, 4018, 219, 4001, 2167, 1733, 4000, 3997, 3996, + 3906, 4012, 4013, 4014, 4015, 3959, 1395, 1396, 1397, 1394, + 3956, 4027, 3954, 183, 223, 3946, 3912, 3941, 1395, 1396, + 1397, 1394, 3865, 3861, 757, 2164, 2291, 3943, 3995, 4020, + 2311, 1320, 3572, 1731, 3481, 3479, 1795, 2081, 1810, 1797, + 1816, 1819, 2081, 2081, 3005, 1807, 1792, 1616, 3953, 2166, + 3955, 3371, 3331, 3325, 3949, 3963, 3250, 3942, 3196, 3189, + 4050, 2569, 3986, 153, 3188, 3180, 240, 3139, 4005, 3069, + 2842, 1395, 1396, 1397, 1394, 2774, 2659, 4002, 2618, 4033, + 4036, 4007, 2617, 2576, 3262, 1935, 4004, 219, 1395, 1396, + 1397, 1394, 219, 2368, 2388, 4017, 2168, 2391, 4049, 2568, + 2394, 2087, 4022, 2396, 1876, 1808, 4025, 1543, 155, 155, + 155, 1178, 4028, 1528, 1524, 1523, 4031, 1522, 4030, 1521, + 4034, 1520, 4037, 1519, 4038, 3360, 1395, 1396, 1397, 1394, + 4035, 2562, 1518, 1517, 4039, 1516, 1515, 1514, 1513, 1512, + 1511, 1510, 2173, 4546, 1952, 4114, 1509, 1076, 2418, 4120, + 1508, 1507, 1315, 4073, 1506, 4126, 1505, 3945, 1395, 1396, + 1397, 1394, 1504, 1503, 1502, 3948, 1501, 1500, 4067, 1499, + 1320, 1395, 1396, 1397, 1394, 1498, 4057, 1348, 4070, 1497, + 1496, 1495, 1494, 1493, 1492, 1491, 1490, 1489, 1488, 1487, + 1430, 1484, 1483, 1320, 1733, 1733, 1482, 1480, 4160, 1479, + 4123, 3626, 4125, 1478, 1475, 1468, 1467, 1465, 1464, 4110, + 1463, 1462, 1461, 1460, 1459, 4168, 1458, 1457, 1456, 1320, + 4168, 70, 4104, 1455, 1454, 1453, 1447, 4157, 1446, 1445, + 1444, 1731, 1950, 1443, 1442, 1320, 4187, 1320, 1359, 1306, + 4544, 4162, 4163, 3754, 3755, 4542, 3998, 2674, 4156, 2633, + 1347, 4687, 4685, 4190, 1733, 4192, 4139, 4138, 4117, 4137, + 4158, 4639, 3757, 3729, 3241, 3337, 3162, 2855, 2484, 2645, + 1625, 4149, 2489, 1358, 3381, 757, 4165, 1320, 1320, 3369, + 2498, 1320, 1320, 4046, 4175, 4161, 4134, 3765, 4170, 4174, + 3368, 1950, 4183, 4047, 3376, 3764, 3763, 3374, 4245, 3377, + 4186, 3946, 3375, 3372, 3760, 3380, 4240, 4279, 3373, 4247, + 3367, 138, 3995, 3360, 2427, 4199, 4196, 4594, 4473, 4203, + 2508, 73, 2173, 72, 69, 4286, 4198, 3441, 2515, 1897, + 3199, 1897, 4226, 4227, 3613, 1608, 4241, 4242, 4195, 4295, + 4296, 2161, 2162, 4045, 3925, 3679, 3986, 3680, 4201, 2156, + 2157, 2158, 1532, 3439, 1733, 4008, 2534, 3803, 3804, 2742, + 3778, 2539, 2540, 2541, 2272, 1788, 2544, 2545, 2546, 2547, + 2548, 2549, 2550, 2551, 2552, 2553, 4281, 2555, 2556, 3194, + 2964, 2965, 745, 4328, 4329, 4246, 757, 4307, 4282, 4284, + 3274, 1731, 746, 4319, 747, 748, 1831, 3275, 3276, 3277, + 4341, 2994, 4343, 2603, 2602, 1828, 2610, 2370, 2284, 1353, + 4413, 4302, 4128, 3587, 3580, 4306, 3228, 3190, 2694, 2643, + 2171, 2131, 4698, 4314, 4411, 4344, 3715, 4346, 4052, 2795, + 4318, 4083, 2788, 4084, 2292, 4085, 2007, 2006, 1685, 4399, + 1539, 1540, 1684, 4184, 4185, 1537, 1538, 4171, 4069, 1535, + 1536, 1533, 1534, 4375, 4324, 4325, 4349, 4380, 1386, 4373, + 2431, 3776, 3769, 2611, 2433, 2176, 1634, 1633, 4350, 1599, + 1076, 1612, 1657, 4141, 1320, 2992, 2667, 4092, 4664, 4662, + 4613, 4591, 4140, 4590, 2991, 4588, 4362, 4368, 4505, 4403, + 4458, 4289, 4397, 1409, 1408, 1418, 1419, 1420, 1421, 1411, + 1412, 1413, 1414, 1415, 1416, 1417, 1410, 4288, 4374, 4377, + 4188, 4116, 4376, 4073, 4063, 3831, 4389, 3797, 3796, 4371, + 3782, 4393, 1742, 2415, 1320, 2727, 750, 2697, 1833, 3781, + 3414, 1631, 4689, 4688, 4668, 4121, 3868, 3854, 4283, 1688, + 3450, 3108, 3107, 3099, 2923, 2505, 1343, 4410, 1702, 1314, + 4688, 4689, 4217, 4048, 1008, 1009, 1010, 1011, 1733, 1312, + 4136, 4449, 751, 3930, 155, 3436, 2637, 1824, 1897, 1739, + 1312, 1649, 81, 2, 1423, 4711, 1427, 4712, 1, 4421, + 3084, 2085, 1541, 1012, 1007, 1709, 2834, 2345, 1737, 2089, + 1014, 4446, 1424, 1426, 1422, 1731, 1425, 1409, 1408, 1418, + 1419, 1420, 1421, 1411, 1412, 1413, 1414, 1415, 1416, 1417, + 1410, 3387, 3388, 3768, 3390, 4485, 2351, 1619, 3114, 2452, + 4479, 3349, 2786, 4490, 2622, 3606, 1617, 1082, 2501, 2013, + 4459, 1855, 4497, 2081, 1334, 2081, 1852, 1333, 1331, 1955, + 2034, 883, 2648, 3913, 3914, 3915, 3326, 3300, 4492, 4285, + 4493, 3921, 3922, 4697, 2081, 2081, 4726, 4656, 4700, 1874, + 867, 4582, 3799, 3434, 4506, 4463, 4660, 4502, 4465, 4305, + 2457, 1391, 155, 3643, 1108, 155, 155, 4494, 927, 895, + 1466, 2419, 3497, 3495, 894, 3901, 3151, 4290, 3406, 155, + 4382, 4500, 1773, 4524, 1109, 2397, 4460, 1320, 4303, 1789, + 1794, 4509, 4508, 2693, 4390, 4525, 4197, 3675, 3236, 4552, + 1823, 4520, 3957, 4088, 4086, 4087, 1320, 4541, 4543, 4545, + 4547, 793, 2324, 722, 4523, 1162, 4244, 2644, 1733, 4570, + 2673, 4249, 4415, 4571, 4559, 4532, 1054, 3882, 4578, 2632, + 1055, 4518, 1047, 2972, 3174, 2975, 3173, 1914, 1400, 1933, + 3518, 3519, 4540, 1440, 837, 2486, 3148, 3980, 3400, 4579, + 80, 4569, 79, 78, 77, 1731, 248, 886, 247, 4338, + 4153, 4577, 4606, 4702, 863, 862, 861, 860, 859, 858, + 2806, 4580, 2807, 2805, 4587, 4585, 2803, 2802, 1733, 2306, + 4603, 2305, 4380, 3413, 4599, 4601, 3780, 1430, 4608, 2376, + 4600, 4602, 2378, 3007, 4604, 3624, 3010, 3265, 4623, 4009, + 1897, 4191, 3260, 2224, 4631, 2222, 1700, 2722, 2729, 3030, + 3031, 4615, 4614, 2221, 4616, 1731, 4636, 4076, 3034, 3035, + 4537, 4538, 4617, 4618, 4214, 3310, 4072, 2155, 2718, 2241, + 3281, 2238, 2237, 3273, 3040, 3041, 3042, 4210, 4204, 4644, + 2269, 4645, 4378, 4646, 4167, 4647, 3964, 3965, 3971, 1264, + 2642, 4648, 1234, 4652, 1229, 1409, 1408, 1418, 1419, 1420, + 1421, 1411, 1412, 1413, 1414, 1415, 1416, 1417, 1410, 1231, + 3072, 1232, 3074, 1230, 3012, 3077, 3742, 1903, 2081, 4663, + 4655, 4665, 4666, 4661, 1320, 4659, 2699, 3582, 3133, 3132, + 4479, 4669, 3130, 3129, 1583, 4496, 4609, 4133, 2860, 4670, + 4672, 4671, 2858, 1303, 3756, 4403, 3752, 3552, 4676, 1549, + 1547, 4678, 4679, 2656, 4677, 4682, 3761, 3370, 2416, 3438, + 2307, 2303, 4686, 4684, 2302, 4696, 1204, 1203, 4704, 4455, + 4456, 4703, 1770, 3859, 4690, 4691, 4692, 4693, 3924, 48, + 3351, 2796, 4352, 2160, 1048, 155, 1320, 2630, 117, 42, + 133, 116, 2115, 2116, 2117, 4708, 201, 63, 200, 4715, + 4524, 62, 4717, 4718, 4714, 18, 4189, 4724, 131, 198, + 4728, 61, 47, 4725, 46, 196, 111, 110, 4674, 109, + 108, 130, 195, 60, 232, 2149, 231, 3218, 3219, 234, + 2154, 4736, 233, 230, 2935, 2936, 229, 1777, 228, 4592, + 4173, 4704, 4744, 4573, 4703, 4743, 1002, 45, 44, 202, + 43, 118, 64, 4728, 4745, 41, 40, 2666, 3570, 4749, + 1409, 1408, 1418, 1419, 1420, 1421, 1411, 1412, 1413, 1414, + 1415, 1416, 1417, 1410, 4277, 2175, 3877, 3125, 2614, 2310, + 1897, 39, 35, 13, 12, 36, 3491, 23, 22, 183, + 223, 182, 214, 184, 1860, 21, 27, 33, 32, 1423, + 148, 1427, 147, 31, 146, 145, 144, 143, 142, 215, + 141, 140, 2219, 2220, 30, 20, 206, 1424, 1426, 1422, + 216, 1425, 1409, 1408, 1418, 1419, 1420, 1421, 1411, 1412, + 1413, 1414, 1415, 1416, 1417, 1410, 3486, 55, 54, 153, + 1409, 1408, 1418, 1419, 1420, 1421, 1411, 1412, 1413, 1414, + 1415, 1416, 1417, 1410, 139, 53, 155, 52, 51, 155, + 155, 50, 155, 219, 1071, 9, 1072, 136, 134, 129, + 127, 29, 128, 125, 126, 2357, 121, 2081, 120, 119, + 114, 2357, 2357, 2357, 112, 92, 91, 90, 105, 104, + 1409, 1408, 1418, 1419, 1420, 1421, 1411, 1412, 1413, 1414, + 1415, 1416, 1417, 1410, 3002, 1052, 103, 102, 101, 100, + 155, 98, 99, 1107, 89, 88, 87, 86, 85, 1066, + 122, 1062, 107, 115, 113, 96, 155, 106, 1409, 1408, + 1418, 1419, 1420, 1421, 1411, 1412, 1413, 1414, 1415, 1416, + 1417, 1410, 97, 95, 94, 183, 223, 182, 214, 184, + 93, 84, 162, 163, 83, 164, 165, 82, 124, 123, + 166, 135, 203, 167, 65, 215, 180, 179, 178, 177, + 176, 174, 206, 175, 173, 172, 216, 171, 170, 169, + 168, 56, 57, 3420, 58, 3422, 59, 191, 190, 1043, + 192, 194, 197, 193, 199, 153, 188, 186, 189, 187, + 185, 74, 11, 132, 19, 4424, 4425, 2418, 4, 0, + 139, 1430, 4429, 4430, 4431, 4432, 4433, 4434, 0, 219, + 0, 4438, 4439, 4440, 4441, 0, 0, 0, 0, 4443, + 4444, 4445, 0, 4447, 0, 0, 0, 0, 0, 181, + 212, 221, 213, 75, 137, 0, 0, 0, 0, 0, + 0, 0, 0, 3466, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 211, 205, 204, 0, 0, 0, 0, + 76, 0, 0, 0, 0, 0, 0, 0, 1068, 0, + 1061, 3489, 0, 0, 0, 0, 0, 0, 161, 1065, + 1064, 2506, 0, 0, 1409, 1408, 1418, 1419, 1420, 1421, + 1411, 1412, 1413, 1414, 1415, 1416, 1417, 1410, 162, 163, + 1053, 164, 165, 3517, 0, 0, 166, 0, 0, 167, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 4507, + 1060, 207, 208, 209, 0, 4512, 4513, 0, 0, 0, + 0, 0, 0, 2479, 0, 0, 0, 0, 0, 1070, + 0, 0, 0, 0, 1059, 0, 0, 0, 1058, 0, + 0, 0, 0, 0, 1046, 0, 4533, 1409, 1408, 1418, + 1419, 1420, 1421, 1411, 1412, 1413, 1414, 1415, 1416, 1417, + 1410, 0, 0, 1051, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 181, 212, 221, 213, 75, + 137, 0, 217, 1418, 1419, 1420, 1421, 1411, 1412, 1413, + 1414, 1415, 1416, 1417, 1410, 0, 0, 0, 0, 211, + 205, 204, 0, 149, 0, 0, 76, 210, 0, 150, + 1049, 0, 0, 0, 0, 0, 0, 2597, 0, 2599, + 0, 2081, 1178, 0, 161, 155, 2081, 0, 1096, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2619, 2620, 2621, 0, 0, 0, 0, 0, 0, 1069, + 0, 0, 0, 0, 0, 0, 2638, 2639, 2640, 2641, + 0, 0, 0, 0, 151, 0, 0, 207, 208, 209, + 0, 0, 1050, 0, 0, 3690, 0, 68, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1092, 1093, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1138, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 3990, 0, 0, 0, 0, 0, 3969, + 71, 0, 0, 0, 0, 0, 0, 0, 217, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1067, 0, 0, 3725, 0, 0, 149, + 0, 0, 0, 210, 2827, 150, 159, 220, 160, 0, + 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 66, 0, 3972, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1056, 3967, 0, 0, 0, 0, 3992, + 3993, 0, 1045, 1702, 0, 3968, 1140, 0, 0, 1139, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 68, 0, 2310, 0, 0, 0, 0, + 0, 0, 0, 155, 0, 3973, 0, 0, 0, 0, + 1739, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1124, 152, 49, 0, 0, 0, 2357, 0, + 67, 0, 1097, 0, 5, 0, 0, 0, 1178, 0, + 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, + 0, 0, 0, 0, 156, 157, 0, 0, 158, 1099, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 3821, + 0, 1044, 0, 0, 0, 1042, 0, 3823, 3824, 0, + 0, 0, 159, 220, 160, 0, 0, 0, 0, 0, + 0, 2970, 0, 0, 0, 0, 0, 66, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 3833, 0, 3835, + 0, 0, 0, 0, 0, 0, 0, 0, 3845, 0, + 0, 0, 0, 0, 0, 0, 3991, 0, 2708, 0, + 0, 0, 0, 0, 0, 0, 1120, 0, 1122, 1119, + 0, 0, 0, 1123, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 3977, 0, 0, 0, 3725, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 3974, 3978, 3976, 3975, 152, + 49, 0, 0, 0, 1118, 0, 67, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1091, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1098, 1133, 0, + 156, 157, 0, 0, 158, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1129, + 0, 0, 0, 3984, 3985, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1130, 1134, 3109, 3110, 3111, + 0, 1395, 1396, 1397, 1394, 155, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1115, 0, 1113, 1117, 1137, + 0, 0, 0, 1114, 1111, 1110, 0, 1116, 1101, 1102, + 1100, 3994, 1090, 1103, 1104, 1105, 1106, 1087, 0, 0, + 1135, 0, 1136, 0, 3970, 0, 0, 3983, 0, 2081, + 0, 0, 0, 1131, 1132, 0, 0, 0, 3206, 0, + 805, 804, 811, 801, 0, 0, 2081, 0, 0, 4040, + 0, 0, 4042, 808, 809, 0, 810, 814, 0, 0, + 795, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 819, 1127, 1990, 0, 0, 0, 4051, 1126, 0, 0, + 0, 0, 0, 0, 0, 1088, 0, 0, 0, 0, + 0, 0, 0, 0, 1121, 805, 804, 811, 801, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 808, 809, + 0, 810, 814, 0, 0, 795, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 819, 0, 0, 2310, 2310, + 2310, 2310, 2310, 2310, 805, 804, 811, 801, 0, 0, + 0, 0, 0, 0, 0, 0, 2310, 808, 809, 0, + 810, 814, 0, 0, 795, 0, 0, 0, 0, 0, + 0, 0, 3988, 0, 819, 0, 0, 0, 0, 0, + 0, 823, 0, 0, 825, 0, 0, 0, 0, 824, + 0, 0, 0, 0, 0, 0, 0, 0, 1125, 0, + 0, 0, 0, 0, 1094, 1095, 0, 0, 1086, 0, + 0, 0, 0, 1089, 0, 0, 0, 0, 0, 0, + 823, 0, 0, 825, 0, 0, 0, 0, 824, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 3404, 3405, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3982, 0, 0, 0, 0, 0, + 0, 3987, 0, 0, 0, 0, 0, 0, 0, 3989, + 155, 0, 0, 0, 0, 155, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 796, + 798, 797, 0, 1986, 0, 0, 0, 0, 0, 0, + 1983, 803, 0, 0, 1985, 1982, 1984, 1988, 1989, 0, + 0, 0, 1987, 807, 0, 0, 0, 0, 0, 0, + 822, 0, 0, 0, 0, 0, 0, 800, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 796, 798, 797, 0, 0, 0, + 0, 0, 0, 0, 2270, 0, 803, 0, 0, 2231, + 0, 0, 2278, 0, 0, 0, 0, 0, 807, 0, + 0, 0, 0, 0, 0, 822, 0, 0, 0, 0, + 0, 0, 800, 796, 798, 797, 790, 0, 0, 0, + 0, 0, 2272, 2240, 0, 803, 0, 0, 0, 0, + 0, 0, 2273, 2274, 0, 0, 0, 807, 0, 0, + 0, 0, 0, 0, 822, 0, 0, 0, 0, 0, + 0, 800, 0, 0, 0, 0, 0, 0, 2239, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 2247, 0, 1971, 1972, + 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1993, + 1994, 1995, 1996, 1997, 1998, 1991, 1992, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 802, 806, 812, + 0, 813, 815, 3550, 0, 816, 817, 818, 0, 0, + 0, 820, 821, 1990, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1178, 0, 155, + 0, 0, 0, 0, 0, 0, 155, 4414, 0, 0, + 0, 0, 0, 155, 0, 0, 2263, 0, 0, 0, + 0, 0, 802, 806, 812, 2310, 813, 815, 3616, 0, + 816, 817, 818, 0, 0, 0, 820, 821, 0, 0, + 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, + 3630, 0, 3631, 0, 0, 0, 0, 0, 0, 0, + 0, 802, 806, 812, 0, 813, 815, 0, 0, 816, + 817, 818, 0, 0, 0, 820, 821, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 2230, 2232, 2229, 0, 0, 1452, 2226, + 0, 0, 0, 0, 2251, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 2257, 0, 0, 0, 0, + 0, 0, 0, 2242, 0, 2225, 0, 0, 0, 0, + 0, 799, 0, 0, 0, 2245, 2279, 3727, 0, 2246, + 2248, 2250, 0, 2252, 2253, 2254, 2258, 2259, 2260, 2262, + 2265, 2266, 2267, 0, 0, 0, 0, 2270, 0, 0, + 2255, 2264, 2256, 0, 0, 183, 223, 0, 0, 0, + 0, 0, 2234, 0, 0, 0, 0, 0, 0, 0, + 0, 2357, 0, 0, 4531, 0, 799, 0, 0, 4166, + 0, 0, 0, 0, 1986, 2272, 0, 0, 0, 0, + 0, 1983, 0, 0, 0, 1985, 1982, 1984, 1988, 1989, + 0, 0, 0, 1987, 2271, 0, 0, 2270, 0, 0, + 0, 0, 0, 0, 0, 799, 0, 0, 0, 0, + 0, 0, 0, 0, 826, 827, 828, 829, 830, 219, + 0, 0, 0, 0, 0, 0, 155, 0, 0, 2247, + 0, 0, 0, 0, 0, 2272, 0, 0, 0, 0, + 0, 2227, 2228, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 826, 827, 828, 829, 830, 0, 2268, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 2244, 0, 4402, + 0, 2243, 0, 0, 4628, 0, 0, 0, 0, 2247, + 4632, 0, 0, 0, 0, 0, 0, 3814, 0, 0, + 0, 0, 0, 0, 0, 2261, 0, 0, 0, 2263, + 0, 0, 0, 0, 2249, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 2276, 2275, 1971, + 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, + 1993, 1994, 1995, 1996, 1997, 1998, 1991, 1992, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 3727, 0, + 0, 0, 0, 0, 0, 0, 155, 0, 0, 2263, + 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, + 4628, 0, 0, 0, 2236, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 2251, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2257, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 2277, 0, 0, 0, 4628, 0, 2245, 2279, + 2310, 0, 2246, 2248, 2250, 2357, 2252, 2253, 2254, 2258, + 2259, 2260, 2262, 2265, 2266, 2267, 0, 0, 0, 0, + 0, 0, 0, 2255, 2264, 2256, 0, 2251, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2257, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 4747, 2245, 2279, + 0, 0, 2246, 2248, 2250, 0, 2252, 2253, 2254, 2258, + 2259, 2260, 2262, 2265, 2266, 2267, 0, 2271, 0, 0, + 0, 0, 0, 2255, 2264, 2256, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 2310, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 155, 0, 0, 0, 0, 0, 2271, 0, 2357, + 0, 0, 2268, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2244, 0, 0, 0, 2243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 2252, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2255, 2228, 0, 0, 0, 2227, 0, 0, 0, 0, - 0, 0, 4595, 0, 0, 0, 0, 0, 4599, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 2245, - 0, 0, 0, 0, 0, 0, 0, 0, 2233, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 2252, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 2341, 0, 0, 0, 0, 0, - 0, 0, 2228, 0, 0, 0, 2227, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 4595, 0, 0, - 2245, 0, 0, 0, 0, 0, 0, 0, 0, 2233, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 4595, 896, 0, 0, 0, 0, 0, - 0, 0, 0, 453, 0, 0, 592, 626, 615, 700, - 580, 0, 0, 0, 0, 0, 0, 847, 0, 0, - 0, 369, 0, 0, 421, 630, 611, 622, 612, 597, - 598, 599, 606, 381, 600, 601, 602, 572, 603, 573, - 604, 605, 887, 629, 579, 491, 437, 0, 646, 0, - 0, 967, 975, 0, 4714, 0, 0, 0, 0, 0, - 0, 963, 0, 0, 0, 0, 839, 0, 0, 876, - 944, 943, 863, 873, 0, 0, 337, 246, 574, 696, - 576, 575, 864, 0, 865, 869, 872, 868, 866, 867, - 0, 958, 0, 0, 0, 0, 0, 0, 831, 843, - 0, 848, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 840, 841, 0, - 0, 0, 0, 897, 0, 842, 0, 0, 0, 0, - 0, 492, 521, 0, 534, 0, 406, 407, 892, 870, - 874, 0, 0, 0, 4197, 324, 499, 518, 338, 486, - 532, 343, 494, 511, 333, 452, 483, 0, 0, 326, - 516, 493, 434, 325, 0, 477, 366, 383, 363, 450, - 871, 0, 895, 899, 362, 981, 893, 526, 328, 0, - 525, 449, 512, 517, 435, 428, 0, 327, 514, 433, - 427, 412, 373, 982, 413, 414, 387, 464, 425, 465, - 388, 439, 438, 440, 389, 390, 391, 392, 393, 394, - 395, 396, 397, 398, 0, 0, 0, 0, 0, 556, - 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 689, 890, 0, 693, - 0, 528, 0, 0, 965, 0, 0, 0, 497, 0, - 0, 415, 0, 4299, 0, 894, 0, 480, 455, 978, - 0, 0, 478, 423, 513, 466, 519, 500, 527, 472, - 467, 318, 501, 365, 436, 334, 336, 721, 367, 370, - 374, 375, 445, 446, 460, 485, 504, 505, 506, 364, - 348, 479, 349, 384, 350, 319, 356, 354, 357, 487, - 358, 321, 461, 510, 0, 380, 475, 431, 322, 430, - 462, 509, 508, 335, 536, 543, 544, 634, 0, 549, - 732, 733, 734, 558, 0, 468, 331, 330, 0, 0, - 0, 360, 463, 344, 346, 347, 345, 458, 459, 563, - 564, 565, 567, 0, 568, 569, 0, 0, 0, 0, - 570, 635, 651, 619, 588, 551, 643, 585, 589, 590, - 401, 402, 403, 654, 1999, 1998, 2000, 542, 416, 417, - 0, 372, 371, 432, 323, 0, 0, 409, 400, 469, - 329, 368, 411, 405, 418, 419, 420, 378, 313, 314, - 727, 962, 451, 656, 691, 692, 581, 0, 977, 957, - 959, 960, 964, 968, 969, 970, 971, 972, 974, 976, - 980, 726, 0, 636, 650, 730, 649, 723, 457, 0, - 484, 647, 594, 0, 640, 613, 614, 0, 641, 609, - 645, 0, 583, 0, 552, 555, 584, 669, 670, 671, - 320, 554, 673, 674, 675, 676, 677, 678, 679, 672, - 979, 617, 593, 620, 533, 596, 595, 0, 0, 631, - 898, 632, 633, 441, 442, 443, 444, 966, 657, 342, - 553, 471, 0, 618, 0, 0, 0, 0, 0, 0, - 0, 0, 623, 624, 621, 735, 0, 680, 681, 0, - 0, 547, 548, 377, 0, 566, 385, 341, 456, 379, - 531, 408, 0, 559, 625, 560, 473, 474, 683, 688, - 684, 685, 687, 707, 448, 399, 404, 488, 410, 424, - 476, 530, 454, 481, 339, 520, 490, 429, 610, 638, - 988, 961, 987, 989, 990, 986, 991, 992, 973, 852, - 0, 905, 906, 984, 983, 985, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 665, 664, 663, - 662, 661, 660, 659, 658, 0, 0, 607, 507, 355, - 307, 351, 352, 359, 724, 720, 725, 708, 711, 710, - 686, 859, 315, 587, 422, 470, 376, 652, 653, 0, - 706, 951, 914, 915, 916, 849, 917, 911, 912, 850, - 913, 952, 903, 948, 949, 878, 908, 918, 947, 919, - 950, 879, 953, 993, 994, 925, 909, 277, 995, 922, - 954, 946, 945, 920, 904, 955, 956, 886, 881, 923, - 924, 910, 931, 932, 933, 936, 851, 937, 938, 939, - 940, 941, 935, 934, 900, 901, 902, 926, 927, 929, - 930, 907, 498, 882, 883, 884, 885, 0, 0, 537, - 538, 539, 562, 0, 540, 522, 586, 386, 316, 502, - 529, 722, 0, 0, 0, 0, 0, 0, 0, 637, - 648, 682, 0, 694, 695, 697, 699, 942, 701, 495, - 496, 709, 0, 0, 928, 704, 705, 702, 426, 482, - 503, 489, 0, 728, 577, 578, 729, 690, 317, 0, - 844, 183, 223, 896, 0, 0, 0, 0, 0, 0, - 0, 0, 453, 0, 0, 592, 626, 615, 700, 580, - 0, 0, 0, 0, 0, 0, 847, 0, 0, 0, - 369, 0, 0, 421, 630, 611, 622, 612, 597, 598, - 599, 606, 381, 600, 601, 602, 572, 603, 573, 604, - 605, 887, 629, 579, 491, 437, 0, 646, 0, 0, - 967, 975, 0, 0, 0, 0, 0, 0, 0, 0, - 963, 0, 0, 0, 0, 839, 0, 0, 876, 944, - 943, 863, 873, 0, 0, 337, 246, 574, 696, 576, - 575, 864, 0, 865, 869, 872, 868, 866, 867, 0, - 958, 0, 0, 0, 0, 0, 0, 831, 843, 0, - 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 840, 841, 0, 0, - 0, 0, 897, 0, 842, 0, 0, 0, 0, 0, - 492, 521, 0, 534, 0, 406, 407, 892, 870, 874, - 0, 0, 0, 0, 324, 499, 518, 338, 486, 532, - 343, 494, 511, 333, 452, 483, 0, 0, 326, 516, - 493, 434, 325, 0, 477, 366, 383, 363, 450, 871, - 0, 895, 899, 362, 981, 893, 526, 328, 0, 525, - 449, 512, 517, 435, 428, 0, 327, 514, 433, 427, - 412, 373, 982, 413, 414, 387, 464, 425, 465, 388, - 439, 438, 440, 389, 390, 391, 392, 393, 394, 395, - 396, 397, 398, 0, 0, 0, 0, 0, 556, 557, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 689, 890, 0, 693, 0, - 528, 0, 0, 965, 0, 0, 0, 497, 0, 0, - 415, 0, 0, 0, 894, 0, 480, 455, 978, 0, - 0, 478, 423, 513, 466, 519, 500, 527, 472, 467, - 318, 501, 365, 436, 334, 336, 721, 367, 370, 374, - 375, 445, 446, 460, 485, 504, 505, 506, 364, 348, - 479, 349, 384, 350, 319, 356, 354, 357, 487, 358, - 321, 461, 510, 0, 380, 475, 431, 322, 430, 462, - 509, 508, 335, 536, 543, 544, 634, 0, 549, 732, - 733, 734, 558, 0, 468, 331, 330, 0, 0, 0, - 360, 463, 344, 346, 347, 345, 458, 459, 563, 564, - 565, 567, 0, 568, 569, 0, 0, 0, 0, 570, - 635, 651, 619, 588, 551, 643, 585, 589, 590, 401, - 402, 403, 654, 0, 0, 0, 542, 416, 417, 0, - 372, 371, 432, 323, 0, 0, 409, 400, 469, 329, - 368, 411, 405, 418, 419, 420, 378, 313, 314, 727, - 962, 451, 656, 691, 692, 581, 0, 977, 957, 959, - 960, 964, 968, 969, 970, 971, 972, 974, 976, 980, - 726, 0, 636, 650, 730, 649, 723, 457, 0, 484, - 647, 594, 0, 640, 613, 614, 0, 641, 609, 645, - 0, 583, 0, 552, 555, 584, 669, 670, 671, 320, - 554, 673, 674, 675, 676, 677, 678, 679, 672, 979, - 617, 593, 620, 533, 596, 595, 0, 0, 631, 898, - 632, 633, 441, 442, 443, 444, 966, 657, 342, 553, - 471, 0, 618, 0, 0, 0, 0, 0, 0, 0, - 0, 623, 624, 621, 735, 0, 680, 681, 0, 0, - 547, 548, 377, 0, 566, 385, 341, 456, 379, 531, - 408, 0, 559, 625, 560, 473, 474, 683, 688, 684, - 685, 687, 707, 448, 399, 404, 488, 410, 424, 476, - 530, 454, 481, 339, 520, 490, 429, 610, 638, 988, - 961, 987, 989, 990, 986, 991, 992, 973, 852, 0, - 905, 906, 984, 983, 985, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 665, 664, 663, 662, - 661, 660, 659, 658, 0, 0, 607, 507, 355, 307, - 351, 352, 359, 724, 720, 725, 708, 711, 710, 686, - 859, 315, 587, 422, 470, 376, 652, 653, 0, 706, - 951, 914, 915, 916, 849, 917, 911, 912, 850, 913, - 952, 903, 948, 949, 878, 908, 918, 947, 919, 950, - 879, 953, 993, 994, 925, 909, 277, 995, 922, 954, - 946, 945, 920, 904, 955, 956, 886, 881, 923, 924, - 910, 931, 932, 933, 936, 851, 937, 938, 939, 940, - 941, 935, 934, 900, 901, 902, 926, 927, 929, 930, - 907, 498, 882, 883, 884, 885, 0, 0, 537, 538, - 539, 562, 0, 540, 522, 586, 386, 316, 502, 529, - 722, 0, 0, 0, 0, 0, 0, 0, 637, 648, - 682, 0, 694, 695, 697, 699, 942, 701, 495, 496, - 709, 0, 0, 928, 704, 705, 702, 426, 482, 503, - 489, 896, 728, 577, 578, 729, 690, 317, 0, 844, - 453, 0, 0, 592, 626, 615, 700, 580, 0, 0, - 0, 0, 0, 0, 847, 0, 0, 0, 369, 2066, - 0, 421, 630, 611, 622, 612, 597, 598, 599, 606, - 381, 600, 601, 602, 572, 603, 573, 604, 605, 887, - 629, 579, 491, 437, 0, 646, 0, 0, 967, 975, - 0, 0, 0, 0, 0, 0, 0, 0, 963, 0, - 2320, 0, 0, 839, 0, 0, 876, 944, 943, 863, - 873, 0, 0, 337, 246, 574, 696, 576, 575, 864, - 0, 865, 869, 872, 868, 866, 867, 0, 958, 0, - 0, 0, 0, 0, 0, 831, 843, 0, 848, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 840, 841, 0, 0, 0, 0, - 897, 0, 842, 0, 0, 0, 0, 0, 492, 521, - 0, 534, 0, 406, 407, 2321, 870, 874, 0, 0, - 0, 0, 324, 499, 518, 338, 486, 532, 343, 494, - 511, 333, 452, 483, 0, 0, 326, 516, 493, 434, - 325, 0, 477, 366, 383, 363, 450, 871, 0, 895, - 899, 362, 981, 893, 526, 328, 0, 525, 449, 512, - 517, 435, 428, 0, 327, 514, 433, 427, 412, 373, - 982, 413, 414, 387, 464, 425, 465, 388, 439, 438, - 440, 389, 390, 391, 392, 393, 394, 395, 396, 397, - 398, 0, 0, 0, 0, 0, 556, 557, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 689, 890, 0, 693, 0, 528, 0, - 0, 965, 0, 0, 0, 497, 0, 0, 415, 0, - 0, 0, 894, 0, 480, 455, 978, 0, 0, 478, - 423, 513, 466, 519, 500, 527, 472, 467, 318, 501, - 365, 436, 334, 336, 721, 367, 370, 374, 375, 445, - 446, 460, 485, 504, 505, 506, 364, 348, 479, 349, - 384, 350, 319, 356, 354, 357, 487, 358, 321, 461, - 510, 0, 380, 475, 431, 322, 430, 462, 509, 508, - 335, 536, 543, 544, 634, 0, 549, 732, 733, 734, - 558, 0, 468, 331, 330, 0, 0, 0, 360, 463, - 344, 346, 347, 345, 458, 459, 563, 564, 565, 567, - 0, 568, 569, 0, 0, 0, 0, 570, 635, 651, - 619, 588, 551, 643, 585, 589, 590, 401, 402, 403, - 654, 0, 0, 0, 542, 416, 417, 0, 372, 371, - 432, 323, 0, 0, 409, 400, 469, 329, 368, 411, - 405, 418, 419, 420, 378, 313, 314, 727, 962, 451, - 656, 691, 692, 581, 0, 977, 957, 959, 960, 964, - 968, 969, 970, 971, 972, 974, 976, 980, 726, 0, - 636, 650, 730, 649, 723, 457, 0, 484, 647, 594, - 0, 640, 613, 614, 0, 641, 609, 645, 0, 583, - 0, 552, 555, 584, 669, 670, 671, 320, 554, 673, - 674, 675, 676, 677, 678, 679, 672, 979, 617, 593, - 620, 533, 596, 595, 0, 0, 631, 898, 632, 633, - 441, 442, 443, 444, 966, 657, 342, 553, 471, 0, - 618, 0, 0, 0, 0, 0, 0, 0, 0, 623, - 624, 621, 735, 0, 680, 681, 0, 0, 547, 548, - 377, 0, 566, 385, 341, 456, 379, 531, 408, 0, - 559, 625, 560, 473, 474, 683, 688, 684, 685, 687, - 707, 448, 399, 404, 488, 410, 424, 476, 530, 454, - 481, 339, 520, 490, 429, 610, 638, 988, 961, 987, - 989, 990, 986, 991, 992, 973, 852, 0, 905, 906, - 984, 983, 985, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 665, 664, 663, 662, 661, 660, - 659, 658, 0, 0, 607, 507, 355, 307, 351, 352, - 359, 724, 720, 725, 708, 711, 710, 686, 859, 315, - 587, 422, 470, 376, 652, 653, 0, 706, 951, 914, - 915, 916, 849, 917, 911, 912, 850, 913, 952, 903, - 948, 949, 878, 908, 918, 947, 919, 950, 879, 953, - 993, 994, 925, 909, 277, 995, 922, 954, 946, 945, - 920, 904, 955, 956, 886, 881, 923, 924, 910, 931, - 932, 933, 936, 851, 937, 938, 939, 940, 941, 935, - 934, 900, 901, 902, 926, 927, 929, 930, 907, 498, - 882, 883, 884, 885, 0, 0, 537, 538, 539, 562, - 0, 540, 522, 586, 386, 316, 502, 529, 722, 0, - 0, 0, 0, 0, 0, 0, 637, 648, 682, 0, - 694, 695, 697, 699, 942, 701, 495, 496, 709, 0, - 0, 928, 704, 705, 702, 426, 482, 503, 489, 0, - 728, 577, 578, 729, 690, 317, 0, 844, 183, 223, - 896, 0, 0, 0, 0, 0, 0, 0, 0, 453, - 0, 0, 592, 626, 615, 700, 580, 0, 0, 0, - 0, 0, 0, 847, 0, 0, 0, 369, 0, 0, - 421, 630, 611, 622, 612, 597, 598, 599, 606, 381, - 600, 601, 602, 572, 603, 573, 604, 605, 1423, 629, - 579, 491, 437, 0, 646, 0, 0, 967, 975, 0, - 0, 0, 0, 0, 0, 0, 0, 963, 0, 0, - 0, 0, 839, 0, 0, 876, 944, 943, 863, 873, - 0, 0, 337, 246, 574, 696, 576, 575, 864, 0, - 865, 869, 872, 868, 866, 867, 0, 958, 0, 0, - 0, 0, 0, 0, 831, 843, 0, 848, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 840, 841, 0, 0, 0, 0, 897, - 0, 842, 0, 0, 0, 0, 0, 492, 521, 0, - 534, 0, 406, 407, 892, 870, 874, 0, 0, 0, - 0, 324, 499, 518, 338, 486, 532, 343, 494, 511, - 333, 452, 483, 0, 0, 326, 516, 493, 434, 325, - 0, 477, 366, 383, 363, 450, 871, 0, 895, 899, - 362, 981, 893, 526, 328, 0, 525, 449, 512, 517, - 435, 428, 0, 327, 514, 433, 427, 412, 373, 982, - 413, 414, 387, 464, 425, 465, 388, 439, 438, 440, - 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, - 0, 0, 0, 0, 0, 556, 557, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 689, 890, 0, 693, 0, 528, 0, 0, - 965, 0, 0, 0, 497, 0, 0, 415, 0, 0, - 0, 894, 0, 480, 455, 978, 0, 0, 478, 423, - 513, 466, 519, 500, 527, 472, 467, 318, 501, 365, - 436, 334, 336, 721, 367, 370, 374, 375, 445, 446, - 460, 485, 504, 505, 506, 364, 348, 479, 349, 384, - 350, 319, 356, 354, 357, 487, 358, 321, 461, 510, - 0, 380, 475, 431, 322, 430, 462, 509, 508, 335, - 536, 543, 544, 634, 0, 549, 732, 733, 734, 558, - 0, 468, 331, 330, 0, 0, 0, 360, 463, 344, - 346, 347, 345, 458, 459, 563, 564, 565, 567, 0, - 568, 569, 0, 0, 0, 0, 570, 635, 651, 619, - 588, 551, 643, 585, 589, 590, 401, 402, 403, 654, - 0, 0, 0, 542, 416, 417, 0, 372, 371, 432, - 323, 0, 0, 409, 400, 469, 329, 368, 411, 405, - 418, 419, 420, 378, 313, 314, 727, 962, 451, 656, - 691, 692, 581, 0, 977, 957, 959, 960, 964, 968, - 969, 970, 971, 972, 974, 976, 980, 726, 0, 636, - 650, 730, 649, 723, 457, 0, 484, 647, 594, 0, - 640, 613, 614, 0, 641, 609, 645, 0, 583, 0, - 552, 555, 584, 669, 670, 671, 320, 554, 673, 674, - 675, 676, 677, 678, 679, 672, 979, 617, 593, 620, - 533, 596, 595, 0, 0, 631, 898, 632, 633, 441, - 442, 443, 444, 966, 657, 342, 553, 471, 0, 618, - 0, 0, 0, 0, 0, 0, 0, 0, 623, 624, - 621, 735, 0, 680, 681, 0, 0, 547, 548, 377, - 0, 566, 385, 341, 456, 379, 531, 408, 0, 559, - 625, 560, 473, 474, 683, 688, 684, 685, 687, 707, - 448, 399, 404, 488, 410, 424, 476, 530, 454, 481, - 339, 520, 490, 429, 610, 638, 988, 961, 987, 989, - 990, 986, 991, 992, 973, 852, 0, 905, 906, 984, - 983, 985, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 665, 664, 663, 662, 661, 660, 659, - 658, 0, 0, 607, 507, 355, 307, 351, 352, 359, - 724, 720, 725, 708, 711, 710, 686, 859, 315, 587, - 422, 470, 376, 652, 653, 0, 706, 951, 914, 915, - 916, 849, 917, 911, 912, 850, 913, 952, 903, 948, - 949, 878, 908, 918, 947, 919, 950, 879, 953, 993, - 994, 925, 909, 277, 995, 922, 954, 946, 945, 920, - 904, 955, 956, 886, 881, 923, 924, 910, 931, 932, - 933, 936, 851, 937, 938, 939, 940, 941, 935, 934, - 900, 901, 902, 926, 927, 929, 930, 907, 498, 882, - 883, 884, 885, 0, 0, 537, 538, 539, 562, 0, - 540, 522, 586, 386, 316, 502, 529, 722, 0, 0, - 0, 0, 0, 0, 0, 637, 648, 682, 0, 694, - 695, 697, 699, 942, 701, 495, 496, 709, 0, 0, - 928, 704, 705, 702, 426, 482, 503, 489, 896, 728, - 577, 578, 729, 690, 317, 0, 844, 453, 0, 0, - 592, 626, 615, 700, 580, 0, 0, 0, 0, 0, - 0, 847, 0, 0, 0, 369, 4713, 0, 421, 630, - 611, 622, 612, 597, 598, 599, 606, 381, 600, 601, - 602, 572, 603, 573, 604, 605, 887, 629, 579, 491, - 437, 0, 646, 0, 0, 967, 975, 0, 0, 0, - 0, 0, 0, 0, 0, 963, 0, 0, 0, 0, - 839, 0, 0, 876, 944, 943, 863, 873, 0, 0, - 337, 246, 574, 696, 576, 575, 864, 0, 865, 869, - 872, 868, 866, 867, 0, 958, 0, 0, 0, 0, - 0, 0, 831, 843, 0, 848, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 840, 841, 0, 0, 0, 0, 897, 0, 842, - 0, 0, 0, 0, 0, 492, 521, 0, 534, 0, - 406, 407, 892, 870, 874, 0, 0, 0, 0, 324, - 499, 518, 338, 486, 532, 343, 494, 511, 333, 452, - 483, 0, 0, 326, 516, 493, 434, 325, 0, 477, - 366, 383, 363, 450, 871, 0, 895, 899, 362, 981, - 893, 526, 328, 0, 525, 449, 512, 517, 435, 428, - 0, 327, 514, 433, 427, 412, 373, 982, 413, 414, - 387, 464, 425, 465, 388, 439, 438, 440, 389, 390, + 0, 0, 0, 0, 0, 0, 0, 0, 2261, 0, + 0, 0, 0, 0, 0, 0, 0, 2249, 0, 0, + 0, 0, 2268, 0, 0, 0, 0, 0, 0, 0, + 0, 3727, 0, 0, 0, 0, 0, 0, 0, 0, + 2244, 0, 0, 0, 2243, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 902, 2261, 0, + 0, 0, 0, 0, 0, 0, 458, 2249, 0, 597, + 631, 620, 705, 585, 0, 0, 0, 0, 0, 155, + 852, 0, 0, 0, 369, 0, 0, 426, 635, 616, + 627, 617, 602, 603, 604, 611, 381, 605, 606, 607, + 577, 608, 578, 609, 610, 893, 634, 584, 496, 442, + 0, 651, 0, 0, 973, 981, 0, 0, 0, 0, + 0, 0, 0, 0, 969, 0, 0, 0, 0, 844, + 0, 0, 882, 950, 949, 869, 879, 0, 0, 337, + 246, 579, 701, 581, 580, 870, 0, 871, 875, 878, + 874, 872, 873, 0, 964, 0, 0, 0, 0, 0, + 0, 836, 848, 0, 853, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 845, 846, 0, 0, 0, 0, 903, 0, 847, 0, + 4225, 0, 0, 0, 497, 526, 0, 539, 0, 407, + 408, 898, 876, 880, 0, 0, 0, 0, 324, 504, + 523, 338, 491, 537, 343, 499, 516, 333, 457, 488, + 0, 0, 326, 521, 498, 439, 325, 0, 482, 366, + 383, 363, 455, 877, 0, 901, 905, 362, 987, 899, + 531, 328, 0, 530, 454, 517, 522, 440, 433, 0, + 327, 519, 438, 432, 413, 373, 988, 414, 415, 416, + 417, 418, 419, 387, 469, 430, 470, 388, 444, 443, + 445, 389, 390, 391, 392, 393, 394, 395, 396, 397, + 398, 0, 0, 0, 0, 0, 561, 562, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 4330, 0, 694, 896, 0, 698, 0, 533, 0, + 0, 971, 155, 0, 0, 502, 0, 0, 420, 0, + 0, 0, 900, 0, 485, 460, 984, 0, 0, 483, + 428, 518, 471, 524, 505, 532, 477, 472, 318, 506, + 365, 441, 334, 336, 726, 367, 370, 374, 375, 450, + 451, 465, 490, 509, 510, 511, 364, 348, 484, 349, + 384, 350, 319, 356, 354, 357, 492, 358, 321, 466, + 515, 0, 380, 480, 436, 322, 435, 467, 514, 513, + 335, 541, 548, 549, 639, 0, 554, 737, 738, 739, + 563, 0, 473, 331, 330, 0, 0, 0, 360, 468, + 344, 346, 347, 345, 463, 464, 568, 569, 570, 572, + 0, 573, 574, 0, 0, 0, 0, 575, 640, 656, + 624, 593, 556, 648, 590, 594, 595, 401, 402, 403, + 865, 659, 2015, 2014, 2016, 547, 421, 422, 0, 372, + 371, 437, 323, 0, 0, 410, 400, 474, 329, 368, + 412, 406, 423, 424, 425, 378, 313, 314, 732, 968, + 456, 661, 696, 697, 586, 0, 983, 963, 965, 966, + 970, 974, 975, 976, 977, 978, 980, 982, 986, 731, + 0, 641, 655, 735, 654, 728, 462, 0, 489, 652, + 599, 0, 645, 618, 619, 0, 646, 614, 650, 0, + 588, 0, 557, 560, 589, 674, 675, 676, 320, 559, + 678, 679, 680, 681, 682, 683, 684, 677, 985, 622, + 598, 625, 538, 601, 600, 0, 0, 636, 904, 637, + 638, 446, 447, 448, 449, 972, 662, 342, 558, 476, + 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, + 628, 629, 626, 740, 0, 685, 686, 0, 0, 552, + 553, 377, 0, 571, 385, 341, 461, 379, 536, 409, + 0, 564, 630, 565, 478, 479, 688, 693, 689, 690, + 692, 712, 453, 399, 405, 493, 411, 429, 481, 535, + 459, 486, 339, 525, 495, 434, 615, 643, 994, 967, + 993, 995, 996, 992, 997, 998, 979, 857, 0, 911, + 912, 990, 989, 991, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 670, 669, 668, 667, 666, + 665, 664, 663, 0, 0, 612, 512, 355, 307, 351, + 352, 359, 729, 725, 730, 713, 716, 715, 691, 864, + 315, 592, 427, 475, 376, 657, 658, 0, 711, 957, + 920, 921, 922, 854, 923, 917, 918, 855, 919, 958, + 909, 954, 955, 884, 914, 924, 953, 925, 956, 885, + 959, 999, 1000, 931, 915, 277, 1001, 928, 960, 952, + 951, 926, 910, 961, 962, 892, 887, 929, 930, 916, + 937, 938, 939, 942, 856, 943, 944, 945, 946, 947, + 941, 940, 906, 907, 908, 932, 933, 935, 936, 913, + 503, 888, 889, 890, 891, 0, 0, 542, 543, 544, + 567, 0, 545, 527, 591, 386, 316, 507, 534, 727, + 0, 0, 0, 0, 0, 0, 0, 642, 653, 687, + 0, 699, 700, 702, 704, 948, 706, 500, 501, 714, + 0, 0, 934, 709, 710, 707, 431, 487, 508, 494, + 0, 733, 582, 583, 734, 695, 317, 0, 849, 183, + 223, 902, 0, 0, 0, 0, 0, 0, 0, 0, + 458, 0, 0, 597, 631, 620, 705, 585, 0, 0, + 0, 0, 0, 0, 852, 0, 0, 0, 369, 0, + 0, 426, 635, 616, 627, 617, 602, 603, 604, 611, + 381, 605, 606, 607, 577, 608, 578, 609, 610, 893, + 634, 584, 496, 442, 0, 651, 0, 0, 973, 981, + 0, 0, 0, 0, 0, 0, 0, 0, 969, 0, + 0, 0, 0, 844, 0, 0, 882, 950, 949, 869, + 879, 0, 0, 337, 246, 579, 701, 581, 580, 870, + 0, 871, 875, 878, 874, 872, 873, 0, 964, 0, + 0, 0, 0, 0, 0, 836, 848, 0, 853, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 845, 846, 0, 0, 0, 0, + 903, 0, 847, 0, 0, 0, 0, 0, 497, 526, + 0, 539, 0, 407, 408, 898, 876, 880, 0, 0, + 0, 0, 324, 504, 523, 338, 491, 537, 343, 499, + 516, 333, 457, 488, 0, 0, 326, 521, 498, 439, + 325, 0, 482, 366, 383, 363, 455, 877, 0, 901, + 905, 362, 987, 899, 531, 328, 0, 530, 454, 517, + 522, 440, 433, 0, 327, 519, 438, 432, 413, 373, + 988, 414, 415, 416, 417, 418, 419, 387, 469, 430, + 470, 388, 444, 443, 445, 389, 390, 391, 392, 393, + 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, + 561, 562, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 694, 896, 0, + 698, 0, 533, 0, 0, 971, 0, 0, 0, 502, + 0, 0, 420, 0, 0, 0, 900, 0, 485, 460, + 984, 0, 0, 483, 428, 518, 471, 524, 505, 532, + 477, 472, 318, 506, 365, 441, 334, 336, 726, 367, + 370, 374, 375, 450, 451, 465, 490, 509, 510, 511, + 364, 348, 484, 349, 384, 350, 319, 356, 354, 357, + 492, 358, 321, 466, 515, 0, 380, 480, 436, 322, + 435, 467, 514, 513, 335, 541, 548, 549, 639, 0, + 554, 737, 738, 739, 563, 0, 473, 331, 330, 0, + 0, 0, 360, 468, 344, 346, 347, 345, 463, 464, + 568, 569, 570, 572, 0, 573, 574, 0, 0, 0, + 0, 575, 640, 656, 624, 593, 556, 648, 590, 594, + 595, 401, 402, 403, 865, 659, 0, 0, 0, 547, + 421, 422, 0, 372, 371, 437, 323, 0, 0, 410, + 400, 474, 329, 368, 412, 406, 423, 424, 425, 378, + 313, 314, 732, 968, 456, 661, 696, 697, 586, 0, + 983, 963, 965, 966, 970, 974, 975, 976, 977, 978, + 980, 982, 986, 731, 0, 641, 655, 735, 654, 728, + 462, 0, 489, 652, 599, 0, 645, 618, 619, 0, + 646, 614, 650, 0, 588, 0, 557, 560, 589, 674, + 675, 676, 320, 559, 678, 679, 680, 681, 682, 683, + 684, 677, 985, 622, 598, 625, 538, 601, 600, 0, + 0, 636, 904, 637, 638, 446, 447, 448, 449, 972, + 662, 342, 558, 476, 0, 623, 0, 0, 0, 0, + 0, 0, 0, 0, 628, 629, 626, 740, 0, 685, + 686, 0, 0, 552, 553, 377, 0, 571, 385, 341, + 461, 379, 536, 409, 0, 564, 630, 565, 478, 479, + 688, 693, 689, 690, 692, 712, 453, 399, 405, 493, + 411, 429, 481, 535, 459, 486, 339, 525, 495, 434, + 615, 643, 994, 967, 993, 995, 996, 992, 997, 998, + 979, 857, 0, 911, 912, 990, 989, 991, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 670, + 669, 668, 667, 666, 665, 664, 663, 0, 0, 612, + 512, 355, 307, 351, 352, 359, 729, 725, 730, 713, + 716, 715, 691, 864, 315, 592, 427, 475, 376, 657, + 658, 0, 711, 957, 920, 921, 922, 854, 923, 917, + 918, 855, 919, 958, 909, 954, 955, 884, 914, 924, + 953, 925, 956, 885, 959, 999, 1000, 931, 915, 277, + 1001, 928, 960, 952, 951, 926, 910, 961, 962, 892, + 887, 929, 930, 916, 937, 938, 939, 942, 856, 943, + 944, 945, 946, 947, 941, 940, 906, 907, 908, 932, + 933, 935, 936, 913, 503, 888, 889, 890, 891, 0, + 0, 542, 543, 544, 567, 0, 545, 527, 591, 386, + 316, 507, 534, 727, 0, 0, 0, 0, 0, 0, + 0, 642, 653, 687, 0, 699, 700, 702, 704, 948, + 706, 500, 501, 714, 0, 0, 934, 709, 710, 707, + 431, 487, 508, 494, 902, 733, 582, 583, 734, 695, + 317, 0, 849, 458, 0, 0, 597, 631, 620, 705, + 585, 0, 0, 0, 0, 0, 0, 852, 0, 0, + 0, 369, 2082, 0, 426, 635, 616, 627, 617, 602, + 603, 604, 611, 381, 605, 606, 607, 577, 608, 578, + 609, 610, 893, 634, 584, 496, 442, 0, 651, 0, + 0, 973, 981, 0, 0, 0, 0, 0, 0, 0, + 0, 969, 0, 2336, 0, 0, 844, 0, 0, 882, + 950, 949, 869, 879, 0, 0, 337, 246, 579, 701, + 581, 580, 870, 0, 871, 875, 878, 874, 872, 873, + 0, 964, 0, 0, 0, 0, 0, 0, 836, 848, + 0, 853, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 845, 846, 0, + 0, 0, 0, 903, 0, 847, 0, 0, 0, 0, + 0, 497, 526, 0, 539, 0, 407, 408, 2337, 876, + 880, 0, 0, 0, 0, 324, 504, 523, 338, 491, + 537, 343, 499, 516, 333, 457, 488, 0, 0, 326, + 521, 498, 439, 325, 0, 482, 366, 383, 363, 455, + 877, 0, 901, 905, 362, 987, 899, 531, 328, 0, + 530, 454, 517, 522, 440, 433, 0, 327, 519, 438, + 432, 413, 373, 988, 414, 415, 416, 417, 418, 419, + 387, 469, 430, 470, 388, 444, 443, 445, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, 0, - 0, 0, 0, 556, 557, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 689, 890, 0, 693, 0, 528, 0, 0, 965, 0, - 0, 0, 497, 0, 0, 415, 0, 0, 0, 894, - 0, 480, 455, 978, 0, 0, 478, 423, 513, 466, - 519, 500, 527, 472, 467, 318, 501, 365, 436, 334, - 336, 721, 367, 370, 374, 375, 445, 446, 460, 485, - 504, 505, 506, 364, 348, 479, 349, 384, 350, 319, - 356, 354, 357, 487, 358, 321, 461, 510, 0, 380, - 475, 431, 322, 430, 462, 509, 508, 335, 536, 543, - 544, 634, 0, 549, 732, 733, 734, 558, 0, 468, - 331, 330, 0, 0, 0, 360, 463, 344, 346, 347, - 345, 458, 459, 563, 564, 565, 567, 0, 568, 569, - 0, 0, 0, 0, 570, 635, 651, 619, 588, 551, - 643, 585, 589, 590, 401, 402, 403, 654, 0, 0, - 0, 542, 416, 417, 0, 372, 371, 432, 323, 0, - 0, 409, 400, 469, 329, 368, 411, 405, 418, 419, - 420, 378, 313, 314, 727, 962, 451, 656, 691, 692, - 581, 0, 977, 957, 959, 960, 964, 968, 969, 970, - 971, 972, 974, 976, 980, 726, 0, 636, 650, 730, - 649, 723, 457, 0, 484, 647, 594, 0, 640, 613, - 614, 0, 641, 609, 645, 0, 583, 0, 552, 555, - 584, 669, 670, 671, 320, 554, 673, 674, 675, 676, - 677, 678, 679, 672, 979, 617, 593, 620, 533, 596, - 595, 0, 0, 631, 898, 632, 633, 441, 442, 443, - 444, 966, 657, 342, 553, 471, 0, 618, 0, 0, - 0, 0, 0, 0, 0, 0, 623, 624, 621, 735, - 0, 680, 681, 0, 0, 547, 548, 377, 0, 566, - 385, 341, 456, 379, 531, 408, 0, 559, 625, 560, - 473, 474, 683, 688, 684, 685, 687, 707, 448, 399, - 404, 488, 410, 424, 476, 530, 454, 481, 339, 520, - 490, 429, 610, 638, 988, 961, 987, 989, 990, 986, - 991, 992, 973, 852, 0, 905, 906, 984, 983, 985, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 665, 664, 663, 662, 661, 660, 659, 658, 0, - 0, 607, 507, 355, 307, 351, 352, 359, 724, 720, - 725, 708, 711, 710, 686, 859, 315, 587, 422, 470, - 376, 652, 653, 0, 706, 951, 914, 915, 916, 849, - 917, 911, 912, 850, 913, 952, 903, 948, 949, 878, - 908, 918, 947, 919, 950, 879, 953, 993, 994, 925, - 909, 277, 995, 922, 954, 946, 945, 920, 904, 955, - 956, 886, 881, 923, 924, 910, 931, 932, 933, 936, - 851, 937, 938, 939, 940, 941, 935, 934, 900, 901, - 902, 926, 927, 929, 930, 907, 498, 882, 883, 884, - 885, 0, 0, 537, 538, 539, 562, 0, 540, 522, - 586, 386, 316, 502, 529, 722, 0, 0, 0, 0, - 0, 0, 0, 637, 648, 682, 0, 694, 695, 697, - 699, 942, 701, 495, 496, 709, 0, 0, 928, 704, - 705, 702, 426, 482, 503, 489, 896, 728, 577, 578, - 729, 690, 317, 0, 844, 453, 0, 0, 592, 626, - 615, 700, 580, 0, 0, 0, 0, 0, 0, 847, - 0, 0, 0, 369, 0, 0, 421, 630, 611, 622, - 612, 597, 598, 599, 606, 381, 600, 601, 602, 572, - 603, 573, 604, 605, 887, 629, 579, 491, 437, 0, - 646, 0, 0, 967, 975, 0, 0, 0, 0, 0, - 0, 0, 0, 963, 0, 0, 0, 0, 839, 0, - 0, 876, 944, 943, 863, 873, 0, 0, 337, 246, - 574, 696, 576, 575, 864, 0, 865, 869, 872, 868, - 866, 867, 0, 958, 0, 0, 0, 0, 0, 0, - 831, 843, 0, 848, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 840, - 841, 0, 0, 0, 0, 897, 0, 842, 0, 0, - 0, 0, 0, 492, 521, 0, 534, 0, 406, 407, - 892, 870, 874, 0, 0, 0, 0, 324, 499, 518, - 338, 486, 532, 343, 494, 511, 333, 452, 483, 0, - 0, 326, 516, 493, 434, 325, 0, 477, 366, 383, - 363, 450, 871, 0, 895, 899, 362, 981, 893, 526, - 328, 0, 525, 449, 512, 517, 435, 428, 0, 327, - 514, 433, 427, 412, 373, 982, 413, 414, 387, 464, - 425, 465, 388, 439, 438, 440, 389, 390, 391, 392, - 393, 394, 395, 396, 397, 398, 0, 0, 0, 0, - 0, 556, 557, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 689, 890, - 0, 693, 0, 528, 0, 0, 965, 0, 0, 0, - 497, 0, 0, 415, 0, 0, 0, 894, 0, 480, - 455, 978, 4596, 0, 478, 423, 513, 466, 519, 500, - 527, 472, 467, 318, 501, 365, 436, 334, 336, 721, - 367, 370, 374, 375, 445, 446, 460, 485, 504, 505, - 506, 364, 348, 479, 349, 384, 350, 319, 356, 354, - 357, 487, 358, 321, 461, 510, 0, 380, 475, 431, - 322, 430, 462, 509, 508, 335, 536, 543, 544, 634, - 0, 549, 732, 733, 734, 558, 0, 468, 331, 330, - 0, 0, 0, 360, 463, 344, 346, 347, 345, 458, - 459, 563, 564, 565, 567, 0, 568, 569, 0, 0, - 0, 0, 570, 635, 651, 619, 588, 551, 643, 585, - 589, 590, 401, 402, 403, 654, 0, 0, 0, 542, - 416, 417, 0, 372, 371, 432, 323, 0, 0, 409, - 400, 469, 329, 368, 411, 405, 418, 419, 420, 378, - 313, 314, 727, 962, 451, 656, 691, 692, 581, 0, - 977, 957, 959, 960, 964, 968, 969, 970, 971, 972, - 974, 976, 980, 726, 0, 636, 650, 730, 649, 723, - 457, 0, 484, 647, 594, 0, 640, 613, 614, 0, - 641, 609, 645, 0, 583, 0, 552, 555, 584, 669, - 670, 671, 320, 554, 673, 674, 675, 676, 677, 678, - 679, 672, 979, 617, 593, 620, 533, 596, 595, 0, - 0, 631, 898, 632, 633, 441, 442, 443, 444, 966, - 657, 342, 553, 471, 0, 618, 0, 0, 0, 0, - 0, 0, 0, 0, 623, 624, 621, 735, 0, 680, - 681, 0, 0, 547, 548, 377, 0, 566, 385, 341, - 456, 379, 531, 408, 0, 559, 625, 560, 473, 474, - 683, 688, 684, 685, 687, 707, 448, 399, 404, 488, - 410, 424, 476, 530, 454, 481, 339, 520, 490, 429, - 610, 638, 988, 961, 987, 989, 990, 986, 991, 992, - 973, 852, 0, 905, 906, 984, 983, 985, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, - 664, 663, 662, 661, 660, 659, 658, 0, 0, 607, - 507, 355, 307, 351, 352, 359, 724, 720, 725, 708, - 711, 710, 686, 859, 315, 587, 422, 470, 376, 652, - 653, 0, 706, 951, 914, 915, 916, 849, 917, 911, - 912, 850, 913, 952, 903, 948, 949, 878, 908, 918, - 947, 919, 950, 879, 953, 993, 994, 925, 909, 277, - 995, 922, 954, 946, 945, 920, 904, 955, 956, 886, - 881, 923, 924, 910, 931, 932, 933, 936, 851, 937, - 938, 939, 940, 941, 935, 934, 900, 901, 902, 926, - 927, 929, 930, 907, 498, 882, 883, 884, 885, 0, - 0, 537, 538, 539, 562, 0, 540, 522, 586, 386, - 316, 502, 529, 722, 0, 0, 0, 0, 0, 0, - 0, 637, 648, 682, 0, 694, 695, 697, 699, 942, - 701, 495, 496, 709, 0, 0, 928, 704, 705, 702, - 426, 482, 503, 489, 896, 728, 577, 578, 729, 690, - 317, 0, 844, 453, 0, 0, 592, 626, 615, 700, - 580, 0, 0, 0, 0, 0, 0, 847, 0, 0, - 0, 369, 2066, 0, 421, 630, 611, 622, 612, 597, - 598, 599, 606, 381, 600, 601, 602, 572, 603, 573, - 604, 605, 887, 629, 579, 491, 437, 0, 646, 0, - 0, 967, 975, 0, 0, 0, 0, 0, 0, 0, - 0, 963, 0, 0, 0, 0, 839, 0, 0, 876, - 944, 943, 863, 873, 0, 0, 337, 246, 574, 696, - 576, 575, 864, 0, 865, 869, 872, 868, 866, 867, - 0, 958, 0, 0, 0, 0, 0, 0, 831, 843, - 0, 848, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 840, 841, 0, - 0, 0, 0, 897, 0, 842, 0, 0, 0, 0, - 0, 492, 521, 0, 534, 0, 406, 407, 892, 870, - 874, 0, 0, 0, 0, 324, 499, 518, 338, 486, - 532, 343, 494, 511, 333, 452, 483, 0, 0, 326, - 516, 493, 434, 325, 0, 477, 366, 383, 363, 450, - 871, 0, 895, 899, 362, 981, 893, 526, 328, 0, - 525, 449, 512, 517, 435, 428, 0, 327, 514, 433, - 427, 412, 373, 982, 413, 414, 387, 464, 425, 465, - 388, 439, 438, 440, 389, 390, 391, 392, 393, 394, - 395, 396, 397, 398, 0, 0, 0, 0, 0, 556, - 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 689, 890, 0, 693, - 0, 528, 0, 0, 965, 0, 0, 0, 497, 0, - 0, 415, 0, 0, 0, 894, 0, 480, 455, 978, - 0, 0, 478, 423, 513, 466, 519, 500, 527, 472, - 467, 318, 501, 365, 436, 334, 336, 721, 367, 370, - 374, 375, 445, 446, 460, 485, 504, 505, 506, 364, - 348, 479, 349, 384, 350, 319, 356, 354, 357, 487, - 358, 321, 461, 510, 0, 380, 475, 431, 322, 430, - 462, 509, 508, 335, 536, 543, 544, 634, 0, 549, - 732, 733, 734, 558, 0, 468, 331, 330, 0, 0, - 0, 360, 463, 344, 346, 347, 345, 458, 459, 563, - 564, 565, 567, 0, 568, 569, 0, 0, 0, 0, - 570, 635, 651, 619, 588, 551, 643, 585, 589, 590, - 401, 402, 403, 654, 0, 0, 0, 542, 416, 417, - 0, 372, 371, 432, 323, 0, 0, 409, 400, 469, - 329, 368, 411, 405, 418, 419, 420, 378, 313, 314, - 727, 962, 451, 656, 691, 692, 581, 0, 977, 957, - 959, 960, 964, 968, 969, 970, 971, 972, 974, 976, - 980, 726, 0, 636, 650, 730, 649, 723, 457, 0, - 484, 647, 594, 0, 640, 613, 614, 0, 641, 609, - 645, 0, 583, 0, 552, 555, 584, 669, 670, 671, - 320, 554, 673, 674, 675, 676, 677, 678, 679, 672, - 979, 617, 593, 620, 533, 596, 595, 0, 0, 631, - 898, 632, 633, 441, 442, 443, 444, 966, 657, 342, - 553, 471, 0, 618, 0, 0, 0, 0, 0, 0, - 0, 0, 623, 624, 621, 735, 0, 680, 681, 0, - 0, 547, 548, 377, 0, 566, 385, 341, 456, 379, - 531, 408, 0, 559, 625, 560, 473, 474, 683, 688, - 684, 685, 687, 707, 448, 399, 404, 488, 410, 424, - 476, 530, 454, 481, 339, 520, 490, 429, 610, 638, - 988, 961, 987, 989, 990, 986, 991, 992, 973, 852, - 0, 905, 906, 984, 983, 985, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 665, 664, 663, - 662, 661, 660, 659, 658, 0, 0, 607, 507, 355, - 307, 351, 352, 359, 724, 720, 725, 708, 711, 710, - 686, 859, 315, 587, 422, 470, 376, 652, 653, 0, - 706, 951, 914, 915, 916, 849, 917, 911, 912, 850, - 913, 952, 903, 948, 949, 878, 908, 918, 947, 919, - 950, 879, 953, 993, 994, 925, 909, 277, 995, 922, - 954, 946, 945, 920, 904, 955, 956, 886, 881, 923, - 924, 910, 931, 932, 933, 936, 851, 937, 938, 939, - 940, 941, 935, 934, 900, 901, 902, 926, 927, 929, - 930, 907, 498, 882, 883, 884, 885, 0, 0, 537, - 538, 539, 562, 0, 540, 522, 586, 386, 316, 502, - 529, 722, 0, 0, 0, 0, 0, 0, 0, 637, - 648, 682, 0, 694, 695, 697, 699, 942, 701, 495, - 496, 709, 0, 0, 928, 704, 705, 702, 426, 482, - 503, 489, 896, 728, 577, 578, 729, 690, 317, 0, - 844, 453, 0, 0, 592, 626, 615, 700, 580, 0, - 0, 0, 0, 0, 0, 847, 0, 0, 0, 369, - 0, 0, 421, 630, 611, 622, 612, 597, 598, 599, - 606, 381, 600, 601, 602, 572, 603, 573, 604, 605, - 887, 629, 579, 491, 437, 0, 646, 0, 0, 967, - 975, 0, 0, 0, 0, 0, 0, 0, 0, 963, - 0, 0, 0, 0, 839, 0, 0, 876, 944, 943, - 863, 873, 0, 0, 337, 246, 574, 696, 576, 575, - 864, 0, 865, 869, 872, 868, 866, 867, 0, 958, - 0, 0, 0, 0, 0, 0, 831, 843, 0, 848, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 840, 841, 1761, 0, 0, - 0, 897, 0, 842, 0, 0, 0, 0, 0, 492, - 521, 0, 534, 0, 406, 407, 892, 870, 874, 0, - 0, 0, 0, 324, 499, 518, 338, 486, 532, 343, - 494, 511, 333, 452, 483, 0, 0, 326, 516, 493, - 434, 325, 0, 477, 366, 383, 363, 450, 871, 0, - 895, 899, 362, 981, 893, 526, 328, 0, 525, 449, - 512, 517, 435, 428, 0, 327, 514, 433, 427, 412, - 373, 982, 413, 414, 387, 464, 425, 465, 388, 439, - 438, 440, 389, 390, 391, 392, 393, 394, 395, 396, - 397, 398, 0, 0, 0, 0, 0, 556, 557, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 689, 890, 0, 693, 0, 528, - 0, 0, 965, 0, 0, 0, 497, 0, 0, 415, - 0, 0, 0, 894, 0, 480, 455, 978, 0, 0, - 478, 423, 513, 466, 519, 500, 527, 472, 467, 318, - 501, 365, 436, 334, 336, 721, 367, 370, 374, 375, - 445, 446, 460, 485, 504, 505, 506, 364, 348, 479, - 349, 384, 350, 319, 356, 354, 357, 487, 358, 321, - 461, 510, 0, 380, 475, 431, 322, 430, 462, 509, - 508, 335, 536, 543, 544, 634, 0, 549, 732, 733, - 734, 558, 0, 468, 331, 330, 0, 0, 0, 360, - 463, 344, 346, 347, 345, 458, 459, 563, 564, 565, - 567, 0, 568, 569, 0, 0, 0, 0, 570, 635, - 651, 619, 588, 551, 643, 585, 589, 590, 401, 402, - 403, 654, 0, 0, 0, 542, 416, 417, 0, 372, - 371, 432, 323, 0, 0, 409, 400, 469, 329, 368, - 411, 405, 418, 419, 420, 378, 313, 314, 727, 962, - 451, 656, 691, 692, 581, 0, 977, 957, 959, 960, - 964, 968, 969, 970, 971, 972, 974, 976, 980, 726, - 0, 636, 650, 730, 649, 723, 457, 0, 484, 647, - 594, 0, 640, 613, 614, 0, 641, 609, 645, 0, - 583, 0, 552, 555, 584, 669, 670, 671, 320, 554, - 673, 674, 675, 676, 677, 678, 679, 672, 979, 617, - 593, 620, 533, 596, 595, 0, 0, 631, 898, 632, - 633, 441, 442, 443, 444, 966, 657, 342, 553, 471, - 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, - 623, 624, 621, 735, 0, 680, 681, 0, 0, 547, - 548, 377, 0, 566, 385, 341, 456, 379, 531, 408, - 0, 559, 625, 560, 473, 474, 683, 688, 684, 685, - 687, 707, 448, 399, 404, 488, 410, 424, 476, 530, - 454, 481, 339, 520, 490, 429, 610, 638, 988, 961, - 987, 989, 990, 986, 991, 992, 973, 852, 0, 905, - 906, 984, 983, 985, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 665, 664, 663, 662, 661, - 660, 659, 658, 0, 0, 607, 507, 355, 307, 351, - 352, 359, 724, 720, 725, 708, 711, 710, 686, 859, - 315, 587, 422, 470, 376, 652, 653, 0, 706, 951, - 914, 915, 916, 849, 917, 911, 912, 850, 913, 952, - 903, 948, 949, 878, 908, 918, 947, 919, 950, 879, - 953, 993, 994, 925, 909, 277, 995, 922, 954, 946, - 945, 920, 904, 955, 956, 886, 881, 923, 924, 910, - 931, 932, 933, 936, 851, 937, 938, 939, 940, 941, - 935, 934, 900, 901, 902, 926, 927, 929, 930, 907, - 498, 882, 883, 884, 885, 0, 0, 537, 538, 539, - 562, 0, 540, 522, 586, 386, 316, 502, 529, 722, - 0, 0, 0, 0, 0, 0, 0, 637, 648, 682, - 0, 694, 695, 697, 699, 942, 701, 495, 496, 709, - 0, 0, 928, 704, 705, 702, 426, 482, 503, 489, - 0, 728, 577, 578, 729, 690, 317, 896, 844, 0, - 2497, 0, 0, 0, 0, 0, 453, 0, 0, 592, - 626, 615, 700, 580, 0, 0, 0, 0, 0, 0, - 847, 0, 0, 0, 369, 0, 0, 421, 630, 611, - 622, 612, 597, 598, 599, 606, 381, 600, 601, 602, - 572, 603, 573, 604, 605, 887, 629, 579, 491, 437, - 0, 646, 0, 0, 967, 975, 0, 0, 0, 0, - 0, 0, 0, 0, 963, 0, 0, 0, 0, 839, - 0, 0, 876, 944, 943, 863, 873, 0, 0, 337, - 246, 574, 696, 576, 575, 864, 0, 865, 869, 872, - 868, 866, 867, 0, 958, 0, 0, 0, 0, 0, - 0, 831, 843, 0, 848, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 840, 841, 0, 0, 0, 0, 897, 0, 842, 0, - 0, 0, 0, 0, 492, 521, 0, 534, 0, 406, - 407, 892, 870, 874, 0, 0, 0, 0, 324, 499, - 518, 338, 486, 532, 343, 494, 511, 333, 452, 483, - 0, 0, 326, 516, 493, 434, 325, 0, 477, 366, - 383, 363, 450, 871, 0, 895, 899, 362, 981, 893, - 526, 328, 0, 525, 449, 512, 517, 435, 428, 0, - 327, 514, 433, 427, 412, 373, 982, 413, 414, 387, - 464, 425, 465, 388, 439, 438, 440, 389, 390, 391, - 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, - 0, 0, 556, 557, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, - 890, 0, 693, 0, 528, 0, 0, 965, 0, 0, - 0, 497, 0, 0, 415, 0, 0, 0, 894, 0, - 480, 455, 978, 0, 0, 478, 423, 513, 466, 519, - 500, 527, 472, 467, 318, 501, 365, 436, 334, 336, - 721, 367, 370, 374, 375, 445, 446, 460, 485, 504, - 505, 506, 364, 348, 479, 349, 384, 350, 319, 356, - 354, 357, 487, 358, 321, 461, 510, 0, 380, 475, - 431, 322, 430, 462, 509, 508, 335, 536, 543, 544, - 634, 0, 549, 732, 733, 734, 558, 0, 468, 331, - 330, 0, 0, 0, 360, 463, 344, 346, 347, 345, - 458, 459, 563, 564, 565, 567, 0, 568, 569, 0, - 0, 0, 0, 570, 635, 651, 619, 588, 551, 643, - 585, 589, 590, 401, 402, 403, 654, 0, 0, 0, - 542, 416, 417, 0, 372, 371, 432, 323, 0, 0, - 409, 400, 469, 329, 368, 411, 405, 418, 419, 420, - 378, 313, 314, 727, 962, 451, 656, 691, 692, 581, - 0, 977, 957, 959, 960, 964, 968, 969, 970, 971, - 972, 974, 976, 980, 726, 0, 636, 650, 730, 649, - 723, 457, 0, 484, 647, 594, 0, 640, 613, 614, - 0, 641, 609, 645, 0, 583, 0, 552, 555, 584, - 669, 670, 671, 320, 554, 673, 674, 675, 676, 677, - 678, 679, 672, 979, 617, 593, 620, 533, 596, 595, - 0, 0, 631, 898, 632, 633, 441, 442, 443, 444, - 966, 657, 342, 553, 471, 0, 618, 0, 0, 0, - 0, 0, 0, 0, 0, 623, 624, 621, 735, 0, - 680, 681, 0, 0, 547, 548, 377, 0, 566, 385, - 341, 456, 379, 531, 408, 0, 559, 625, 560, 473, - 474, 683, 688, 684, 685, 687, 707, 448, 399, 404, - 488, 410, 424, 476, 530, 454, 481, 339, 520, 490, - 429, 610, 638, 988, 961, 987, 989, 990, 986, 991, - 992, 973, 852, 0, 905, 906, 984, 983, 985, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 665, 664, 663, 662, 661, 660, 659, 658, 0, 0, - 607, 507, 355, 307, 351, 352, 359, 724, 720, 725, - 708, 711, 710, 686, 859, 315, 587, 422, 470, 376, - 652, 653, 0, 706, 951, 914, 915, 916, 849, 917, - 911, 912, 850, 913, 952, 903, 948, 949, 878, 908, - 918, 947, 919, 950, 879, 953, 993, 994, 925, 909, - 277, 995, 922, 954, 946, 945, 920, 904, 955, 956, - 886, 881, 923, 924, 910, 931, 932, 933, 936, 851, - 937, 938, 939, 940, 941, 935, 934, 900, 901, 902, - 926, 927, 929, 930, 907, 498, 882, 883, 884, 885, - 0, 0, 537, 538, 539, 562, 0, 540, 522, 586, - 386, 316, 502, 529, 722, 0, 0, 0, 0, 0, - 0, 0, 637, 648, 682, 0, 694, 695, 697, 699, - 942, 701, 495, 496, 709, 0, 0, 928, 704, 705, - 702, 426, 482, 503, 489, 896, 728, 577, 578, 729, - 690, 317, 0, 844, 453, 0, 0, 592, 626, 615, - 700, 580, 0, 0, 0, 0, 0, 0, 847, 0, - 0, 0, 369, 0, 0, 421, 630, 611, 622, 612, - 597, 598, 599, 606, 381, 600, 601, 602, 572, 603, - 573, 604, 605, 887, 629, 579, 491, 437, 0, 646, - 0, 0, 967, 975, 0, 0, 0, 0, 0, 0, - 0, 0, 963, 0, 0, 0, 0, 839, 0, 0, - 876, 944, 943, 863, 873, 0, 0, 337, 246, 574, - 696, 576, 575, 864, 0, 865, 869, 872, 868, 866, - 867, 0, 958, 0, 0, 0, 0, 0, 0, 831, - 843, 0, 848, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 840, 841, - 2059, 0, 0, 0, 897, 0, 842, 0, 0, 0, - 0, 0, 492, 521, 0, 534, 0, 406, 407, 892, - 870, 874, 0, 0, 0, 0, 324, 499, 518, 338, - 486, 532, 343, 494, 511, 333, 452, 483, 0, 0, - 326, 516, 493, 434, 325, 0, 477, 366, 383, 363, - 450, 871, 0, 895, 899, 362, 981, 893, 526, 328, - 0, 525, 449, 512, 517, 435, 428, 0, 327, 514, - 433, 427, 412, 373, 982, 413, 414, 387, 464, 425, - 465, 388, 439, 438, 440, 389, 390, 391, 392, 393, + 0, 0, 0, 561, 562, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 694, 896, 0, 698, 0, 533, 0, 0, 971, 0, + 0, 0, 502, 0, 0, 420, 0, 0, 0, 900, + 0, 485, 460, 984, 0, 0, 483, 428, 518, 471, + 524, 505, 532, 477, 472, 318, 506, 365, 441, 334, + 336, 726, 367, 370, 374, 375, 450, 451, 465, 490, + 509, 510, 511, 364, 348, 484, 349, 384, 350, 319, + 356, 354, 357, 492, 358, 321, 466, 515, 0, 380, + 480, 436, 322, 435, 467, 514, 513, 335, 541, 548, + 549, 639, 0, 554, 737, 738, 739, 563, 0, 473, + 331, 330, 0, 0, 0, 360, 468, 344, 346, 347, + 345, 463, 464, 568, 569, 570, 572, 0, 573, 574, + 0, 0, 0, 0, 575, 640, 656, 624, 593, 556, + 648, 590, 594, 595, 401, 402, 403, 865, 659, 0, + 0, 0, 547, 421, 422, 0, 372, 371, 437, 323, + 0, 0, 410, 400, 474, 329, 368, 412, 406, 423, + 424, 425, 378, 313, 314, 732, 968, 456, 661, 696, + 697, 586, 0, 983, 963, 965, 966, 970, 974, 975, + 976, 977, 978, 980, 982, 986, 731, 0, 641, 655, + 735, 654, 728, 462, 0, 489, 652, 599, 0, 645, + 618, 619, 0, 646, 614, 650, 0, 588, 0, 557, + 560, 589, 674, 675, 676, 320, 559, 678, 679, 680, + 681, 682, 683, 684, 677, 985, 622, 598, 625, 538, + 601, 600, 0, 0, 636, 904, 637, 638, 446, 447, + 448, 449, 972, 662, 342, 558, 476, 0, 623, 0, + 0, 0, 0, 0, 0, 0, 0, 628, 629, 626, + 740, 0, 685, 686, 0, 0, 552, 553, 377, 0, + 571, 385, 341, 461, 379, 536, 409, 0, 564, 630, + 565, 478, 479, 688, 693, 689, 690, 692, 712, 453, + 399, 405, 493, 411, 429, 481, 535, 459, 486, 339, + 525, 495, 434, 615, 643, 994, 967, 993, 995, 996, + 992, 997, 998, 979, 857, 0, 911, 912, 990, 989, + 991, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 670, 669, 668, 667, 666, 665, 664, 663, + 0, 0, 612, 512, 355, 307, 351, 352, 359, 729, + 725, 730, 713, 716, 715, 691, 864, 315, 592, 427, + 475, 376, 657, 658, 0, 711, 957, 920, 921, 922, + 854, 923, 917, 918, 855, 919, 958, 909, 954, 955, + 884, 914, 924, 953, 925, 956, 885, 959, 999, 1000, + 931, 915, 277, 1001, 928, 960, 952, 951, 926, 910, + 961, 962, 892, 887, 929, 930, 916, 937, 938, 939, + 942, 856, 943, 944, 945, 946, 947, 941, 940, 906, + 907, 908, 932, 933, 935, 936, 913, 503, 888, 889, + 890, 891, 0, 0, 542, 543, 544, 567, 0, 545, + 527, 591, 386, 316, 507, 534, 727, 0, 0, 0, + 0, 0, 0, 0, 642, 653, 687, 0, 699, 700, + 702, 704, 948, 706, 500, 501, 714, 0, 0, 934, + 709, 710, 707, 431, 487, 508, 494, 0, 733, 582, + 583, 734, 695, 317, 0, 849, 183, 223, 902, 0, + 0, 0, 0, 0, 0, 0, 0, 458, 0, 0, + 597, 631, 620, 705, 585, 0, 0, 0, 0, 0, + 0, 852, 0, 0, 0, 369, 0, 0, 426, 635, + 616, 627, 617, 602, 603, 604, 611, 381, 605, 606, + 607, 577, 608, 578, 609, 610, 1433, 634, 584, 496, + 442, 0, 651, 0, 0, 973, 981, 0, 0, 0, + 0, 0, 0, 0, 0, 969, 0, 0, 0, 0, + 844, 0, 0, 882, 950, 949, 869, 879, 0, 0, + 337, 246, 579, 701, 581, 580, 870, 0, 871, 875, + 878, 874, 872, 873, 0, 964, 0, 0, 0, 0, + 0, 0, 836, 848, 0, 853, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 845, 846, 0, 0, 0, 0, 903, 0, 847, + 0, 0, 0, 0, 0, 497, 526, 0, 539, 0, + 407, 408, 898, 876, 880, 0, 0, 0, 0, 324, + 504, 523, 338, 491, 537, 343, 499, 516, 333, 457, + 488, 0, 0, 326, 521, 498, 439, 325, 0, 482, + 366, 383, 363, 455, 877, 0, 901, 905, 362, 987, + 899, 531, 328, 0, 530, 454, 517, 522, 440, 433, + 0, 327, 519, 438, 432, 413, 373, 988, 414, 415, + 416, 417, 418, 419, 387, 469, 430, 470, 388, 444, + 443, 445, 389, 390, 391, 392, 393, 394, 395, 396, + 397, 398, 0, 0, 0, 0, 0, 561, 562, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 694, 896, 0, 698, 0, 533, + 0, 0, 971, 0, 0, 0, 502, 0, 0, 420, + 0, 0, 0, 900, 0, 485, 460, 984, 0, 0, + 483, 428, 518, 471, 524, 505, 532, 477, 472, 318, + 506, 365, 441, 334, 336, 726, 367, 370, 374, 375, + 450, 451, 465, 490, 509, 510, 511, 364, 348, 484, + 349, 384, 350, 319, 356, 354, 357, 492, 358, 321, + 466, 515, 0, 380, 480, 436, 322, 435, 467, 514, + 513, 335, 541, 548, 549, 639, 0, 554, 737, 738, + 739, 563, 0, 473, 331, 330, 0, 0, 0, 360, + 468, 344, 346, 347, 345, 463, 464, 568, 569, 570, + 572, 0, 573, 574, 0, 0, 0, 0, 575, 640, + 656, 624, 593, 556, 648, 590, 594, 595, 401, 402, + 403, 865, 659, 0, 0, 0, 547, 421, 422, 0, + 372, 371, 437, 323, 0, 0, 410, 400, 474, 329, + 368, 412, 406, 423, 424, 425, 378, 313, 314, 732, + 968, 456, 661, 696, 697, 586, 0, 983, 963, 965, + 966, 970, 974, 975, 976, 977, 978, 980, 982, 986, + 731, 0, 641, 655, 735, 654, 728, 462, 0, 489, + 652, 599, 0, 645, 618, 619, 0, 646, 614, 650, + 0, 588, 0, 557, 560, 589, 674, 675, 676, 320, + 559, 678, 679, 680, 681, 682, 683, 684, 677, 985, + 622, 598, 625, 538, 601, 600, 0, 0, 636, 904, + 637, 638, 446, 447, 448, 449, 972, 662, 342, 558, + 476, 0, 623, 0, 0, 0, 0, 0, 0, 0, + 0, 628, 629, 626, 740, 0, 685, 686, 0, 0, + 552, 553, 377, 0, 571, 385, 341, 461, 379, 536, + 409, 0, 564, 630, 565, 478, 479, 688, 693, 689, + 690, 692, 712, 453, 399, 405, 493, 411, 429, 481, + 535, 459, 486, 339, 525, 495, 434, 615, 643, 994, + 967, 993, 995, 996, 992, 997, 998, 979, 857, 0, + 911, 912, 990, 989, 991, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 670, 669, 668, 667, + 666, 665, 664, 663, 0, 0, 612, 512, 355, 307, + 351, 352, 359, 729, 725, 730, 713, 716, 715, 691, + 864, 315, 592, 427, 475, 376, 657, 658, 0, 711, + 957, 920, 921, 922, 854, 923, 917, 918, 855, 919, + 958, 909, 954, 955, 884, 914, 924, 953, 925, 956, + 885, 959, 999, 1000, 931, 915, 277, 1001, 928, 960, + 952, 951, 926, 910, 961, 962, 892, 887, 929, 930, + 916, 937, 938, 939, 942, 856, 943, 944, 945, 946, + 947, 941, 940, 906, 907, 908, 932, 933, 935, 936, + 913, 503, 888, 889, 890, 891, 0, 0, 542, 543, + 544, 567, 0, 545, 527, 591, 386, 316, 507, 534, + 727, 0, 0, 0, 0, 0, 0, 0, 642, 653, + 687, 0, 699, 700, 702, 704, 948, 706, 500, 501, + 714, 0, 0, 934, 709, 710, 707, 431, 487, 508, + 494, 902, 733, 582, 583, 734, 695, 317, 0, 849, + 458, 0, 0, 597, 631, 620, 705, 585, 0, 0, + 0, 0, 0, 0, 852, 0, 0, 0, 369, 4746, + 0, 426, 635, 616, 627, 617, 602, 603, 604, 611, + 381, 605, 606, 607, 577, 608, 578, 609, 610, 893, + 634, 584, 496, 442, 0, 651, 0, 0, 973, 981, + 0, 0, 0, 0, 0, 0, 0, 0, 969, 0, + 0, 0, 0, 844, 0, 0, 882, 950, 949, 869, + 879, 0, 0, 337, 246, 579, 701, 581, 580, 870, + 0, 871, 875, 878, 874, 872, 873, 0, 964, 0, + 0, 0, 0, 0, 0, 836, 848, 0, 853, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 845, 846, 0, 0, 0, 0, + 903, 0, 847, 0, 0, 0, 0, 0, 497, 526, + 0, 539, 0, 407, 408, 898, 876, 880, 0, 0, + 0, 0, 324, 504, 523, 338, 491, 537, 343, 499, + 516, 333, 457, 488, 0, 0, 326, 521, 498, 439, + 325, 0, 482, 366, 383, 363, 455, 877, 0, 901, + 905, 362, 987, 899, 531, 328, 0, 530, 454, 517, + 522, 440, 433, 0, 327, 519, 438, 432, 413, 373, + 988, 414, 415, 416, 417, 418, 419, 387, 469, 430, + 470, 388, 444, 443, 445, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, - 556, 557, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 689, 890, 0, - 693, 0, 528, 0, 0, 965, 0, 0, 0, 497, - 0, 0, 415, 0, 0, 0, 894, 0, 480, 455, - 978, 0, 0, 478, 423, 513, 466, 519, 500, 527, - 472, 467, 318, 501, 365, 436, 334, 336, 721, 367, - 370, 374, 375, 445, 446, 460, 485, 504, 505, 506, - 364, 348, 479, 349, 384, 350, 319, 356, 354, 357, - 487, 358, 321, 461, 510, 0, 380, 475, 431, 322, - 430, 462, 509, 508, 335, 536, 543, 544, 634, 0, - 549, 732, 733, 734, 558, 0, 468, 331, 330, 0, - 0, 0, 360, 463, 344, 346, 347, 345, 458, 459, - 563, 564, 565, 567, 0, 568, 569, 0, 0, 0, - 0, 570, 635, 651, 619, 588, 551, 643, 585, 589, - 590, 401, 402, 403, 654, 0, 0, 0, 542, 416, - 417, 0, 372, 371, 432, 323, 0, 0, 409, 400, - 469, 329, 368, 411, 405, 418, 419, 420, 378, 313, - 314, 727, 962, 451, 656, 691, 692, 581, 0, 977, - 957, 959, 960, 964, 968, 969, 970, 971, 972, 974, - 976, 980, 726, 0, 636, 650, 730, 649, 723, 457, - 0, 484, 647, 594, 0, 640, 613, 614, 0, 641, - 609, 645, 0, 583, 0, 552, 555, 584, 669, 670, - 671, 320, 554, 673, 674, 675, 676, 677, 678, 679, - 672, 979, 617, 593, 620, 533, 596, 595, 0, 0, - 631, 898, 632, 633, 441, 442, 443, 444, 966, 657, - 342, 553, 471, 0, 618, 0, 0, 0, 0, 0, - 0, 0, 0, 623, 624, 621, 735, 0, 680, 681, - 0, 0, 547, 548, 377, 0, 566, 385, 341, 456, - 379, 531, 408, 0, 559, 625, 560, 473, 474, 683, - 688, 684, 685, 687, 707, 448, 399, 404, 488, 410, - 424, 476, 530, 454, 481, 339, 520, 490, 429, 610, - 638, 988, 961, 987, 989, 990, 986, 991, 992, 973, - 852, 0, 905, 906, 984, 983, 985, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 665, 664, - 663, 662, 661, 660, 659, 658, 0, 0, 607, 507, - 355, 307, 351, 352, 359, 724, 720, 725, 708, 711, - 710, 686, 859, 315, 587, 422, 470, 376, 652, 653, - 0, 706, 951, 914, 915, 916, 849, 917, 911, 912, - 850, 913, 952, 903, 948, 949, 878, 908, 918, 947, - 919, 950, 879, 953, 993, 994, 925, 909, 277, 995, - 922, 954, 946, 945, 920, 904, 955, 956, 886, 881, - 923, 924, 910, 931, 932, 933, 936, 851, 937, 938, - 939, 940, 941, 935, 934, 900, 901, 902, 926, 927, - 929, 930, 907, 498, 882, 883, 884, 885, 0, 0, - 537, 538, 539, 562, 0, 540, 522, 586, 386, 316, - 502, 529, 722, 0, 0, 0, 0, 0, 0, 0, - 637, 648, 682, 0, 694, 695, 697, 699, 942, 701, - 495, 496, 709, 0, 0, 928, 704, 705, 702, 426, - 482, 503, 489, 896, 728, 577, 578, 729, 690, 317, - 0, 844, 453, 0, 0, 592, 626, 615, 700, 580, - 0, 0, 0, 0, 0, 0, 847, 0, 0, 0, - 369, 0, 0, 421, 630, 611, 622, 612, 597, 598, - 599, 606, 381, 600, 601, 602, 572, 603, 573, 604, - 605, 887, 629, 579, 491, 437, 0, 646, 0, 0, - 967, 975, 0, 0, 0, 0, 0, 0, 0, 0, - 963, 0, 0, 0, 0, 839, 0, 0, 876, 944, - 943, 863, 873, 0, 0, 337, 246, 574, 696, 576, - 575, 864, 0, 865, 869, 872, 868, 866, 867, 0, - 958, 0, 0, 0, 0, 0, 0, 831, 843, 0, - 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 840, 841, 0, 0, - 0, 0, 897, 0, 842, 0, 0, 0, 0, 0, - 492, 521, 0, 534, 0, 406, 407, 892, 870, 874, - 0, 0, 0, 0, 324, 499, 518, 338, 486, 532, - 343, 494, 511, 333, 452, 483, 0, 0, 326, 516, - 493, 434, 325, 0, 477, 366, 383, 363, 450, 871, - 0, 895, 899, 362, 981, 893, 526, 328, 0, 525, - 449, 512, 517, 435, 428, 0, 327, 514, 433, 427, - 412, 373, 982, 413, 414, 387, 464, 425, 465, 388, - 439, 438, 440, 389, 390, 391, 392, 393, 394, 395, - 396, 397, 398, 0, 0, 0, 0, 0, 556, 557, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 689, 890, 0, 693, 0, - 528, 0, 0, 965, 0, 0, 0, 497, 0, 0, - 415, 0, 0, 0, 894, 0, 480, 455, 978, 0, - 0, 478, 423, 513, 466, 519, 500, 527, 472, 467, - 318, 501, 365, 436, 334, 336, 721, 367, 370, 374, - 375, 445, 446, 460, 485, 504, 505, 506, 364, 348, - 479, 349, 384, 350, 319, 356, 354, 357, 487, 358, - 321, 461, 510, 0, 380, 475, 431, 322, 430, 462, - 509, 508, 335, 536, 543, 544, 634, 0, 549, 732, - 733, 734, 558, 0, 468, 331, 330, 0, 0, 0, - 360, 463, 344, 346, 347, 345, 458, 459, 563, 564, - 565, 567, 0, 568, 569, 0, 0, 0, 0, 570, - 635, 651, 619, 588, 551, 643, 585, 589, 590, 401, - 402, 403, 654, 0, 0, 0, 542, 416, 417, 0, - 372, 371, 432, 323, 0, 0, 409, 400, 469, 329, - 368, 411, 405, 418, 419, 420, 378, 313, 314, 727, - 962, 451, 656, 691, 692, 581, 0, 977, 957, 959, - 960, 964, 968, 969, 970, 971, 972, 974, 976, 980, - 726, 0, 636, 650, 730, 649, 723, 457, 0, 484, - 647, 594, 0, 640, 613, 614, 0, 641, 609, 645, - 0, 583, 0, 552, 555, 584, 669, 670, 671, 320, - 554, 673, 674, 675, 676, 677, 678, 679, 672, 979, - 617, 593, 620, 533, 596, 595, 0, 0, 631, 898, - 632, 633, 441, 442, 443, 444, 966, 657, 342, 553, - 471, 0, 618, 0, 0, 0, 0, 0, 0, 0, - 0, 623, 624, 621, 735, 0, 680, 681, 0, 0, - 547, 548, 377, 0, 566, 385, 341, 456, 379, 531, - 408, 0, 559, 625, 560, 473, 474, 683, 688, 684, - 685, 687, 707, 448, 399, 404, 488, 410, 424, 476, - 530, 454, 481, 339, 520, 490, 429, 610, 638, 988, - 961, 987, 989, 990, 986, 991, 992, 973, 852, 0, - 905, 906, 984, 983, 985, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 665, 664, 663, 662, - 661, 660, 659, 658, 0, 0, 607, 507, 355, 307, - 351, 352, 359, 724, 720, 725, 708, 711, 710, 686, - 859, 315, 587, 422, 470, 376, 652, 653, 0, 706, - 951, 914, 915, 916, 849, 917, 911, 912, 850, 913, - 952, 903, 948, 949, 878, 908, 918, 947, 919, 950, - 879, 953, 993, 994, 925, 909, 277, 995, 922, 954, - 946, 945, 920, 904, 955, 956, 886, 881, 923, 924, - 910, 931, 932, 933, 936, 851, 937, 938, 939, 940, - 941, 935, 934, 900, 901, 902, 926, 927, 929, 930, - 907, 498, 882, 883, 884, 885, 0, 0, 537, 538, - 539, 562, 0, 540, 522, 586, 386, 316, 502, 529, - 722, 0, 0, 0, 0, 0, 0, 0, 637, 648, - 682, 0, 694, 695, 697, 699, 942, 701, 495, 496, - 709, 0, 0, 928, 704, 705, 702, 426, 482, 503, - 489, 896, 728, 577, 578, 729, 690, 317, 0, 844, - 453, 0, 0, 592, 626, 615, 700, 580, 0, 0, - 0, 0, 0, 0, 847, 0, 0, 0, 369, 0, - 0, 421, 630, 611, 622, 612, 597, 598, 599, 606, - 381, 600, 601, 602, 572, 603, 573, 604, 605, 887, - 629, 579, 491, 437, 0, 646, 0, 0, 967, 975, - 0, 0, 0, 0, 0, 0, 0, 0, 963, 0, - 0, 0, 0, 839, 0, 0, 876, 944, 943, 863, - 873, 0, 0, 337, 246, 574, 696, 576, 575, 864, - 0, 865, 869, 872, 868, 866, 867, 0, 958, 0, - 0, 0, 0, 0, 0, 831, 843, 0, 848, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 840, 841, 0, 0, 0, 0, - 897, 0, 842, 0, 0, 0, 0, 0, 492, 521, - 0, 534, 0, 406, 407, 892, 870, 874, 0, 0, - 0, 0, 324, 499, 518, 338, 486, 532, 343, 494, - 511, 333, 452, 483, 0, 0, 326, 516, 493, 434, - 325, 0, 477, 366, 383, 363, 450, 871, 0, 895, - 899, 362, 981, 893, 526, 328, 0, 525, 449, 512, - 517, 435, 428, 0, 327, 514, 433, 427, 412, 373, - 982, 413, 414, 387, 464, 425, 465, 388, 439, 438, - 440, 389, 390, 391, 392, 393, 394, 395, 396, 397, - 398, 0, 0, 0, 0, 0, 556, 557, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 689, 890, 0, 693, 0, 528, 0, - 0, 965, 0, 0, 0, 497, 0, 0, 415, 0, - 0, 0, 894, 0, 480, 455, 978, 0, 0, 478, - 423, 513, 466, 519, 500, 527, 472, 467, 318, 501, - 365, 436, 334, 336, 721, 367, 370, 374, 375, 445, - 446, 460, 485, 504, 505, 506, 364, 348, 479, 349, - 384, 350, 319, 356, 354, 357, 487, 358, 321, 461, - 510, 0, 380, 475, 431, 322, 430, 462, 509, 508, - 335, 536, 543, 544, 634, 0, 549, 732, 733, 734, - 558, 0, 468, 331, 330, 0, 0, 0, 360, 463, - 344, 346, 347, 345, 458, 459, 563, 564, 565, 567, - 0, 568, 569, 0, 0, 0, 0, 570, 635, 651, - 619, 588, 551, 643, 585, 589, 590, 401, 402, 403, - 654, 0, 0, 0, 542, 416, 417, 0, 372, 371, - 432, 323, 0, 0, 409, 400, 469, 329, 368, 411, - 405, 418, 419, 420, 378, 313, 314, 727, 962, 451, - 656, 691, 692, 581, 0, 977, 957, 959, 960, 964, - 968, 969, 970, 971, 972, 974, 976, 980, 726, 0, - 636, 650, 730, 649, 723, 457, 0, 484, 647, 594, - 0, 640, 613, 614, 0, 641, 609, 645, 0, 583, - 0, 552, 555, 584, 669, 670, 671, 320, 554, 673, - 674, 675, 676, 677, 678, 679, 672, 979, 617, 593, - 620, 533, 596, 595, 0, 0, 631, 898, 632, 633, - 441, 442, 443, 444, 966, 657, 342, 553, 471, 0, - 618, 0, 0, 0, 0, 0, 0, 0, 0, 623, - 624, 621, 735, 0, 680, 681, 0, 0, 547, 548, - 377, 0, 566, 385, 341, 456, 379, 531, 408, 0, - 559, 625, 560, 473, 474, 683, 688, 684, 685, 687, - 707, 448, 399, 404, 488, 410, 424, 476, 530, 454, - 481, 339, 520, 490, 429, 610, 638, 988, 961, 987, - 989, 990, 986, 991, 992, 973, 852, 0, 905, 906, - 984, 983, 985, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 665, 664, 663, 662, 661, 660, - 659, 658, 0, 0, 607, 507, 355, 307, 351, 352, - 359, 724, 720, 725, 708, 711, 710, 686, 859, 315, - 587, 422, 470, 376, 652, 653, 0, 706, 951, 914, - 915, 916, 849, 917, 911, 912, 850, 913, 952, 903, - 948, 949, 878, 908, 918, 947, 919, 950, 879, 953, - 993, 994, 925, 909, 277, 995, 922, 954, 946, 945, - 920, 904, 955, 956, 886, 881, 923, 924, 910, 931, - 932, 933, 936, 851, 937, 938, 939, 940, 941, 935, - 934, 900, 901, 902, 926, 927, 929, 930, 907, 498, - 882, 883, 884, 885, 0, 0, 537, 538, 539, 562, - 0, 540, 522, 586, 386, 316, 502, 529, 722, 0, - 0, 0, 0, 0, 0, 0, 637, 648, 682, 0, - 694, 695, 697, 699, 942, 701, 495, 496, 709, 0, - 0, 4026, 704, 4027, 4028, 426, 482, 503, 489, 896, - 728, 577, 578, 729, 690, 317, 0, 844, 453, 0, - 0, 592, 626, 615, 700, 580, 0, 0, 0, 0, - 0, 0, 847, 0, 0, 0, 369, 0, 0, 421, - 630, 611, 622, 612, 597, 598, 599, 606, 381, 600, - 601, 602, 572, 603, 573, 604, 605, 887, 629, 579, - 491, 437, 0, 646, 0, 0, 967, 975, 0, 0, - 0, 0, 0, 0, 0, 0, 963, 0, 0, 0, - 0, 839, 0, 0, 876, 944, 943, 863, 873, 0, - 0, 337, 246, 574, 696, 576, 575, 3058, 0, 3059, - 869, 872, 868, 866, 867, 0, 958, 0, 0, 0, - 0, 0, 0, 831, 843, 0, 848, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 840, 841, 0, 0, 0, 0, 897, 0, - 842, 0, 0, 0, 0, 0, 492, 521, 0, 534, - 0, 406, 407, 892, 870, 874, 0, 0, 0, 0, - 324, 499, 518, 338, 486, 532, 343, 494, 511, 333, - 452, 483, 0, 0, 326, 516, 493, 434, 325, 0, - 477, 366, 383, 363, 450, 871, 0, 895, 899, 362, - 981, 893, 526, 328, 0, 525, 449, 512, 517, 435, - 428, 0, 327, 514, 433, 427, 412, 373, 982, 413, - 414, 387, 464, 425, 465, 388, 439, 438, 440, 389, - 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, - 0, 0, 0, 0, 556, 557, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 689, 890, 0, 693, 0, 528, 0, 0, 965, - 0, 0, 0, 497, 0, 0, 415, 0, 0, 0, - 894, 0, 480, 455, 978, 0, 0, 478, 423, 513, - 466, 519, 500, 527, 472, 467, 318, 501, 365, 436, - 334, 336, 721, 367, 370, 374, 375, 445, 446, 460, - 485, 504, 505, 506, 364, 348, 479, 349, 384, 350, - 319, 356, 354, 357, 487, 358, 321, 461, 510, 0, - 380, 475, 431, 322, 430, 462, 509, 508, 335, 536, - 543, 544, 634, 0, 549, 732, 733, 734, 558, 0, - 468, 331, 330, 0, 0, 0, 360, 463, 344, 346, - 347, 345, 458, 459, 563, 564, 565, 567, 0, 568, - 569, 0, 0, 0, 0, 570, 635, 651, 619, 588, - 551, 643, 585, 589, 590, 401, 402, 403, 654, 0, - 0, 0, 542, 416, 417, 0, 372, 371, 432, 323, - 0, 0, 409, 400, 469, 329, 368, 411, 405, 418, - 419, 420, 378, 313, 314, 727, 962, 451, 656, 691, - 692, 581, 0, 977, 957, 959, 960, 964, 968, 969, - 970, 971, 972, 974, 976, 980, 726, 0, 636, 650, - 730, 649, 723, 457, 0, 484, 647, 594, 0, 640, - 613, 614, 0, 641, 609, 645, 0, 583, 0, 552, - 555, 584, 669, 670, 671, 320, 554, 673, 674, 675, - 676, 677, 678, 679, 672, 979, 617, 593, 620, 533, - 596, 595, 0, 0, 631, 898, 632, 633, 441, 442, - 443, 444, 966, 657, 342, 553, 471, 0, 618, 0, - 0, 0, 0, 0, 0, 0, 0, 623, 624, 621, - 735, 0, 680, 681, 0, 0, 547, 548, 377, 0, - 566, 385, 341, 456, 379, 531, 408, 0, 559, 625, - 560, 473, 474, 683, 688, 684, 685, 687, 707, 448, - 399, 404, 488, 410, 424, 476, 530, 454, 481, 339, - 520, 490, 429, 610, 638, 988, 961, 987, 989, 990, - 986, 991, 992, 973, 852, 0, 905, 906, 984, 983, - 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 665, 664, 663, 662, 661, 660, 659, 658, - 0, 0, 607, 507, 355, 307, 351, 352, 359, 724, - 720, 725, 708, 711, 710, 686, 859, 315, 587, 422, - 470, 376, 652, 653, 0, 706, 951, 914, 915, 916, - 849, 917, 911, 912, 850, 913, 952, 903, 948, 949, - 878, 908, 918, 947, 919, 950, 879, 953, 993, 994, - 925, 909, 277, 995, 922, 954, 946, 945, 920, 904, - 955, 956, 886, 881, 923, 924, 910, 931, 932, 933, - 936, 851, 937, 938, 939, 940, 941, 935, 934, 900, - 901, 902, 926, 927, 929, 930, 907, 498, 882, 883, - 884, 885, 0, 0, 537, 538, 539, 562, 0, 540, - 522, 586, 386, 316, 502, 529, 722, 0, 0, 0, - 0, 0, 0, 0, 637, 648, 682, 0, 694, 695, - 697, 699, 942, 701, 495, 496, 709, 0, 0, 928, - 704, 705, 702, 426, 482, 503, 489, 896, 728, 577, - 578, 729, 690, 317, 0, 844, 453, 0, 0, 592, - 626, 615, 700, 580, 0, 0, 1900, 0, 0, 0, - 847, 0, 0, 0, 369, 0, 0, 421, 630, 611, - 622, 612, 597, 598, 599, 606, 381, 600, 601, 602, - 572, 603, 573, 604, 605, 887, 629, 579, 491, 437, - 0, 646, 0, 0, 967, 975, 0, 0, 0, 0, - 0, 0, 0, 0, 963, 0, 0, 0, 0, 839, - 0, 0, 876, 944, 943, 863, 873, 0, 0, 337, - 246, 574, 696, 576, 575, 864, 0, 865, 869, 872, - 868, 866, 867, 0, 958, 0, 0, 0, 0, 0, - 0, 0, 843, 0, 848, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 840, 841, 0, 0, 0, 0, 897, 0, 842, 0, - 0, 0, 0, 0, 492, 521, 0, 534, 0, 406, - 407, 892, 870, 874, 0, 0, 0, 0, 324, 499, - 518, 338, 486, 532, 343, 494, 511, 333, 452, 483, - 0, 0, 326, 516, 493, 434, 325, 0, 477, 366, - 383, 363, 450, 871, 0, 895, 899, 362, 981, 893, - 526, 328, 0, 525, 449, 512, 517, 435, 428, 0, - 327, 514, 433, 427, 412, 373, 982, 413, 414, 387, - 464, 425, 465, 388, 439, 438, 440, 389, 390, 391, + 561, 562, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 694, 896, 0, + 698, 0, 533, 0, 0, 971, 0, 0, 0, 502, + 0, 0, 420, 0, 0, 0, 900, 0, 485, 460, + 984, 0, 0, 483, 428, 518, 471, 524, 505, 532, + 477, 472, 318, 506, 365, 441, 334, 336, 726, 367, + 370, 374, 375, 450, 451, 465, 490, 509, 510, 511, + 364, 348, 484, 349, 384, 350, 319, 356, 354, 357, + 492, 358, 321, 466, 515, 0, 380, 480, 436, 322, + 435, 467, 514, 513, 335, 541, 548, 549, 639, 0, + 554, 737, 738, 739, 563, 0, 473, 331, 330, 0, + 0, 0, 360, 468, 344, 346, 347, 345, 463, 464, + 568, 569, 570, 572, 0, 573, 574, 0, 0, 0, + 0, 575, 640, 656, 624, 593, 556, 648, 590, 594, + 595, 401, 402, 403, 865, 659, 0, 0, 0, 547, + 421, 422, 0, 372, 371, 437, 323, 0, 0, 410, + 400, 474, 329, 368, 412, 406, 423, 424, 425, 378, + 313, 314, 732, 968, 456, 661, 696, 697, 586, 0, + 983, 963, 965, 966, 970, 974, 975, 976, 977, 978, + 980, 982, 986, 731, 0, 641, 655, 735, 654, 728, + 462, 0, 489, 652, 599, 0, 645, 618, 619, 0, + 646, 614, 650, 0, 588, 0, 557, 560, 589, 674, + 675, 676, 320, 559, 678, 679, 680, 681, 682, 683, + 684, 677, 985, 622, 598, 625, 538, 601, 600, 0, + 0, 636, 904, 637, 638, 446, 447, 448, 449, 972, + 662, 342, 558, 476, 0, 623, 0, 0, 0, 0, + 0, 0, 0, 0, 628, 629, 626, 740, 0, 685, + 686, 0, 0, 552, 553, 377, 0, 571, 385, 341, + 461, 379, 536, 409, 0, 564, 630, 565, 478, 479, + 688, 693, 689, 690, 692, 712, 453, 399, 405, 493, + 411, 429, 481, 535, 459, 486, 339, 525, 495, 434, + 615, 643, 994, 967, 993, 995, 996, 992, 997, 998, + 979, 857, 0, 911, 912, 990, 989, 991, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 670, + 669, 668, 667, 666, 665, 664, 663, 0, 0, 612, + 512, 355, 307, 351, 352, 359, 729, 725, 730, 713, + 716, 715, 691, 864, 315, 592, 427, 475, 376, 657, + 658, 0, 711, 957, 920, 921, 922, 854, 923, 917, + 918, 855, 919, 958, 909, 954, 955, 884, 914, 924, + 953, 925, 956, 885, 959, 999, 1000, 931, 915, 277, + 1001, 928, 960, 952, 951, 926, 910, 961, 962, 892, + 887, 929, 930, 916, 937, 938, 939, 942, 856, 943, + 944, 945, 946, 947, 941, 940, 906, 907, 908, 932, + 933, 935, 936, 913, 503, 888, 889, 890, 891, 0, + 0, 542, 543, 544, 567, 0, 545, 527, 591, 386, + 316, 507, 534, 727, 0, 0, 0, 0, 0, 0, + 0, 642, 653, 687, 0, 699, 700, 702, 704, 948, + 706, 500, 501, 714, 0, 0, 934, 709, 710, 707, + 431, 487, 508, 494, 902, 733, 582, 583, 734, 695, + 317, 0, 849, 458, 0, 0, 597, 631, 620, 705, + 585, 0, 0, 0, 0, 0, 0, 852, 0, 0, + 0, 369, 0, 0, 426, 635, 616, 627, 617, 602, + 603, 604, 611, 381, 605, 606, 607, 577, 608, 578, + 609, 610, 893, 634, 584, 496, 442, 0, 651, 0, + 0, 973, 981, 0, 0, 0, 0, 0, 0, 0, + 0, 969, 0, 0, 0, 0, 844, 0, 0, 882, + 950, 949, 869, 879, 0, 0, 337, 246, 579, 701, + 581, 580, 870, 0, 871, 875, 878, 874, 872, 873, + 0, 964, 0, 0, 0, 0, 0, 0, 836, 848, + 0, 853, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 845, 846, 0, + 0, 0, 0, 903, 0, 847, 0, 0, 0, 0, + 0, 497, 526, 0, 539, 0, 407, 408, 898, 876, + 880, 0, 0, 0, 0, 324, 504, 523, 338, 491, + 537, 343, 499, 516, 333, 457, 488, 0, 0, 326, + 521, 498, 439, 325, 0, 482, 366, 383, 363, 455, + 877, 0, 901, 905, 362, 987, 899, 531, 328, 0, + 530, 454, 517, 522, 440, 433, 0, 327, 519, 438, + 432, 413, 373, 988, 414, 415, 416, 417, 418, 419, + 387, 469, 430, 470, 388, 444, 443, 445, 389, 390, + 391, 392, 393, 394, 395, 396, 397, 398, 0, 0, + 0, 0, 0, 561, 562, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 694, 896, 0, 698, 0, 533, 0, 0, 971, 0, + 0, 0, 502, 0, 0, 420, 0, 0, 0, 900, + 0, 485, 460, 984, 4629, 0, 483, 428, 518, 471, + 524, 505, 532, 477, 472, 318, 506, 365, 441, 334, + 336, 726, 367, 370, 374, 375, 450, 451, 465, 490, + 509, 510, 511, 364, 348, 484, 349, 384, 350, 319, + 356, 354, 357, 492, 358, 321, 466, 515, 0, 380, + 480, 436, 322, 435, 467, 514, 513, 335, 541, 548, + 549, 639, 0, 554, 737, 738, 739, 563, 0, 473, + 331, 330, 0, 0, 0, 360, 468, 344, 346, 347, + 345, 463, 464, 568, 569, 570, 572, 0, 573, 574, + 0, 0, 0, 0, 575, 640, 656, 624, 593, 556, + 648, 590, 594, 595, 401, 402, 403, 865, 659, 0, + 0, 0, 547, 421, 422, 0, 372, 371, 437, 323, + 0, 0, 410, 400, 474, 329, 368, 412, 406, 423, + 424, 425, 378, 313, 314, 732, 968, 456, 661, 696, + 697, 586, 0, 983, 963, 965, 966, 970, 974, 975, + 976, 977, 978, 980, 982, 986, 731, 0, 641, 655, + 735, 654, 728, 462, 0, 489, 652, 599, 0, 645, + 618, 619, 0, 646, 614, 650, 0, 588, 0, 557, + 560, 589, 674, 675, 676, 320, 559, 678, 679, 680, + 681, 682, 683, 684, 677, 985, 622, 598, 625, 538, + 601, 600, 0, 0, 636, 904, 637, 638, 446, 447, + 448, 449, 972, 662, 342, 558, 476, 0, 623, 0, + 0, 0, 0, 0, 0, 0, 0, 628, 629, 626, + 740, 0, 685, 686, 0, 0, 552, 553, 377, 0, + 571, 385, 341, 461, 379, 536, 409, 0, 564, 630, + 565, 478, 479, 688, 693, 689, 690, 692, 712, 453, + 399, 405, 493, 411, 429, 481, 535, 459, 486, 339, + 525, 495, 434, 615, 643, 994, 967, 993, 995, 996, + 992, 997, 998, 979, 857, 0, 911, 912, 990, 989, + 991, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 670, 669, 668, 667, 666, 665, 664, 663, + 0, 0, 612, 512, 355, 307, 351, 352, 359, 729, + 725, 730, 713, 716, 715, 691, 864, 315, 592, 427, + 475, 376, 657, 658, 0, 711, 957, 920, 921, 922, + 854, 923, 917, 918, 855, 919, 958, 909, 954, 955, + 884, 914, 924, 953, 925, 956, 885, 959, 999, 1000, + 931, 915, 277, 1001, 928, 960, 952, 951, 926, 910, + 961, 962, 892, 887, 929, 930, 916, 937, 938, 939, + 942, 856, 943, 944, 945, 946, 947, 941, 940, 906, + 907, 908, 932, 933, 935, 936, 913, 503, 888, 889, + 890, 891, 0, 0, 542, 543, 544, 567, 0, 545, + 527, 591, 386, 316, 507, 534, 727, 0, 0, 0, + 0, 0, 0, 0, 642, 653, 687, 0, 699, 700, + 702, 704, 948, 706, 500, 501, 714, 0, 0, 934, + 709, 710, 707, 431, 487, 508, 494, 902, 733, 582, + 583, 734, 695, 317, 0, 849, 458, 0, 0, 597, + 631, 620, 705, 585, 0, 0, 0, 0, 0, 0, + 852, 0, 0, 0, 369, 2082, 0, 426, 635, 616, + 627, 617, 602, 603, 604, 611, 381, 605, 606, 607, + 577, 608, 578, 609, 610, 893, 634, 584, 496, 442, + 0, 651, 0, 0, 973, 981, 0, 0, 0, 0, + 0, 0, 0, 0, 969, 0, 0, 0, 0, 844, + 0, 0, 882, 950, 949, 869, 879, 0, 0, 337, + 246, 579, 701, 581, 580, 870, 0, 871, 875, 878, + 874, 872, 873, 0, 964, 0, 0, 0, 0, 0, + 0, 836, 848, 0, 853, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 845, 846, 0, 0, 0, 0, 903, 0, 847, 0, + 0, 0, 0, 0, 497, 526, 0, 539, 0, 407, + 408, 898, 876, 880, 0, 0, 0, 0, 324, 504, + 523, 338, 491, 537, 343, 499, 516, 333, 457, 488, + 0, 0, 326, 521, 498, 439, 325, 0, 482, 366, + 383, 363, 455, 877, 0, 901, 905, 362, 987, 899, + 531, 328, 0, 530, 454, 517, 522, 440, 433, 0, + 327, 519, 438, 432, 413, 373, 988, 414, 415, 416, + 417, 418, 419, 387, 469, 430, 470, 388, 444, 443, + 445, 389, 390, 391, 392, 393, 394, 395, 396, 397, + 398, 0, 0, 0, 0, 0, 561, 562, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 694, 896, 0, 698, 0, 533, 0, + 0, 971, 0, 0, 0, 502, 0, 0, 420, 0, + 0, 0, 900, 0, 485, 460, 984, 0, 0, 483, + 428, 518, 471, 524, 505, 532, 477, 472, 318, 506, + 365, 441, 334, 336, 726, 367, 370, 374, 375, 450, + 451, 465, 490, 509, 510, 511, 364, 348, 484, 349, + 384, 350, 319, 356, 354, 357, 492, 358, 321, 466, + 515, 0, 380, 480, 436, 322, 435, 467, 514, 513, + 335, 541, 548, 549, 639, 0, 554, 737, 738, 739, + 563, 0, 473, 331, 330, 0, 0, 0, 360, 468, + 344, 346, 347, 345, 463, 464, 568, 569, 570, 572, + 0, 573, 574, 0, 0, 0, 0, 575, 640, 656, + 624, 593, 556, 648, 590, 594, 595, 401, 402, 403, + 865, 659, 0, 0, 0, 547, 421, 422, 0, 372, + 371, 437, 323, 0, 0, 410, 400, 474, 329, 368, + 412, 406, 423, 424, 425, 378, 313, 314, 732, 968, + 456, 661, 696, 697, 586, 0, 983, 963, 965, 966, + 970, 974, 975, 976, 977, 978, 980, 982, 986, 731, + 0, 641, 655, 735, 654, 728, 462, 0, 489, 652, + 599, 0, 645, 618, 619, 0, 646, 614, 650, 0, + 588, 0, 557, 560, 589, 674, 675, 676, 320, 559, + 678, 679, 680, 681, 682, 683, 684, 677, 985, 622, + 598, 625, 538, 601, 600, 0, 0, 636, 904, 637, + 638, 446, 447, 448, 449, 972, 662, 342, 558, 476, + 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, + 628, 629, 626, 740, 0, 685, 686, 0, 0, 552, + 553, 377, 0, 571, 385, 341, 461, 379, 536, 409, + 0, 564, 630, 565, 478, 479, 688, 693, 689, 690, + 692, 712, 453, 399, 405, 493, 411, 429, 481, 535, + 459, 486, 339, 525, 495, 434, 615, 643, 994, 967, + 993, 995, 996, 992, 997, 998, 979, 857, 0, 911, + 912, 990, 989, 991, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 670, 669, 668, 667, 666, + 665, 664, 663, 0, 0, 612, 512, 355, 307, 351, + 352, 359, 729, 725, 730, 713, 716, 715, 691, 864, + 315, 592, 427, 475, 376, 657, 658, 0, 711, 957, + 920, 921, 922, 854, 923, 917, 918, 855, 919, 958, + 909, 954, 955, 884, 914, 924, 953, 925, 956, 885, + 959, 999, 1000, 931, 915, 277, 1001, 928, 960, 952, + 951, 926, 910, 961, 962, 892, 887, 929, 930, 916, + 937, 938, 939, 942, 856, 943, 944, 945, 946, 947, + 941, 940, 906, 907, 908, 932, 933, 935, 936, 913, + 503, 888, 889, 890, 891, 0, 0, 542, 543, 544, + 567, 0, 545, 527, 591, 386, 316, 507, 534, 727, + 0, 0, 0, 0, 0, 0, 0, 642, 653, 687, + 0, 699, 700, 702, 704, 948, 706, 500, 501, 714, + 0, 0, 934, 709, 710, 707, 431, 487, 508, 494, + 902, 733, 582, 583, 734, 695, 317, 0, 849, 458, + 0, 0, 597, 631, 620, 705, 585, 0, 0, 0, + 0, 0, 0, 852, 0, 0, 0, 369, 0, 0, + 426, 635, 616, 627, 617, 602, 603, 604, 611, 381, + 605, 606, 607, 577, 608, 578, 609, 610, 893, 634, + 584, 496, 442, 0, 651, 0, 0, 973, 981, 0, + 0, 0, 0, 0, 0, 0, 0, 969, 0, 0, + 0, 0, 844, 0, 0, 882, 950, 949, 869, 879, + 0, 0, 337, 246, 579, 701, 581, 580, 870, 0, + 871, 875, 878, 874, 872, 873, 0, 964, 0, 0, + 0, 0, 0, 0, 836, 848, 0, 853, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 845, 846, 1772, 0, 0, 0, 903, + 0, 847, 0, 0, 0, 0, 0, 497, 526, 0, + 539, 0, 407, 408, 898, 876, 880, 0, 0, 0, + 0, 324, 504, 523, 338, 491, 537, 343, 499, 516, + 333, 457, 488, 0, 0, 326, 521, 498, 439, 325, + 0, 482, 366, 383, 363, 455, 877, 0, 901, 905, + 362, 987, 899, 531, 328, 0, 530, 454, 517, 522, + 440, 433, 0, 327, 519, 438, 432, 413, 373, 988, + 414, 415, 416, 417, 418, 419, 387, 469, 430, 470, + 388, 444, 443, 445, 389, 390, 391, 392, 393, 394, + 395, 396, 397, 398, 0, 0, 0, 0, 0, 561, + 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 694, 896, 0, 698, + 0, 533, 0, 0, 971, 0, 0, 0, 502, 0, + 0, 420, 0, 0, 0, 900, 0, 485, 460, 984, + 0, 0, 483, 428, 518, 471, 524, 505, 532, 477, + 472, 318, 506, 365, 441, 334, 336, 726, 367, 370, + 374, 375, 450, 451, 465, 490, 509, 510, 511, 364, + 348, 484, 349, 384, 350, 319, 356, 354, 357, 492, + 358, 321, 466, 515, 0, 380, 480, 436, 322, 435, + 467, 514, 513, 335, 541, 548, 549, 639, 0, 554, + 737, 738, 739, 563, 0, 473, 331, 330, 0, 0, + 0, 360, 468, 344, 346, 347, 345, 463, 464, 568, + 569, 570, 572, 0, 573, 574, 0, 0, 0, 0, + 575, 640, 656, 624, 593, 556, 648, 590, 594, 595, + 401, 402, 403, 865, 659, 0, 0, 0, 547, 421, + 422, 0, 372, 371, 437, 323, 0, 0, 410, 400, + 474, 329, 368, 412, 406, 423, 424, 425, 378, 313, + 314, 732, 968, 456, 661, 696, 697, 586, 0, 983, + 963, 965, 966, 970, 974, 975, 976, 977, 978, 980, + 982, 986, 731, 0, 641, 655, 735, 654, 728, 462, + 0, 489, 652, 599, 0, 645, 618, 619, 0, 646, + 614, 650, 0, 588, 0, 557, 560, 589, 674, 675, + 676, 320, 559, 678, 679, 680, 681, 682, 683, 684, + 677, 985, 622, 598, 625, 538, 601, 600, 0, 0, + 636, 904, 637, 638, 446, 447, 448, 449, 972, 662, + 342, 558, 476, 0, 623, 0, 0, 0, 0, 0, + 0, 0, 0, 628, 629, 626, 740, 0, 685, 686, + 0, 0, 552, 553, 377, 0, 571, 385, 341, 461, + 379, 536, 409, 0, 564, 630, 565, 478, 479, 688, + 693, 689, 690, 692, 712, 453, 399, 405, 493, 411, + 429, 481, 535, 459, 486, 339, 525, 495, 434, 615, + 643, 994, 967, 993, 995, 996, 992, 997, 998, 979, + 857, 0, 911, 912, 990, 989, 991, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 670, 669, + 668, 667, 666, 665, 664, 663, 0, 0, 612, 512, + 355, 307, 351, 352, 359, 729, 725, 730, 713, 716, + 715, 691, 864, 315, 592, 427, 475, 376, 657, 658, + 0, 711, 957, 920, 921, 922, 854, 923, 917, 918, + 855, 919, 958, 909, 954, 955, 884, 914, 924, 953, + 925, 956, 885, 959, 999, 1000, 931, 915, 277, 1001, + 928, 960, 952, 951, 926, 910, 961, 962, 892, 887, + 929, 930, 916, 937, 938, 939, 942, 856, 943, 944, + 945, 946, 947, 941, 940, 906, 907, 908, 932, 933, + 935, 936, 913, 503, 888, 889, 890, 891, 0, 0, + 542, 543, 544, 567, 0, 545, 527, 591, 386, 316, + 507, 534, 727, 0, 0, 0, 0, 0, 0, 0, + 642, 653, 687, 0, 699, 700, 702, 704, 948, 706, + 500, 501, 714, 0, 0, 934, 709, 710, 707, 431, + 487, 508, 494, 0, 733, 582, 583, 734, 695, 317, + 902, 849, 0, 2514, 0, 0, 0, 0, 0, 458, + 0, 0, 597, 631, 620, 705, 585, 0, 0, 0, + 0, 0, 0, 852, 0, 0, 0, 369, 0, 0, + 426, 635, 616, 627, 617, 602, 603, 604, 611, 381, + 605, 606, 607, 577, 608, 578, 609, 610, 893, 634, + 584, 496, 442, 0, 651, 0, 0, 973, 981, 0, + 0, 0, 0, 0, 0, 0, 0, 969, 0, 0, + 0, 0, 844, 0, 0, 882, 950, 949, 869, 879, + 0, 0, 337, 246, 579, 701, 581, 580, 870, 0, + 871, 875, 878, 874, 872, 873, 0, 964, 0, 0, + 0, 0, 0, 0, 836, 848, 0, 853, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 845, 846, 0, 0, 0, 0, 903, + 0, 847, 0, 0, 0, 0, 0, 497, 526, 0, + 539, 0, 407, 408, 898, 876, 880, 0, 0, 0, + 0, 324, 504, 523, 338, 491, 537, 343, 499, 516, + 333, 457, 488, 0, 0, 326, 521, 498, 439, 325, + 0, 482, 366, 383, 363, 455, 877, 0, 901, 905, + 362, 987, 899, 531, 328, 0, 530, 454, 517, 522, + 440, 433, 0, 327, 519, 438, 432, 413, 373, 988, + 414, 415, 416, 417, 418, 419, 387, 469, 430, 470, + 388, 444, 443, 445, 389, 390, 391, 392, 393, 394, + 395, 396, 397, 398, 0, 0, 0, 0, 0, 561, + 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 694, 896, 0, 698, + 0, 533, 0, 0, 971, 0, 0, 0, 502, 0, + 0, 420, 0, 0, 0, 900, 0, 485, 460, 984, + 0, 0, 483, 428, 518, 471, 524, 505, 532, 477, + 472, 318, 506, 365, 441, 334, 336, 726, 367, 370, + 374, 375, 450, 451, 465, 490, 509, 510, 511, 364, + 348, 484, 349, 384, 350, 319, 356, 354, 357, 492, + 358, 321, 466, 515, 0, 380, 480, 436, 322, 435, + 467, 514, 513, 335, 541, 548, 549, 639, 0, 554, + 737, 738, 739, 563, 0, 473, 331, 330, 0, 0, + 0, 360, 468, 344, 346, 347, 345, 463, 464, 568, + 569, 570, 572, 0, 573, 574, 0, 0, 0, 0, + 575, 640, 656, 624, 593, 556, 648, 590, 594, 595, + 401, 402, 403, 865, 659, 0, 0, 0, 547, 421, + 422, 0, 372, 371, 437, 323, 0, 0, 410, 400, + 474, 329, 368, 412, 406, 423, 424, 425, 378, 313, + 314, 732, 968, 456, 661, 696, 697, 586, 0, 983, + 963, 965, 966, 970, 974, 975, 976, 977, 978, 980, + 982, 986, 731, 0, 641, 655, 735, 654, 728, 462, + 0, 489, 652, 599, 0, 645, 618, 619, 0, 646, + 614, 650, 0, 588, 0, 557, 560, 589, 674, 675, + 676, 320, 559, 678, 679, 680, 681, 682, 683, 684, + 677, 985, 622, 598, 625, 538, 601, 600, 0, 0, + 636, 904, 637, 638, 446, 447, 448, 449, 972, 662, + 342, 558, 476, 0, 623, 0, 0, 0, 0, 0, + 0, 0, 0, 628, 629, 626, 740, 0, 685, 686, + 0, 0, 552, 553, 377, 0, 571, 385, 341, 461, + 379, 536, 409, 0, 564, 630, 565, 478, 479, 688, + 693, 689, 690, 692, 712, 453, 399, 405, 493, 411, + 429, 481, 535, 459, 486, 339, 525, 495, 434, 615, + 643, 994, 967, 993, 995, 996, 992, 997, 998, 979, + 857, 0, 911, 912, 990, 989, 991, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 670, 669, + 668, 667, 666, 665, 664, 663, 0, 0, 612, 512, + 355, 307, 351, 352, 359, 729, 725, 730, 713, 716, + 715, 691, 864, 315, 592, 427, 475, 376, 657, 658, + 0, 711, 957, 920, 921, 922, 854, 923, 917, 918, + 855, 919, 958, 909, 954, 955, 884, 914, 924, 953, + 925, 956, 885, 959, 999, 1000, 931, 915, 277, 1001, + 928, 960, 952, 951, 926, 910, 961, 962, 892, 887, + 929, 930, 916, 937, 938, 939, 942, 856, 943, 944, + 945, 946, 947, 941, 940, 906, 907, 908, 932, 933, + 935, 936, 913, 503, 888, 889, 890, 891, 0, 0, + 542, 543, 544, 567, 0, 545, 527, 591, 386, 316, + 507, 534, 727, 0, 0, 0, 0, 0, 0, 0, + 642, 653, 687, 0, 699, 700, 702, 704, 948, 706, + 500, 501, 714, 0, 0, 934, 709, 710, 707, 431, + 487, 508, 494, 902, 733, 582, 583, 734, 695, 317, + 0, 849, 458, 0, 0, 597, 631, 620, 705, 585, + 0, 0, 0, 0, 0, 0, 852, 0, 0, 0, + 369, 0, 0, 426, 635, 616, 627, 617, 602, 603, + 604, 611, 381, 605, 606, 607, 577, 608, 578, 609, + 610, 893, 634, 584, 496, 442, 0, 651, 0, 0, + 973, 981, 0, 0, 0, 0, 0, 0, 0, 0, + 969, 0, 0, 0, 0, 844, 0, 0, 882, 950, + 949, 869, 879, 0, 0, 337, 246, 579, 701, 581, + 580, 870, 0, 871, 875, 878, 874, 872, 873, 0, + 964, 0, 0, 0, 0, 0, 0, 836, 848, 0, + 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 845, 846, 2075, 0, + 0, 0, 903, 0, 847, 0, 0, 0, 0, 0, + 497, 526, 0, 539, 0, 407, 408, 898, 876, 880, + 0, 0, 0, 0, 324, 504, 523, 338, 491, 537, + 343, 499, 516, 333, 457, 488, 0, 0, 326, 521, + 498, 439, 325, 0, 482, 366, 383, 363, 455, 877, + 0, 901, 905, 362, 987, 899, 531, 328, 0, 530, + 454, 517, 522, 440, 433, 0, 327, 519, 438, 432, + 413, 373, 988, 414, 415, 416, 417, 418, 419, 387, + 469, 430, 470, 388, 444, 443, 445, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, - 0, 0, 556, 557, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, - 890, 0, 693, 0, 528, 0, 0, 965, 0, 0, - 0, 497, 0, 0, 415, 0, 0, 0, 894, 0, - 480, 455, 978, 0, 0, 478, 423, 513, 466, 519, - 500, 527, 472, 467, 318, 501, 365, 436, 334, 336, - 721, 367, 370, 374, 375, 445, 446, 460, 485, 504, - 505, 506, 364, 348, 479, 349, 384, 350, 319, 356, - 354, 357, 487, 358, 321, 461, 510, 0, 380, 475, - 431, 322, 430, 462, 509, 508, 335, 536, 1901, 1902, - 634, 0, 549, 732, 733, 734, 558, 0, 468, 331, - 330, 0, 0, 0, 360, 463, 344, 346, 347, 345, - 458, 459, 563, 564, 565, 567, 0, 568, 569, 0, - 0, 0, 0, 570, 635, 651, 619, 588, 551, 643, - 585, 589, 590, 401, 402, 403, 654, 0, 0, 0, - 542, 416, 417, 0, 372, 371, 432, 323, 0, 0, - 409, 400, 469, 329, 368, 411, 405, 418, 419, 420, - 378, 313, 314, 727, 962, 451, 656, 691, 692, 581, - 0, 977, 957, 959, 960, 964, 968, 969, 970, 971, - 972, 974, 976, 980, 726, 0, 636, 650, 730, 649, - 723, 457, 0, 484, 647, 594, 0, 640, 613, 614, - 0, 641, 609, 645, 0, 583, 0, 552, 555, 584, - 669, 670, 671, 320, 554, 673, 674, 675, 676, 677, - 678, 679, 672, 979, 617, 593, 620, 533, 596, 595, - 0, 0, 631, 898, 632, 633, 441, 442, 443, 444, - 966, 657, 342, 553, 471, 0, 618, 0, 0, 0, - 0, 0, 0, 0, 0, 623, 624, 621, 735, 0, - 680, 681, 0, 0, 547, 548, 377, 0, 566, 385, - 341, 456, 379, 531, 408, 0, 559, 625, 560, 473, - 474, 683, 688, 684, 685, 687, 707, 448, 399, 404, - 488, 410, 424, 476, 530, 454, 481, 339, 520, 490, - 429, 610, 638, 988, 961, 987, 989, 990, 986, 991, - 992, 973, 852, 0, 905, 906, 984, 983, 985, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 665, 664, 663, 662, 661, 660, 659, 658, 0, 0, - 607, 507, 355, 307, 351, 352, 359, 724, 720, 725, - 708, 711, 710, 686, 859, 315, 587, 422, 470, 376, - 652, 653, 0, 706, 951, 914, 915, 916, 849, 917, - 911, 912, 850, 913, 952, 903, 948, 949, 878, 908, - 918, 947, 919, 950, 879, 953, 993, 994, 925, 909, - 277, 995, 922, 954, 946, 945, 920, 904, 955, 956, - 886, 881, 923, 924, 910, 931, 932, 933, 936, 851, - 937, 938, 939, 940, 941, 935, 934, 900, 901, 902, - 926, 927, 929, 930, 907, 498, 882, 883, 884, 885, - 0, 0, 537, 538, 539, 562, 0, 540, 522, 586, - 386, 316, 502, 529, 722, 0, 0, 0, 0, 0, - 0, 0, 637, 648, 682, 0, 694, 695, 697, 699, - 942, 701, 495, 496, 709, 0, 0, 928, 704, 705, - 702, 426, 482, 503, 489, 896, 728, 577, 578, 729, - 690, 317, 0, 844, 453, 0, 0, 592, 626, 615, - 700, 580, 0, 0, 0, 0, 0, 0, 847, 0, - 0, 0, 369, 0, 0, 421, 630, 611, 622, 612, - 597, 598, 599, 606, 381, 600, 601, 602, 572, 603, - 573, 604, 605, 887, 629, 579, 491, 437, 0, 646, - 0, 0, 967, 975, 0, 0, 0, 0, 0, 0, - 0, 0, 963, 0, 0, 0, 0, 1440, 0, 0, - 876, 944, 943, 863, 873, 0, 0, 337, 246, 574, - 696, 576, 575, 864, 0, 865, 869, 872, 868, 866, - 867, 0, 958, 0, 0, 0, 0, 0, 0, 831, - 843, 0, 848, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 840, 841, - 0, 0, 0, 0, 897, 0, 842, 0, 0, 0, - 0, 0, 492, 521, 0, 534, 0, 406, 407, 892, - 870, 874, 0, 0, 0, 0, 324, 499, 518, 338, - 486, 532, 343, 494, 511, 333, 452, 483, 0, 0, - 326, 516, 493, 434, 325, 0, 477, 366, 383, 363, - 450, 871, 0, 895, 899, 362, 981, 893, 526, 328, - 0, 525, 449, 512, 517, 435, 428, 0, 327, 514, - 433, 427, 412, 373, 982, 413, 414, 387, 464, 425, - 465, 388, 439, 438, 440, 389, 390, 391, 392, 393, + 0, 0, 561, 562, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, + 896, 0, 698, 0, 533, 0, 0, 971, 0, 0, + 0, 502, 0, 0, 420, 0, 0, 0, 900, 0, + 485, 460, 984, 0, 0, 483, 428, 518, 471, 524, + 505, 532, 477, 472, 318, 506, 365, 441, 334, 336, + 726, 367, 370, 374, 375, 450, 451, 465, 490, 509, + 510, 511, 364, 348, 484, 349, 384, 350, 319, 356, + 354, 357, 492, 358, 321, 466, 515, 0, 380, 480, + 436, 322, 435, 467, 514, 513, 335, 541, 548, 549, + 639, 0, 554, 737, 738, 739, 563, 0, 473, 331, + 330, 0, 0, 0, 360, 468, 344, 346, 347, 345, + 463, 464, 568, 569, 570, 572, 0, 573, 574, 0, + 0, 0, 0, 575, 640, 656, 624, 593, 556, 648, + 590, 594, 595, 401, 402, 403, 865, 659, 0, 0, + 0, 547, 421, 422, 0, 372, 371, 437, 323, 0, + 0, 410, 400, 474, 329, 368, 412, 406, 423, 424, + 425, 378, 313, 314, 732, 968, 456, 661, 696, 697, + 586, 0, 983, 963, 965, 966, 970, 974, 975, 976, + 977, 978, 980, 982, 986, 731, 0, 641, 655, 735, + 654, 728, 462, 0, 489, 652, 599, 0, 645, 618, + 619, 0, 646, 614, 650, 0, 588, 0, 557, 560, + 589, 674, 675, 676, 320, 559, 678, 679, 680, 681, + 682, 683, 684, 677, 985, 622, 598, 625, 538, 601, + 600, 0, 0, 636, 904, 637, 638, 446, 447, 448, + 449, 972, 662, 342, 558, 476, 0, 623, 0, 0, + 0, 0, 0, 0, 0, 0, 628, 629, 626, 740, + 0, 685, 686, 0, 0, 552, 553, 377, 0, 571, + 385, 341, 461, 379, 536, 409, 0, 564, 630, 565, + 478, 479, 688, 693, 689, 690, 692, 712, 453, 399, + 405, 493, 411, 429, 481, 535, 459, 486, 339, 525, + 495, 434, 615, 643, 994, 967, 993, 995, 996, 992, + 997, 998, 979, 857, 0, 911, 912, 990, 989, 991, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 670, 669, 668, 667, 666, 665, 664, 663, 0, + 0, 612, 512, 355, 307, 351, 352, 359, 729, 725, + 730, 713, 716, 715, 691, 864, 315, 592, 427, 475, + 376, 657, 658, 0, 711, 957, 920, 921, 922, 854, + 923, 917, 918, 855, 919, 958, 909, 954, 955, 884, + 914, 924, 953, 925, 956, 885, 959, 999, 1000, 931, + 915, 277, 1001, 928, 960, 952, 951, 926, 910, 961, + 962, 892, 887, 929, 930, 916, 937, 938, 939, 942, + 856, 943, 944, 945, 946, 947, 941, 940, 906, 907, + 908, 932, 933, 935, 936, 913, 503, 888, 889, 890, + 891, 0, 0, 542, 543, 544, 567, 0, 545, 527, + 591, 386, 316, 507, 534, 727, 0, 0, 0, 0, + 0, 0, 0, 642, 653, 687, 0, 699, 700, 702, + 704, 948, 706, 500, 501, 714, 0, 0, 934, 709, + 710, 707, 431, 487, 508, 494, 902, 733, 582, 583, + 734, 695, 317, 0, 849, 458, 0, 0, 597, 631, + 620, 705, 585, 0, 0, 0, 0, 0, 0, 852, + 0, 0, 0, 369, 0, 0, 426, 635, 616, 627, + 617, 602, 603, 604, 611, 381, 605, 606, 607, 577, + 608, 578, 609, 610, 893, 634, 584, 496, 442, 0, + 651, 0, 0, 973, 981, 0, 0, 0, 0, 0, + 0, 0, 0, 969, 0, 0, 0, 0, 844, 0, + 0, 882, 950, 949, 869, 879, 0, 0, 337, 246, + 579, 701, 581, 580, 870, 0, 871, 875, 878, 874, + 872, 873, 0, 964, 0, 0, 0, 0, 0, 0, + 836, 848, 0, 853, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 845, + 846, 0, 0, 0, 0, 903, 0, 847, 0, 0, + 0, 0, 0, 497, 526, 0, 539, 0, 407, 408, + 898, 876, 880, 0, 0, 0, 0, 324, 504, 523, + 338, 491, 537, 343, 499, 516, 333, 457, 488, 0, + 0, 326, 521, 498, 439, 325, 0, 482, 366, 383, + 363, 455, 877, 0, 901, 905, 362, 987, 899, 531, + 328, 0, 530, 454, 517, 522, 440, 433, 0, 327, + 519, 438, 432, 413, 373, 988, 414, 415, 416, 417, + 418, 419, 387, 469, 430, 470, 388, 444, 443, 445, + 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, + 0, 0, 0, 0, 0, 561, 562, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 694, 896, 0, 698, 0, 533, 0, 0, + 971, 0, 0, 0, 502, 0, 0, 420, 0, 0, + 0, 900, 0, 485, 460, 984, 0, 0, 483, 428, + 518, 471, 524, 505, 532, 477, 472, 318, 506, 365, + 441, 334, 336, 726, 367, 370, 374, 375, 450, 451, + 465, 490, 509, 510, 511, 364, 348, 484, 349, 384, + 350, 319, 356, 354, 357, 492, 358, 321, 466, 515, + 0, 380, 480, 436, 322, 435, 467, 514, 513, 335, + 541, 548, 549, 639, 0, 554, 737, 738, 739, 563, + 0, 473, 331, 330, 0, 0, 0, 360, 468, 344, + 346, 347, 345, 463, 464, 568, 569, 570, 572, 0, + 573, 574, 0, 0, 0, 0, 575, 640, 656, 624, + 593, 556, 648, 590, 594, 595, 401, 402, 403, 865, + 659, 0, 0, 0, 547, 421, 422, 0, 372, 371, + 437, 323, 0, 0, 410, 400, 474, 329, 368, 412, + 406, 423, 424, 425, 378, 313, 314, 732, 968, 456, + 661, 696, 697, 586, 0, 983, 963, 965, 966, 970, + 974, 975, 976, 977, 978, 980, 982, 986, 731, 0, + 641, 655, 735, 654, 728, 462, 0, 489, 652, 599, + 0, 645, 618, 619, 0, 646, 614, 650, 0, 588, + 0, 557, 560, 589, 674, 675, 676, 320, 559, 678, + 679, 680, 681, 682, 683, 684, 677, 985, 622, 598, + 625, 538, 601, 600, 0, 0, 636, 904, 637, 638, + 446, 447, 448, 449, 972, 662, 342, 558, 476, 0, + 623, 0, 0, 0, 0, 0, 0, 0, 0, 628, + 629, 626, 740, 0, 685, 686, 0, 0, 552, 553, + 377, 0, 571, 385, 341, 461, 379, 536, 409, 0, + 564, 630, 565, 478, 479, 688, 693, 689, 690, 692, + 712, 453, 399, 405, 493, 411, 429, 481, 535, 459, + 486, 339, 525, 495, 434, 615, 643, 994, 967, 993, + 995, 996, 992, 997, 998, 979, 857, 0, 911, 912, + 990, 989, 991, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 670, 669, 668, 667, 666, 665, + 664, 663, 0, 0, 612, 512, 355, 307, 351, 352, + 359, 729, 725, 730, 713, 716, 715, 691, 864, 315, + 592, 427, 475, 376, 657, 658, 0, 711, 957, 920, + 921, 922, 854, 923, 917, 918, 855, 919, 958, 909, + 954, 955, 884, 914, 924, 953, 925, 956, 885, 959, + 999, 1000, 931, 915, 277, 1001, 928, 960, 952, 951, + 926, 910, 961, 962, 892, 887, 929, 930, 916, 937, + 938, 939, 942, 856, 943, 944, 945, 946, 947, 941, + 940, 906, 907, 908, 932, 933, 935, 936, 913, 503, + 888, 889, 890, 891, 0, 0, 542, 543, 544, 567, + 0, 545, 527, 591, 386, 316, 507, 534, 727, 0, + 0, 0, 0, 0, 0, 0, 642, 653, 687, 0, + 699, 700, 702, 704, 948, 706, 500, 501, 714, 0, + 0, 934, 709, 710, 707, 431, 487, 508, 494, 902, + 733, 582, 583, 734, 695, 317, 0, 849, 458, 0, + 0, 597, 631, 620, 705, 585, 0, 0, 0, 0, + 0, 0, 852, 0, 0, 0, 369, 0, 0, 426, + 635, 616, 627, 617, 602, 603, 604, 611, 381, 605, + 606, 607, 577, 608, 578, 609, 610, 893, 634, 584, + 496, 442, 0, 651, 0, 0, 973, 981, 0, 0, + 0, 0, 0, 0, 0, 0, 969, 0, 0, 0, + 0, 844, 0, 0, 882, 950, 949, 869, 879, 0, + 0, 337, 246, 579, 701, 581, 580, 870, 0, 871, + 875, 878, 874, 872, 873, 0, 964, 0, 0, 0, + 0, 0, 0, 836, 848, 0, 853, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 845, 846, 0, 0, 0, 0, 903, 0, + 847, 0, 0, 0, 0, 0, 497, 526, 0, 539, + 0, 407, 408, 898, 876, 880, 0, 0, 0, 0, + 324, 504, 523, 338, 491, 537, 343, 499, 516, 333, + 457, 488, 0, 0, 326, 521, 498, 439, 325, 0, + 482, 366, 383, 363, 455, 877, 0, 901, 905, 362, + 987, 899, 531, 328, 0, 530, 454, 517, 522, 440, + 433, 0, 327, 519, 438, 432, 413, 373, 988, 414, + 415, 416, 417, 418, 419, 387, 469, 430, 470, 388, + 444, 443, 445, 389, 390, 391, 392, 393, 394, 395, + 396, 397, 398, 0, 0, 0, 0, 0, 561, 562, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 694, 896, 0, 698, 0, + 533, 0, 0, 971, 0, 0, 0, 502, 0, 0, + 420, 0, 0, 0, 900, 0, 485, 460, 984, 0, + 0, 483, 428, 518, 471, 524, 505, 532, 477, 472, + 318, 506, 365, 441, 334, 336, 726, 367, 370, 374, + 375, 450, 451, 465, 490, 509, 510, 511, 364, 348, + 484, 349, 384, 350, 319, 356, 354, 357, 492, 358, + 321, 466, 515, 0, 380, 480, 436, 322, 435, 467, + 514, 513, 335, 541, 548, 549, 639, 0, 554, 737, + 738, 739, 563, 0, 473, 331, 330, 0, 0, 0, + 360, 468, 344, 346, 347, 345, 463, 464, 568, 569, + 570, 572, 0, 573, 574, 0, 0, 0, 0, 575, + 640, 656, 624, 593, 556, 648, 590, 594, 595, 401, + 402, 403, 865, 659, 0, 0, 0, 547, 421, 422, + 0, 372, 371, 437, 323, 0, 0, 410, 400, 474, + 329, 368, 412, 406, 423, 424, 425, 378, 313, 314, + 732, 968, 456, 661, 696, 697, 586, 0, 983, 963, + 965, 966, 970, 974, 975, 976, 977, 978, 980, 982, + 986, 731, 0, 641, 655, 735, 654, 728, 462, 0, + 489, 652, 599, 0, 645, 618, 619, 0, 646, 614, + 650, 0, 588, 0, 557, 560, 589, 674, 675, 676, + 320, 559, 678, 679, 680, 681, 682, 683, 684, 677, + 985, 622, 598, 625, 538, 601, 600, 0, 0, 636, + 904, 637, 638, 446, 447, 448, 449, 972, 662, 342, + 558, 476, 0, 623, 0, 0, 0, 0, 0, 0, + 0, 0, 628, 629, 626, 740, 0, 685, 686, 0, + 0, 552, 553, 377, 0, 571, 385, 341, 461, 379, + 536, 409, 0, 564, 630, 565, 478, 479, 688, 693, + 689, 690, 692, 712, 453, 399, 405, 493, 411, 429, + 481, 535, 459, 486, 339, 525, 495, 434, 615, 643, + 994, 967, 993, 995, 996, 992, 997, 998, 979, 857, + 0, 911, 912, 990, 989, 991, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 670, 669, 668, + 667, 666, 665, 664, 663, 0, 0, 612, 512, 355, + 307, 351, 352, 359, 729, 725, 730, 713, 716, 715, + 691, 864, 315, 592, 427, 475, 376, 657, 658, 0, + 711, 957, 920, 921, 922, 854, 923, 917, 918, 855, + 919, 958, 909, 954, 955, 884, 914, 924, 953, 925, + 956, 885, 959, 999, 1000, 931, 915, 277, 1001, 928, + 960, 952, 951, 926, 910, 961, 962, 892, 887, 929, + 930, 916, 937, 938, 939, 942, 856, 943, 944, 945, + 946, 947, 941, 940, 906, 907, 908, 932, 933, 935, + 936, 913, 503, 888, 889, 890, 891, 0, 0, 542, + 543, 544, 567, 0, 545, 527, 591, 386, 316, 507, + 534, 727, 0, 0, 0, 0, 0, 0, 0, 642, + 653, 687, 0, 699, 700, 702, 704, 948, 706, 500, + 501, 714, 0, 0, 4053, 709, 4054, 4055, 431, 487, + 508, 494, 902, 733, 582, 583, 734, 695, 317, 0, + 849, 458, 0, 0, 597, 631, 620, 705, 585, 0, + 0, 0, 0, 0, 0, 852, 0, 0, 0, 369, + 0, 0, 426, 635, 616, 627, 617, 602, 603, 604, + 611, 381, 605, 606, 607, 577, 608, 578, 609, 610, + 893, 634, 584, 496, 442, 0, 651, 0, 0, 973, + 981, 0, 0, 0, 0, 0, 0, 0, 0, 969, + 0, 0, 0, 0, 844, 0, 0, 882, 950, 949, + 869, 879, 0, 0, 337, 246, 579, 701, 581, 580, + 3080, 0, 3081, 875, 878, 874, 872, 873, 0, 964, + 0, 0, 0, 0, 0, 0, 836, 848, 0, 853, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 845, 846, 0, 0, 0, + 0, 903, 0, 847, 0, 0, 0, 0, 0, 497, + 526, 0, 539, 0, 407, 408, 898, 876, 880, 0, + 0, 0, 0, 324, 504, 523, 338, 491, 537, 343, + 499, 516, 333, 457, 488, 0, 0, 326, 521, 498, + 439, 325, 0, 482, 366, 383, 363, 455, 877, 0, + 901, 905, 362, 987, 899, 531, 328, 0, 530, 454, + 517, 522, 440, 433, 0, 327, 519, 438, 432, 413, + 373, 988, 414, 415, 416, 417, 418, 419, 387, 469, + 430, 470, 388, 444, 443, 445, 389, 390, 391, 392, + 393, 394, 395, 396, 397, 398, 0, 0, 0, 0, + 0, 561, 562, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 694, 896, + 0, 698, 0, 533, 0, 0, 971, 0, 0, 0, + 502, 0, 0, 420, 0, 0, 0, 900, 0, 485, + 460, 984, 0, 0, 483, 428, 518, 471, 524, 505, + 532, 477, 472, 318, 506, 365, 441, 334, 336, 726, + 367, 370, 374, 375, 450, 451, 465, 490, 509, 510, + 511, 364, 348, 484, 349, 384, 350, 319, 356, 354, + 357, 492, 358, 321, 466, 515, 0, 380, 480, 436, + 322, 435, 467, 514, 513, 335, 541, 548, 549, 639, + 0, 554, 737, 738, 739, 563, 0, 473, 331, 330, + 0, 0, 0, 360, 468, 344, 346, 347, 345, 463, + 464, 568, 569, 570, 572, 0, 573, 574, 0, 0, + 0, 0, 575, 640, 656, 624, 593, 556, 648, 590, + 594, 595, 401, 402, 403, 865, 659, 0, 0, 0, + 547, 421, 422, 0, 372, 371, 437, 323, 0, 0, + 410, 400, 474, 329, 368, 412, 406, 423, 424, 425, + 378, 313, 314, 732, 968, 456, 661, 696, 697, 586, + 0, 983, 963, 965, 966, 970, 974, 975, 976, 977, + 978, 980, 982, 986, 731, 0, 641, 655, 735, 654, + 728, 462, 0, 489, 652, 599, 0, 645, 618, 619, + 0, 646, 614, 650, 0, 588, 0, 557, 560, 589, + 674, 675, 676, 320, 559, 678, 679, 680, 681, 682, + 683, 684, 677, 985, 622, 598, 625, 538, 601, 600, + 0, 0, 636, 904, 637, 638, 446, 447, 448, 449, + 972, 662, 342, 558, 476, 0, 623, 0, 0, 0, + 0, 0, 0, 0, 0, 628, 629, 626, 740, 0, + 685, 686, 0, 0, 552, 553, 377, 0, 571, 385, + 341, 461, 379, 536, 409, 0, 564, 630, 565, 478, + 479, 688, 693, 689, 690, 692, 712, 453, 399, 405, + 493, 411, 429, 481, 535, 459, 486, 339, 525, 495, + 434, 615, 643, 994, 967, 993, 995, 996, 992, 997, + 998, 979, 857, 0, 911, 912, 990, 989, 991, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 670, 669, 668, 667, 666, 665, 664, 663, 0, 0, + 612, 512, 355, 307, 351, 352, 359, 729, 725, 730, + 713, 716, 715, 691, 864, 315, 592, 427, 475, 376, + 657, 658, 0, 711, 957, 920, 921, 922, 854, 923, + 917, 918, 855, 919, 958, 909, 954, 955, 884, 914, + 924, 953, 925, 956, 885, 959, 999, 1000, 931, 915, + 277, 1001, 928, 960, 952, 951, 926, 910, 961, 962, + 892, 887, 929, 930, 916, 937, 938, 939, 942, 856, + 943, 944, 945, 946, 947, 941, 940, 906, 907, 908, + 932, 933, 935, 936, 913, 503, 888, 889, 890, 891, + 0, 0, 542, 543, 544, 567, 0, 545, 527, 591, + 386, 316, 507, 534, 727, 0, 0, 0, 0, 0, + 0, 0, 642, 653, 687, 0, 699, 700, 702, 704, + 948, 706, 500, 501, 714, 0, 0, 934, 709, 710, + 707, 431, 487, 508, 494, 902, 733, 582, 583, 734, + 695, 317, 0, 849, 458, 0, 0, 597, 631, 620, + 705, 585, 0, 0, 1915, 0, 0, 0, 852, 0, + 0, 0, 369, 0, 0, 426, 635, 616, 627, 617, + 602, 603, 604, 611, 381, 605, 606, 607, 577, 608, + 578, 609, 610, 893, 634, 584, 496, 442, 0, 651, + 0, 0, 973, 981, 0, 0, 0, 0, 0, 0, + 0, 0, 969, 0, 0, 0, 0, 844, 0, 0, + 882, 950, 949, 869, 879, 0, 0, 337, 246, 579, + 701, 581, 580, 870, 0, 871, 875, 878, 874, 872, + 873, 0, 964, 0, 0, 0, 0, 0, 0, 0, + 848, 0, 853, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 845, 846, + 0, 0, 0, 0, 903, 0, 847, 0, 0, 0, + 0, 0, 497, 526, 0, 539, 0, 407, 408, 898, + 876, 880, 0, 0, 0, 0, 324, 504, 523, 338, + 491, 537, 343, 499, 516, 333, 457, 488, 0, 0, + 326, 521, 498, 439, 325, 0, 482, 366, 383, 363, + 455, 877, 0, 901, 905, 362, 987, 899, 531, 328, + 0, 530, 454, 517, 522, 440, 433, 0, 327, 519, + 438, 432, 413, 373, 988, 414, 415, 416, 417, 418, + 419, 387, 469, 430, 470, 388, 444, 443, 445, 389, + 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, + 0, 0, 0, 0, 561, 562, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 694, 896, 0, 698, 0, 533, 0, 0, 971, + 0, 0, 0, 502, 0, 0, 420, 0, 0, 0, + 900, 0, 485, 460, 984, 0, 0, 483, 428, 518, + 471, 524, 505, 532, 477, 472, 318, 506, 365, 441, + 334, 336, 726, 367, 370, 374, 375, 450, 451, 465, + 490, 509, 510, 511, 364, 348, 484, 349, 384, 350, + 319, 356, 354, 357, 492, 358, 321, 466, 515, 0, + 380, 480, 436, 322, 435, 467, 514, 513, 335, 541, + 1916, 1917, 639, 0, 554, 737, 738, 739, 563, 0, + 473, 331, 330, 0, 0, 0, 360, 468, 344, 346, + 347, 345, 463, 464, 568, 569, 570, 572, 0, 573, + 574, 0, 0, 0, 0, 575, 640, 656, 624, 593, + 556, 648, 590, 594, 595, 401, 402, 403, 865, 659, + 0, 0, 0, 547, 421, 422, 0, 372, 371, 437, + 323, 0, 0, 410, 400, 474, 329, 368, 412, 406, + 423, 424, 425, 378, 313, 314, 732, 968, 456, 661, + 696, 697, 586, 0, 983, 963, 965, 966, 970, 974, + 975, 976, 977, 978, 980, 982, 986, 731, 0, 641, + 655, 735, 654, 728, 462, 0, 489, 652, 599, 0, + 645, 618, 619, 0, 646, 614, 650, 0, 588, 0, + 557, 560, 589, 674, 675, 676, 320, 559, 678, 679, + 680, 681, 682, 683, 684, 677, 985, 622, 598, 625, + 538, 601, 600, 0, 0, 636, 904, 637, 638, 446, + 447, 448, 449, 972, 662, 342, 558, 476, 0, 623, + 0, 0, 0, 0, 0, 0, 0, 0, 628, 629, + 626, 740, 0, 685, 686, 0, 0, 552, 553, 377, + 0, 571, 385, 341, 461, 379, 536, 409, 0, 564, + 630, 565, 478, 479, 688, 693, 689, 690, 692, 712, + 453, 399, 405, 493, 411, 429, 481, 535, 459, 486, + 339, 525, 495, 434, 615, 643, 994, 967, 993, 995, + 996, 992, 997, 998, 979, 857, 0, 911, 912, 990, + 989, 991, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 670, 669, 668, 667, 666, 665, 664, + 663, 0, 0, 612, 512, 355, 307, 351, 352, 359, + 729, 725, 730, 713, 716, 715, 691, 864, 315, 592, + 427, 475, 376, 657, 658, 0, 711, 957, 920, 921, + 922, 854, 923, 917, 918, 855, 919, 958, 909, 954, + 955, 884, 914, 924, 953, 925, 956, 885, 959, 999, + 1000, 931, 915, 277, 1001, 928, 960, 952, 951, 926, + 910, 961, 962, 892, 887, 929, 930, 916, 937, 938, + 939, 942, 856, 943, 944, 945, 946, 947, 941, 940, + 906, 907, 908, 932, 933, 935, 936, 913, 503, 888, + 889, 890, 891, 0, 0, 542, 543, 544, 567, 0, + 545, 527, 591, 386, 316, 507, 534, 727, 0, 0, + 0, 0, 0, 0, 0, 642, 653, 687, 0, 699, + 700, 702, 704, 948, 706, 500, 501, 714, 0, 0, + 934, 709, 710, 707, 431, 487, 508, 494, 902, 733, + 582, 583, 734, 695, 317, 0, 849, 458, 0, 0, + 597, 631, 620, 705, 585, 0, 0, 0, 0, 0, + 0, 852, 0, 0, 0, 369, 0, 0, 426, 635, + 616, 627, 617, 602, 603, 604, 611, 381, 605, 606, + 607, 577, 608, 578, 609, 610, 893, 634, 584, 496, + 442, 0, 651, 0, 0, 973, 981, 0, 0, 0, + 0, 0, 0, 0, 0, 969, 0, 0, 0, 0, + 1451, 0, 0, 882, 950, 949, 869, 879, 0, 0, + 337, 246, 579, 701, 581, 580, 870, 0, 871, 875, + 878, 874, 872, 873, 0, 964, 0, 0, 0, 0, + 0, 0, 836, 848, 0, 853, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 845, 846, 0, 0, 0, 0, 903, 0, 847, + 0, 0, 0, 0, 0, 497, 526, 0, 539, 0, + 407, 408, 898, 876, 880, 0, 0, 0, 0, 324, + 504, 523, 338, 491, 537, 343, 499, 516, 333, 457, + 488, 0, 0, 326, 521, 498, 439, 325, 0, 482, + 366, 383, 363, 455, 877, 0, 901, 905, 362, 987, + 899, 531, 328, 0, 530, 454, 517, 522, 440, 433, + 0, 327, 519, 438, 432, 413, 373, 988, 414, 415, + 416, 417, 418, 419, 387, 469, 430, 470, 388, 444, + 443, 445, 389, 390, 391, 392, 393, 394, 395, 396, + 397, 398, 0, 0, 0, 0, 0, 561, 562, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 694, 896, 0, 698, 0, 533, + 0, 0, 971, 0, 0, 0, 502, 0, 0, 420, + 0, 0, 0, 900, 0, 485, 460, 984, 0, 0, + 483, 428, 518, 471, 524, 505, 532, 477, 472, 318, + 506, 365, 441, 334, 336, 726, 367, 370, 374, 375, + 450, 451, 465, 490, 509, 510, 511, 364, 348, 484, + 349, 384, 350, 319, 356, 354, 357, 492, 358, 321, + 466, 515, 0, 380, 480, 436, 322, 435, 467, 514, + 513, 335, 541, 548, 549, 639, 0, 554, 737, 738, + 739, 563, 0, 473, 331, 330, 0, 0, 0, 360, + 468, 344, 346, 347, 345, 463, 464, 568, 569, 570, + 572, 0, 573, 574, 0, 0, 0, 0, 575, 640, + 656, 624, 593, 556, 648, 590, 594, 595, 401, 402, + 403, 865, 659, 0, 0, 0, 547, 421, 422, 0, + 372, 371, 437, 323, 0, 0, 410, 400, 474, 329, + 368, 412, 406, 423, 424, 425, 378, 313, 314, 732, + 968, 456, 661, 696, 697, 586, 0, 983, 963, 965, + 966, 970, 974, 975, 976, 977, 978, 980, 982, 986, + 731, 0, 641, 655, 735, 654, 728, 462, 0, 489, + 652, 599, 0, 645, 618, 619, 0, 646, 614, 650, + 0, 588, 0, 557, 560, 589, 674, 675, 676, 320, + 559, 678, 679, 680, 681, 682, 683, 684, 677, 985, + 622, 598, 625, 538, 601, 600, 0, 0, 636, 904, + 637, 638, 446, 447, 448, 449, 972, 662, 342, 558, + 476, 0, 623, 0, 0, 0, 0, 0, 0, 0, + 0, 628, 629, 626, 740, 0, 685, 686, 0, 0, + 552, 553, 377, 0, 571, 385, 341, 461, 379, 536, + 409, 0, 564, 630, 565, 478, 479, 688, 693, 689, + 690, 692, 712, 453, 399, 405, 493, 411, 429, 481, + 535, 459, 486, 339, 525, 495, 434, 615, 643, 994, + 967, 993, 995, 996, 992, 997, 998, 979, 857, 0, + 911, 912, 990, 989, 991, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 670, 669, 668, 667, + 666, 665, 664, 663, 0, 0, 612, 512, 355, 307, + 351, 352, 359, 729, 725, 730, 713, 716, 715, 691, + 864, 315, 592, 427, 475, 376, 657, 658, 0, 711, + 957, 920, 921, 922, 854, 923, 917, 918, 855, 919, + 958, 909, 954, 955, 884, 914, 924, 953, 925, 956, + 885, 959, 999, 1000, 931, 915, 277, 1001, 928, 960, + 952, 951, 926, 910, 961, 962, 892, 887, 929, 930, + 916, 937, 938, 939, 942, 856, 943, 944, 945, 946, + 947, 941, 940, 906, 907, 908, 932, 933, 935, 936, + 913, 503, 888, 889, 890, 891, 0, 0, 542, 543, + 544, 567, 0, 545, 527, 591, 386, 316, 507, 534, + 727, 0, 0, 0, 0, 0, 0, 0, 642, 653, + 687, 0, 699, 700, 702, 704, 948, 706, 500, 501, + 714, 0, 0, 934, 709, 710, 707, 431, 487, 508, + 494, 902, 733, 582, 583, 734, 695, 317, 0, 849, + 458, 0, 0, 597, 631, 620, 705, 585, 0, 0, + 0, 0, 0, 0, 852, 0, 0, 0, 369, 0, + 0, 426, 635, 616, 627, 617, 602, 603, 604, 611, + 381, 605, 606, 607, 577, 608, 578, 609, 610, 893, + 634, 584, 496, 442, 0, 651, 0, 0, 973, 981, + 0, 0, 0, 0, 0, 0, 0, 0, 969, 0, + 0, 0, 0, 844, 0, 0, 882, 950, 949, 869, + 879, 0, 0, 337, 246, 579, 701, 581, 580, 870, + 0, 871, 875, 878, 874, 872, 873, 0, 964, 0, + 0, 0, 0, 0, 0, 0, 848, 0, 853, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 845, 846, 0, 0, 0, 0, + 903, 0, 847, 0, 0, 0, 0, 0, 497, 526, + 0, 539, 0, 407, 408, 898, 876, 880, 0, 0, + 0, 0, 324, 504, 523, 338, 491, 537, 343, 499, + 516, 333, 457, 488, 0, 0, 326, 521, 498, 439, + 325, 0, 482, 366, 383, 363, 455, 877, 0, 901, + 905, 362, 987, 899, 531, 328, 0, 530, 454, 517, + 522, 440, 433, 0, 327, 519, 438, 432, 413, 373, + 988, 414, 415, 416, 417, 418, 419, 387, 469, 430, + 470, 388, 444, 443, 445, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, - 556, 557, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 689, 890, 0, - 693, 0, 528, 0, 0, 965, 0, 0, 0, 497, - 0, 0, 415, 0, 0, 0, 894, 0, 480, 455, - 978, 0, 0, 478, 423, 513, 466, 519, 500, 527, - 472, 467, 318, 501, 365, 436, 334, 336, 721, 367, - 370, 374, 375, 445, 446, 460, 485, 504, 505, 506, - 364, 348, 479, 349, 384, 350, 319, 356, 354, 357, - 487, 358, 321, 461, 510, 0, 380, 475, 431, 322, - 430, 462, 509, 508, 335, 536, 543, 544, 634, 0, - 549, 732, 733, 734, 558, 0, 468, 331, 330, 0, - 0, 0, 360, 463, 344, 346, 347, 345, 458, 459, - 563, 564, 565, 567, 0, 568, 569, 0, 0, 0, - 0, 570, 635, 651, 619, 588, 551, 643, 585, 589, - 590, 401, 402, 403, 654, 0, 0, 0, 542, 416, - 417, 0, 372, 371, 432, 323, 0, 0, 409, 400, - 469, 329, 368, 411, 405, 418, 419, 420, 378, 313, - 314, 727, 962, 451, 656, 691, 692, 581, 0, 977, - 957, 959, 960, 964, 968, 969, 970, 971, 972, 974, - 976, 980, 726, 0, 636, 650, 730, 649, 723, 457, - 0, 484, 647, 594, 0, 640, 613, 614, 0, 641, - 609, 645, 0, 583, 0, 552, 555, 584, 669, 670, - 671, 320, 554, 673, 674, 675, 676, 677, 678, 679, - 672, 979, 617, 593, 620, 533, 596, 595, 0, 0, - 631, 898, 632, 633, 441, 442, 443, 444, 966, 657, - 342, 553, 471, 0, 618, 0, 0, 0, 0, 0, - 0, 0, 0, 623, 624, 621, 735, 0, 680, 681, - 0, 0, 547, 548, 377, 0, 566, 385, 341, 456, - 379, 531, 408, 0, 559, 625, 560, 473, 474, 683, - 688, 684, 685, 687, 707, 448, 399, 404, 488, 410, - 424, 476, 530, 454, 481, 339, 520, 490, 429, 610, - 638, 988, 961, 987, 989, 990, 986, 991, 992, 973, - 852, 0, 905, 906, 984, 983, 985, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 665, 664, - 663, 662, 661, 660, 659, 658, 0, 0, 607, 507, - 355, 307, 351, 352, 359, 724, 720, 725, 708, 711, - 710, 686, 859, 315, 587, 422, 470, 376, 652, 653, - 0, 706, 951, 914, 915, 916, 849, 917, 911, 912, - 850, 913, 952, 903, 948, 949, 878, 908, 918, 947, - 919, 950, 879, 953, 993, 994, 925, 909, 277, 995, - 922, 954, 946, 945, 920, 904, 955, 956, 886, 881, - 923, 924, 910, 931, 932, 933, 936, 851, 937, 938, - 939, 940, 941, 935, 934, 900, 901, 902, 926, 927, - 929, 930, 907, 498, 882, 883, 884, 885, 0, 0, - 537, 538, 539, 562, 0, 540, 522, 586, 386, 316, - 502, 529, 722, 0, 0, 0, 0, 0, 0, 0, - 637, 648, 682, 0, 694, 695, 697, 699, 942, 701, - 495, 496, 709, 0, 0, 928, 704, 705, 702, 426, - 482, 503, 489, 896, 728, 577, 578, 729, 690, 317, - 0, 844, 453, 0, 0, 592, 626, 615, 700, 580, - 0, 0, 0, 0, 0, 0, 847, 0, 0, 0, - 369, 0, 0, 421, 630, 611, 622, 612, 597, 598, - 599, 606, 381, 600, 601, 602, 572, 603, 573, 604, - 605, 887, 629, 579, 491, 437, 0, 646, 0, 0, - 967, 975, 0, 0, 0, 0, 0, 0, 0, 0, - 963, 0, 0, 0, 0, 839, 0, 0, 876, 944, - 943, 863, 873, 0, 0, 337, 246, 574, 696, 576, - 575, 864, 0, 865, 869, 872, 868, 866, 867, 0, - 958, 0, 0, 0, 0, 0, 0, 0, 843, 0, - 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 840, 841, 0, 0, - 0, 0, 897, 0, 842, 0, 0, 0, 0, 0, - 492, 521, 0, 534, 0, 406, 407, 892, 870, 874, - 0, 0, 0, 0, 324, 499, 518, 338, 486, 532, - 343, 494, 511, 333, 452, 483, 0, 0, 326, 516, - 493, 434, 325, 0, 477, 366, 383, 363, 450, 871, - 0, 895, 899, 362, 981, 893, 526, 328, 0, 525, - 449, 512, 517, 435, 428, 0, 327, 514, 433, 427, - 412, 373, 982, 413, 414, 387, 464, 425, 465, 388, - 439, 438, 440, 389, 390, 391, 392, 393, 394, 395, - 396, 397, 398, 0, 0, 0, 0, 0, 556, 557, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 689, 890, 0, 693, 0, - 528, 0, 0, 965, 0, 0, 0, 497, 0, 0, - 415, 0, 0, 0, 894, 0, 480, 455, 978, 0, - 0, 478, 423, 513, 466, 519, 500, 527, 472, 467, - 318, 501, 365, 436, 334, 336, 721, 367, 370, 374, - 375, 445, 446, 460, 485, 504, 505, 506, 364, 348, - 479, 349, 384, 350, 319, 356, 354, 357, 487, 358, - 321, 461, 510, 0, 380, 475, 431, 322, 430, 462, - 509, 508, 335, 536, 543, 544, 634, 0, 549, 732, - 733, 734, 558, 0, 468, 331, 330, 0, 0, 0, - 360, 463, 344, 346, 347, 345, 458, 459, 563, 564, - 565, 567, 0, 568, 569, 0, 0, 0, 0, 570, - 635, 651, 619, 588, 551, 643, 585, 589, 590, 401, - 402, 403, 654, 0, 0, 0, 542, 416, 417, 0, - 372, 371, 432, 323, 0, 0, 409, 400, 469, 329, - 368, 411, 405, 418, 419, 420, 378, 313, 314, 727, - 962, 451, 656, 691, 692, 581, 0, 977, 957, 959, - 960, 964, 968, 969, 970, 971, 972, 974, 976, 980, - 726, 0, 636, 650, 730, 649, 723, 457, 0, 484, - 647, 594, 0, 640, 613, 614, 0, 641, 609, 645, - 0, 583, 0, 552, 555, 584, 669, 670, 671, 320, - 554, 673, 674, 675, 676, 677, 678, 679, 672, 979, - 617, 593, 620, 533, 596, 595, 0, 0, 631, 898, - 632, 633, 441, 442, 443, 444, 966, 657, 342, 553, - 471, 0, 618, 0, 0, 0, 0, 0, 0, 0, - 0, 623, 624, 621, 735, 0, 680, 681, 0, 0, - 547, 548, 377, 0, 566, 385, 341, 456, 379, 531, - 408, 0, 559, 625, 560, 473, 474, 683, 688, 684, - 685, 687, 707, 448, 399, 404, 488, 410, 424, 476, - 530, 454, 481, 339, 520, 490, 429, 610, 638, 988, - 961, 987, 989, 990, 986, 991, 992, 973, 852, 0, - 905, 906, 984, 983, 985, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 665, 664, 663, 662, - 661, 660, 659, 658, 0, 0, 607, 507, 355, 307, - 351, 352, 359, 724, 720, 725, 708, 711, 710, 686, - 859, 315, 587, 422, 470, 376, 652, 653, 0, 706, - 951, 914, 915, 916, 849, 917, 911, 912, 850, 913, - 952, 903, 948, 949, 878, 908, 918, 947, 919, 950, - 879, 953, 993, 994, 925, 909, 277, 995, 922, 954, - 946, 945, 920, 904, 955, 956, 886, 881, 923, 924, - 910, 931, 932, 933, 936, 851, 937, 938, 939, 940, - 941, 935, 934, 900, 901, 902, 926, 927, 929, 930, - 907, 498, 882, 883, 884, 885, 0, 0, 537, 538, - 539, 562, 0, 540, 522, 586, 386, 316, 502, 529, - 722, 0, 0, 0, 0, 0, 0, 0, 637, 648, - 682, 0, 694, 695, 697, 699, 942, 701, 495, 496, - 709, 0, 0, 928, 704, 705, 702, 426, 482, 503, - 489, 0, 728, 577, 578, 729, 690, 317, 0, 844, - 183, 223, 182, 214, 184, 0, 0, 0, 0, 0, - 0, 453, 0, 0, 592, 626, 615, 700, 580, 0, - 215, 0, 0, 0, 0, 0, 0, 206, 0, 369, - 0, 216, 421, 630, 611, 622, 612, 597, 598, 599, - 606, 381, 600, 601, 602, 572, 603, 573, 604, 605, - 153, 629, 579, 491, 437, 0, 646, 0, 0, 0, - 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, - 0, 0, 0, 0, 219, 0, 0, 245, 0, 0, - 0, 0, 0, 0, 337, 246, 574, 696, 576, 575, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, - 521, 0, 534, 0, 406, 407, 0, 0, 0, 0, - 0, 0, 0, 324, 499, 518, 338, 486, 532, 343, - 494, 511, 333, 452, 483, 0, 0, 326, 516, 493, - 434, 325, 0, 477, 366, 383, 363, 450, 0, 0, - 515, 545, 362, 535, 0, 526, 328, 0, 525, 449, - 512, 517, 435, 428, 0, 327, 514, 433, 427, 412, - 373, 561, 413, 414, 387, 464, 425, 465, 388, 439, - 438, 440, 389, 390, 391, 392, 393, 394, 395, 396, - 397, 398, 0, 0, 0, 0, 0, 556, 557, 0, - 0, 0, 0, 0, 0, 0, 181, 212, 221, 213, - 75, 137, 0, 0, 689, 0, 0, 693, 0, 528, - 0, 0, 238, 0, 0, 0, 497, 0, 0, 415, - 211, 205, 204, 546, 0, 480, 455, 250, 0, 0, - 478, 423, 513, 466, 519, 500, 527, 472, 467, 318, - 501, 365, 436, 334, 336, 258, 367, 370, 374, 375, - 445, 446, 460, 485, 504, 505, 506, 364, 348, 479, - 349, 384, 350, 319, 356, 354, 357, 487, 358, 321, - 461, 510, 0, 380, 475, 431, 322, 430, 462, 509, - 508, 335, 536, 543, 544, 634, 0, 549, 666, 667, - 668, 558, 0, 468, 331, 330, 0, 0, 0, 360, - 463, 344, 346, 347, 345, 458, 459, 563, 564, 565, - 567, 0, 568, 569, 0, 0, 0, 0, 570, 635, - 651, 619, 588, 551, 643, 585, 589, 590, 401, 402, - 403, 654, 0, 0, 0, 542, 416, 417, 0, 372, - 371, 432, 323, 0, 0, 409, 400, 469, 329, 368, - 411, 405, 418, 419, 420, 378, 313, 314, 523, 361, - 451, 656, 691, 692, 581, 0, 644, 582, 591, 353, - 616, 628, 627, 447, 541, 241, 639, 642, 571, 251, - 0, 636, 650, 608, 649, 252, 457, 0, 484, 647, - 594, 0, 640, 613, 614, 0, 641, 609, 645, 0, - 583, 0, 552, 555, 584, 669, 670, 671, 320, 554, - 673, 674, 675, 676, 677, 678, 679, 672, 524, 617, - 593, 620, 533, 596, 595, 0, 0, 631, 550, 632, - 633, 441, 442, 443, 444, 382, 657, 342, 553, 471, - 151, 618, 0, 0, 0, 0, 0, 0, 0, 0, - 623, 624, 621, 249, 0, 680, 681, 0, 0, 547, - 548, 377, 0, 566, 385, 341, 456, 379, 531, 408, - 0, 559, 625, 560, 473, 474, 683, 688, 684, 685, - 687, 707, 448, 399, 404, 488, 410, 424, 476, 530, - 454, 481, 339, 520, 490, 429, 610, 638, 0, 0, - 0, 0, 0, 0, 0, 0, 71, 0, 0, 300, + 561, 562, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 694, 896, 0, + 698, 0, 533, 0, 0, 971, 0, 0, 0, 502, + 0, 0, 420, 0, 0, 0, 900, 0, 485, 460, + 984, 0, 0, 483, 428, 518, 471, 524, 505, 532, + 477, 472, 318, 506, 365, 441, 334, 336, 726, 367, + 370, 374, 375, 450, 451, 465, 490, 509, 510, 511, + 364, 348, 484, 349, 384, 350, 319, 356, 354, 357, + 492, 358, 321, 466, 515, 0, 380, 480, 436, 322, + 435, 467, 514, 513, 335, 541, 548, 549, 639, 0, + 554, 737, 738, 739, 563, 0, 473, 331, 330, 0, + 0, 0, 360, 468, 344, 346, 347, 345, 463, 464, + 568, 569, 570, 572, 0, 573, 574, 0, 0, 0, + 0, 575, 640, 656, 624, 593, 556, 648, 590, 594, + 595, 401, 402, 403, 865, 659, 0, 0, 0, 547, + 421, 422, 0, 372, 371, 437, 323, 0, 0, 410, + 400, 474, 329, 368, 412, 406, 423, 424, 425, 378, + 313, 314, 732, 968, 456, 661, 696, 697, 586, 0, + 983, 963, 965, 966, 970, 974, 975, 976, 977, 978, + 980, 982, 986, 731, 0, 641, 655, 735, 654, 728, + 462, 0, 489, 652, 599, 0, 645, 618, 619, 0, + 646, 614, 650, 0, 588, 0, 557, 560, 589, 674, + 675, 676, 320, 559, 678, 679, 680, 681, 682, 683, + 684, 677, 985, 622, 598, 625, 538, 601, 600, 0, + 0, 636, 904, 637, 638, 446, 447, 448, 449, 972, + 662, 342, 558, 476, 0, 623, 0, 0, 0, 0, + 0, 0, 0, 0, 628, 629, 626, 740, 0, 685, + 686, 0, 0, 552, 553, 377, 0, 571, 385, 341, + 461, 379, 536, 409, 0, 564, 630, 565, 478, 479, + 688, 693, 689, 690, 692, 712, 453, 399, 405, 493, + 411, 429, 481, 535, 459, 486, 339, 525, 495, 434, + 615, 643, 994, 967, 993, 995, 996, 992, 997, 998, + 979, 857, 0, 911, 912, 990, 989, 991, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 670, + 669, 668, 667, 666, 665, 664, 663, 0, 0, 612, + 512, 355, 307, 351, 352, 359, 729, 725, 730, 713, + 716, 715, 691, 864, 315, 592, 427, 475, 376, 657, + 658, 0, 711, 957, 920, 921, 922, 854, 923, 917, + 918, 855, 919, 958, 909, 954, 955, 884, 914, 924, + 953, 925, 956, 885, 959, 999, 1000, 931, 915, 277, + 1001, 928, 960, 952, 951, 926, 910, 961, 962, 892, + 887, 929, 930, 916, 937, 938, 939, 942, 856, 943, + 944, 945, 946, 947, 941, 940, 906, 907, 908, 932, + 933, 935, 936, 913, 503, 888, 889, 890, 891, 0, + 0, 542, 543, 544, 567, 0, 545, 527, 591, 386, + 316, 507, 534, 727, 0, 0, 0, 0, 0, 0, + 0, 642, 653, 687, 0, 699, 700, 702, 704, 948, + 706, 500, 501, 714, 0, 0, 934, 709, 710, 707, + 431, 487, 508, 494, 0, 733, 582, 583, 734, 695, + 317, 0, 849, 183, 223, 182, 214, 184, 0, 0, + 0, 0, 0, 0, 458, 0, 0, 597, 631, 620, + 705, 585, 0, 215, 0, 0, 0, 0, 0, 0, + 206, 0, 369, 0, 216, 426, 635, 616, 627, 617, + 602, 603, 604, 611, 381, 605, 606, 607, 577, 608, + 578, 609, 610, 153, 634, 584, 496, 442, 0, 651, + 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, + 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, + 245, 0, 0, 0, 0, 0, 0, 337, 246, 579, + 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 497, 526, 0, 539, 0, 407, 408, 0, + 0, 0, 0, 0, 0, 0, 324, 504, 523, 338, + 491, 537, 343, 499, 516, 333, 457, 488, 0, 0, + 326, 521, 498, 439, 325, 0, 482, 366, 383, 363, + 455, 0, 0, 520, 550, 362, 540, 0, 531, 328, + 0, 530, 454, 517, 522, 440, 433, 0, 327, 519, + 438, 432, 413, 373, 566, 414, 415, 416, 417, 418, + 419, 387, 469, 430, 470, 388, 444, 443, 445, 389, + 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, + 0, 0, 0, 0, 561, 562, 0, 0, 0, 0, + 0, 0, 0, 181, 212, 221, 213, 75, 137, 0, + 0, 694, 0, 0, 698, 0, 533, 0, 0, 238, + 0, 0, 0, 502, 0, 0, 420, 211, 205, 204, + 551, 0, 485, 460, 250, 0, 0, 483, 428, 518, + 471, 524, 505, 532, 477, 472, 318, 506, 365, 441, + 334, 336, 258, 367, 370, 374, 375, 450, 451, 465, + 490, 509, 510, 511, 364, 348, 484, 349, 384, 350, + 319, 356, 354, 357, 492, 358, 321, 466, 515, 0, + 380, 480, 436, 322, 435, 467, 514, 513, 335, 541, + 548, 549, 639, 0, 554, 671, 672, 673, 563, 0, + 473, 331, 330, 0, 0, 0, 360, 468, 344, 346, + 347, 345, 463, 464, 568, 569, 570, 572, 0, 573, + 574, 0, 0, 0, 0, 575, 640, 656, 624, 593, + 556, 648, 590, 594, 595, 401, 402, 403, 404, 659, + 0, 0, 0, 547, 421, 422, 0, 372, 371, 437, + 323, 0, 0, 410, 400, 474, 329, 368, 412, 406, + 423, 424, 425, 378, 313, 314, 528, 361, 456, 661, + 696, 697, 586, 0, 649, 587, 596, 353, 621, 633, + 632, 452, 546, 241, 644, 647, 576, 251, 0, 641, + 655, 613, 654, 252, 462, 0, 489, 652, 599, 0, + 645, 618, 619, 0, 646, 614, 650, 0, 588, 0, + 557, 560, 589, 674, 675, 676, 320, 559, 678, 679, + 680, 681, 682, 683, 684, 677, 529, 622, 598, 625, + 538, 601, 600, 0, 0, 636, 555, 637, 638, 446, + 447, 448, 449, 382, 662, 342, 558, 476, 151, 623, + 0, 0, 0, 0, 0, 0, 0, 0, 628, 629, + 626, 249, 0, 685, 686, 0, 0, 552, 553, 377, + 0, 571, 385, 341, 461, 379, 536, 409, 0, 564, + 630, 565, 478, 479, 688, 693, 689, 690, 692, 712, + 453, 399, 405, 493, 411, 429, 481, 535, 459, 486, + 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, + 0, 0, 0, 0, 71, 0, 0, 300, 301, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 670, 669, 668, 667, 666, 665, 664, + 663, 0, 0, 612, 512, 355, 307, 351, 352, 359, + 256, 332, 257, 713, 716, 715, 691, 0, 315, 592, + 427, 475, 376, 657, 658, 66, 711, 259, 260, 261, + 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, + 271, 272, 273, 278, 279, 280, 281, 282, 283, 284, + 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 298, 299, 0, 0, + 0, 0, 309, 717, 718, 719, 720, 721, 0, 0, + 310, 311, 312, 0, 0, 274, 275, 302, 503, 303, + 304, 305, 306, 0, 0, 542, 543, 544, 567, 0, + 545, 527, 591, 386, 316, 507, 534, 253, 49, 239, + 242, 244, 243, 0, 67, 642, 653, 687, 5, 699, + 700, 702, 704, 703, 706, 500, 501, 714, 0, 0, + 708, 709, 710, 707, 431, 487, 508, 494, 156, 254, + 582, 583, 255, 695, 317, 183, 223, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 458, 0, 0, 597, + 631, 620, 705, 585, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 369, 0, 0, 426, 635, 616, + 627, 617, 602, 603, 604, 611, 381, 605, 606, 607, + 577, 608, 578, 609, 610, 153, 634, 584, 496, 442, + 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, + 0, 0, 245, 0, 0, 0, 0, 0, 0, 337, + 246, 579, 701, 581, 580, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 340, 2710, 2713, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 497, 526, 0, 539, 0, 407, + 408, 0, 0, 0, 0, 0, 0, 0, 324, 504, + 523, 338, 491, 537, 343, 499, 516, 333, 457, 488, + 0, 0, 326, 521, 498, 439, 325, 0, 482, 366, + 383, 363, 455, 0, 0, 520, 550, 362, 540, 0, + 531, 328, 0, 530, 454, 517, 522, 440, 433, 0, + 327, 519, 438, 432, 413, 373, 566, 414, 415, 416, + 417, 418, 419, 387, 469, 430, 470, 388, 444, 443, + 445, 389, 390, 391, 392, 393, 394, 395, 396, 397, + 398, 0, 0, 0, 0, 0, 561, 562, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 694, 0, 0, 698, 2714, 533, 0, + 0, 0, 2709, 0, 2708, 502, 2706, 2711, 420, 0, + 0, 0, 551, 0, 485, 460, 736, 0, 0, 483, + 428, 518, 471, 524, 505, 532, 477, 472, 318, 506, + 365, 441, 334, 336, 726, 367, 370, 374, 375, 450, + 451, 465, 490, 509, 510, 511, 364, 348, 484, 349, + 384, 350, 319, 356, 354, 357, 492, 358, 321, 466, + 515, 2712, 380, 480, 436, 322, 435, 467, 514, 513, + 335, 541, 548, 549, 639, 0, 554, 737, 738, 739, + 563, 0, 473, 331, 330, 0, 0, 0, 360, 468, + 344, 346, 347, 345, 463, 464, 568, 569, 570, 572, + 0, 573, 574, 0, 0, 0, 0, 575, 640, 656, + 624, 593, 556, 648, 590, 594, 595, 401, 402, 403, + 404, 659, 0, 0, 0, 547, 421, 422, 0, 372, + 371, 437, 323, 0, 0, 410, 400, 474, 329, 368, + 412, 406, 423, 424, 425, 378, 313, 314, 732, 361, + 456, 661, 696, 697, 586, 0, 649, 587, 596, 353, + 621, 633, 632, 452, 546, 0, 644, 647, 576, 731, + 0, 641, 655, 735, 654, 728, 462, 0, 489, 652, + 599, 0, 645, 618, 619, 0, 646, 614, 650, 0, + 588, 0, 557, 560, 589, 674, 675, 676, 320, 559, + 678, 679, 680, 681, 682, 683, 684, 677, 529, 622, + 598, 625, 538, 601, 600, 0, 0, 636, 555, 637, + 638, 446, 447, 448, 449, 382, 662, 342, 558, 476, + 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, + 628, 629, 626, 740, 0, 685, 686, 0, 0, 552, + 553, 377, 0, 571, 385, 341, 461, 379, 536, 409, + 0, 564, 630, 565, 478, 479, 688, 693, 689, 690, + 692, 712, 453, 399, 405, 493, 411, 429, 481, 535, + 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 665, 664, 663, 662, 661, - 660, 659, 658, 0, 0, 607, 507, 355, 307, 351, - 352, 359, 256, 332, 257, 708, 711, 710, 686, 0, - 315, 587, 422, 470, 376, 652, 653, 66, 706, 259, + 0, 0, 0, 0, 0, 670, 669, 668, 667, 666, + 665, 664, 663, 0, 0, 612, 512, 355, 307, 351, + 352, 359, 729, 725, 730, 713, 716, 715, 691, 0, + 315, 592, 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, - 283, 284, 285, 655, 276, 277, 286, 287, 288, 289, + 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, - 0, 0, 0, 0, 309, 712, 713, 714, 715, 716, + 0, 0, 0, 0, 309, 717, 718, 719, 720, 721, 0, 0, 310, 311, 312, 0, 0, 274, 275, 302, - 498, 303, 304, 305, 306, 0, 0, 537, 538, 539, - 562, 0, 540, 522, 586, 386, 316, 502, 529, 253, - 49, 239, 242, 244, 243, 0, 67, 637, 648, 682, - 5, 694, 695, 697, 699, 698, 701, 495, 496, 709, - 0, 0, 703, 704, 705, 702, 426, 482, 503, 489, - 156, 254, 577, 578, 255, 690, 317, 183, 223, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 453, 0, - 0, 592, 626, 615, 700, 580, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 369, 0, 0, 421, - 630, 611, 622, 612, 597, 598, 599, 606, 381, 600, - 601, 602, 572, 603, 573, 604, 605, 153, 629, 579, - 491, 437, 0, 646, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 219, 0, 0, 245, 0, 0, 0, 0, 0, - 0, 337, 246, 574, 696, 576, 575, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 340, 2693, 2696, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 492, 521, 0, 534, - 0, 406, 407, 0, 0, 0, 0, 0, 0, 0, - 324, 499, 518, 338, 486, 532, 343, 494, 511, 333, - 452, 483, 0, 0, 326, 516, 493, 434, 325, 0, - 477, 366, 383, 363, 450, 0, 0, 515, 545, 362, - 535, 0, 526, 328, 0, 525, 449, 512, 517, 435, - 428, 0, 327, 514, 433, 427, 412, 373, 561, 413, - 414, 387, 464, 425, 465, 388, 439, 438, 440, 389, - 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, - 0, 0, 0, 0, 556, 557, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 689, 0, 0, 693, 2697, 528, 0, 0, 0, - 2692, 0, 2691, 497, 2689, 2694, 415, 0, 0, 0, - 546, 0, 480, 455, 731, 0, 0, 478, 423, 513, - 466, 519, 500, 527, 472, 467, 318, 501, 365, 436, - 334, 336, 721, 367, 370, 374, 375, 445, 446, 460, - 485, 504, 505, 506, 364, 348, 479, 349, 384, 350, - 319, 356, 354, 357, 487, 358, 321, 461, 510, 2695, - 380, 475, 431, 322, 430, 462, 509, 508, 335, 536, - 543, 544, 634, 0, 549, 732, 733, 734, 558, 0, - 468, 331, 330, 0, 0, 0, 360, 463, 344, 346, - 347, 345, 458, 459, 563, 564, 565, 567, 0, 568, - 569, 0, 0, 0, 0, 570, 635, 651, 619, 588, - 551, 643, 585, 589, 590, 401, 402, 403, 654, 0, - 0, 0, 542, 416, 417, 0, 372, 371, 432, 323, - 0, 0, 409, 400, 469, 329, 368, 411, 405, 418, - 419, 420, 378, 313, 314, 727, 361, 451, 656, 691, - 692, 581, 0, 644, 582, 591, 353, 616, 628, 627, - 447, 541, 0, 639, 642, 571, 726, 0, 636, 650, - 730, 649, 723, 457, 0, 484, 647, 594, 0, 640, - 613, 614, 0, 641, 609, 645, 0, 583, 0, 552, - 555, 584, 669, 670, 671, 320, 554, 673, 674, 675, - 676, 677, 678, 679, 672, 524, 617, 593, 620, 533, - 596, 595, 0, 0, 631, 550, 632, 633, 441, 442, - 443, 444, 382, 657, 342, 553, 471, 0, 618, 0, - 0, 0, 0, 0, 0, 0, 0, 623, 624, 621, - 735, 0, 680, 681, 0, 0, 547, 548, 377, 0, - 566, 385, 341, 456, 379, 531, 408, 0, 559, 625, - 560, 473, 474, 683, 688, 684, 685, 687, 707, 448, - 399, 404, 488, 410, 424, 476, 530, 454, 481, 339, - 520, 490, 429, 610, 638, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, + 503, 303, 304, 305, 306, 0, 0, 542, 543, 544, + 567, 0, 545, 527, 591, 386, 316, 507, 534, 727, + 0, 0, 0, 0, 0, 0, 0, 642, 653, 687, + 0, 699, 700, 702, 704, 703, 706, 500, 501, 714, + 0, 0, 708, 709, 710, 707, 431, 487, 508, 494, + 0, 733, 582, 583, 734, 695, 317, 458, 0, 0, + 597, 631, 620, 705, 585, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 369, 0, 0, 426, 635, + 616, 627, 617, 602, 603, 604, 611, 381, 605, 606, + 607, 577, 608, 578, 609, 610, 0, 634, 584, 496, + 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1472, 0, 0, 245, 0, 0, 869, 879, 0, 0, + 337, 246, 579, 701, 581, 580, 870, 0, 871, 875, + 878, 874, 872, 873, 0, 340, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 497, 526, 0, 539, 0, + 407, 408, 0, 876, 0, 0, 0, 0, 0, 324, + 504, 523, 338, 491, 537, 343, 499, 516, 333, 457, + 488, 0, 0, 326, 521, 498, 439, 325, 0, 482, + 366, 383, 363, 455, 877, 0, 520, 550, 362, 540, + 0, 531, 328, 0, 530, 454, 517, 522, 440, 433, + 0, 327, 519, 438, 432, 413, 373, 566, 414, 415, + 416, 417, 418, 419, 387, 469, 430, 470, 388, 444, + 443, 445, 389, 390, 391, 392, 393, 394, 395, 396, + 397, 398, 0, 0, 0, 0, 0, 561, 562, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 694, 0, 0, 698, 0, 533, + 0, 0, 0, 0, 0, 0, 502, 0, 0, 420, + 0, 0, 0, 551, 0, 485, 460, 736, 0, 0, + 483, 428, 518, 471, 524, 505, 532, 477, 472, 318, + 506, 365, 441, 334, 336, 726, 367, 370, 374, 375, + 450, 451, 465, 490, 509, 510, 511, 364, 348, 484, + 349, 384, 350, 319, 356, 354, 357, 492, 358, 321, + 466, 515, 0, 380, 480, 436, 322, 435, 467, 514, + 513, 335, 541, 548, 549, 639, 0, 554, 737, 738, + 739, 563, 0, 473, 331, 330, 0, 0, 0, 360, + 468, 344, 346, 347, 345, 463, 464, 568, 569, 570, + 572, 0, 573, 574, 0, 0, 0, 0, 575, 640, + 656, 624, 593, 556, 648, 590, 594, 595, 401, 402, + 403, 404, 659, 0, 0, 0, 547, 421, 422, 0, + 372, 371, 437, 323, 0, 0, 410, 400, 474, 329, + 368, 412, 406, 423, 424, 425, 378, 313, 314, 732, + 361, 456, 661, 696, 697, 586, 0, 649, 587, 596, + 353, 621, 633, 632, 452, 546, 0, 644, 647, 576, + 731, 0, 641, 655, 735, 654, 728, 462, 0, 489, + 652, 599, 0, 645, 618, 619, 0, 646, 614, 650, + 0, 588, 0, 557, 560, 589, 674, 675, 676, 320, + 559, 678, 679, 680, 681, 682, 683, 684, 677, 529, + 622, 598, 625, 538, 601, 600, 0, 0, 636, 555, + 637, 638, 446, 447, 448, 449, 382, 662, 342, 558, + 476, 0, 623, 0, 0, 0, 0, 0, 0, 0, + 0, 628, 629, 626, 740, 0, 685, 686, 0, 0, + 552, 553, 377, 0, 571, 385, 341, 461, 379, 536, + 409, 0, 564, 630, 565, 478, 479, 688, 693, 689, + 690, 692, 712, 453, 399, 405, 493, 411, 429, 481, + 535, 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 665, 664, 663, 662, 661, 660, 659, 658, - 0, 0, 607, 507, 355, 307, 351, 352, 359, 724, - 720, 725, 708, 711, 710, 686, 0, 315, 587, 422, - 470, 376, 652, 653, 0, 706, 259, 260, 261, 262, - 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, - 272, 273, 278, 279, 280, 281, 282, 283, 284, 285, - 655, 276, 277, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, - 0, 309, 712, 713, 714, 715, 716, 0, 0, 310, - 311, 312, 0, 0, 274, 275, 302, 498, 303, 304, - 305, 306, 0, 0, 537, 538, 539, 562, 0, 540, - 522, 586, 386, 316, 502, 529, 722, 0, 0, 0, - 0, 0, 0, 0, 637, 648, 682, 0, 694, 695, - 697, 699, 698, 701, 495, 496, 709, 0, 0, 703, - 704, 705, 702, 426, 482, 503, 489, 0, 728, 577, - 578, 729, 690, 317, 453, 0, 0, 592, 626, 615, - 700, 580, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 369, 0, 0, 421, 630, 611, 622, 612, - 597, 598, 599, 606, 381, 600, 601, 602, 572, 603, - 573, 604, 605, 0, 629, 579, 491, 437, 0, 646, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1461, 0, 0, - 245, 0, 0, 863, 873, 0, 0, 337, 246, 574, - 696, 576, 575, 864, 0, 865, 869, 872, 868, 866, - 867, 0, 340, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 492, 521, 0, 534, 0, 406, 407, 0, - 870, 0, 0, 0, 0, 0, 324, 499, 518, 338, - 486, 532, 343, 494, 511, 333, 452, 483, 0, 0, - 326, 516, 493, 434, 325, 0, 477, 366, 383, 363, - 450, 871, 0, 515, 545, 362, 535, 0, 526, 328, - 0, 525, 449, 512, 517, 435, 428, 0, 327, 514, - 433, 427, 412, 373, 561, 413, 414, 387, 464, 425, - 465, 388, 439, 438, 440, 389, 390, 391, 392, 393, - 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, - 556, 557, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 689, 0, 0, - 693, 0, 528, 0, 0, 0, 0, 0, 0, 497, - 0, 0, 415, 0, 0, 0, 546, 0, 480, 455, - 731, 0, 0, 478, 423, 513, 466, 519, 500, 527, - 472, 467, 318, 501, 365, 436, 334, 336, 721, 367, - 370, 374, 375, 445, 446, 460, 485, 504, 505, 506, - 364, 348, 479, 349, 384, 350, 319, 356, 354, 357, - 487, 358, 321, 461, 510, 0, 380, 475, 431, 322, - 430, 462, 509, 508, 335, 536, 543, 544, 634, 0, - 549, 732, 733, 734, 558, 0, 468, 331, 330, 0, - 0, 0, 360, 463, 344, 346, 347, 345, 458, 459, - 563, 564, 565, 567, 0, 568, 569, 0, 0, 0, - 0, 570, 635, 651, 619, 588, 551, 643, 585, 589, - 590, 401, 402, 403, 654, 0, 0, 0, 542, 416, - 417, 0, 372, 371, 432, 323, 0, 0, 409, 400, - 469, 329, 368, 411, 405, 418, 419, 420, 378, 313, - 314, 727, 361, 451, 656, 691, 692, 581, 0, 644, - 582, 591, 353, 616, 628, 627, 447, 541, 0, 639, - 642, 571, 726, 0, 636, 650, 730, 649, 723, 457, - 0, 484, 647, 594, 0, 640, 613, 614, 0, 641, - 609, 645, 0, 583, 0, 552, 555, 584, 669, 670, - 671, 320, 554, 673, 674, 675, 676, 677, 678, 679, - 672, 524, 617, 593, 620, 533, 596, 595, 0, 0, - 631, 550, 632, 633, 441, 442, 443, 444, 382, 657, - 342, 553, 471, 0, 618, 0, 0, 0, 0, 0, - 0, 0, 0, 623, 624, 621, 735, 0, 680, 681, - 0, 0, 547, 548, 377, 0, 566, 385, 341, 456, - 379, 531, 408, 0, 559, 625, 560, 473, 474, 683, - 688, 684, 685, 687, 707, 448, 399, 404, 488, 410, - 424, 476, 530, 454, 481, 339, 520, 490, 429, 610, - 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 670, 669, 668, 667, + 666, 665, 664, 663, 0, 0, 612, 512, 355, 307, + 351, 352, 359, 729, 725, 730, 713, 716, 715, 691, + 0, 315, 592, 427, 475, 376, 657, 658, 0, 711, + 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, + 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, + 282, 283, 284, 285, 660, 276, 277, 286, 287, 288, + 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, + 299, 0, 0, 0, 0, 309, 717, 718, 719, 720, + 721, 0, 0, 310, 311, 312, 0, 0, 274, 275, + 302, 503, 303, 304, 305, 306, 0, 0, 542, 543, + 544, 567, 0, 545, 527, 591, 386, 316, 507, 534, + 727, 0, 0, 0, 0, 0, 0, 0, 642, 653, + 687, 0, 699, 700, 702, 704, 703, 706, 500, 501, + 714, 0, 0, 708, 709, 710, 707, 431, 487, 508, + 494, 0, 733, 582, 583, 734, 695, 317, 183, 223, + 182, 214, 184, 0, 0, 0, 0, 0, 0, 458, + 759, 0, 597, 631, 620, 705, 585, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, + 426, 635, 616, 627, 617, 602, 603, 604, 611, 381, + 605, 606, 607, 577, 608, 578, 609, 610, 0, 634, + 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 766, 0, 0, 0, 0, 0, + 0, 0, 765, 0, 0, 245, 0, 0, 0, 0, + 0, 0, 337, 246, 579, 701, 581, 580, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 497, 526, 0, + 539, 0, 407, 408, 0, 0, 0, 0, 0, 0, + 0, 324, 504, 523, 338, 491, 537, 343, 499, 516, + 333, 457, 488, 0, 0, 326, 521, 498, 439, 325, + 0, 482, 366, 383, 363, 455, 0, 0, 520, 550, + 362, 540, 0, 531, 328, 0, 530, 454, 517, 522, + 440, 433, 0, 327, 519, 438, 432, 413, 373, 566, + 414, 415, 416, 417, 418, 419, 387, 469, 430, 470, + 388, 444, 443, 445, 389, 390, 391, 392, 393, 394, + 395, 396, 397, 398, 0, 0, 0, 0, 0, 561, + 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 763, 764, 0, 694, 0, 0, 698, + 0, 533, 0, 0, 0, 0, 0, 0, 502, 0, + 0, 420, 0, 0, 0, 551, 0, 485, 460, 736, + 0, 0, 483, 428, 518, 471, 524, 505, 532, 477, + 472, 318, 506, 365, 441, 334, 336, 726, 367, 370, + 374, 375, 450, 451, 465, 490, 509, 510, 511, 364, + 348, 484, 349, 384, 350, 319, 356, 354, 357, 492, + 358, 321, 466, 515, 0, 380, 480, 436, 322, 435, + 467, 514, 513, 335, 541, 548, 549, 639, 0, 554, + 737, 738, 739, 563, 0, 473, 331, 330, 0, 0, + 0, 360, 468, 344, 346, 347, 345, 463, 464, 568, + 569, 570, 572, 0, 573, 574, 0, 0, 0, 0, + 575, 640, 656, 624, 593, 556, 648, 590, 594, 595, + 401, 402, 403, 404, 659, 0, 0, 0, 547, 421, + 422, 0, 372, 371, 437, 323, 0, 0, 410, 400, + 474, 329, 368, 412, 406, 423, 424, 425, 378, 313, + 314, 732, 361, 456, 661, 696, 697, 586, 0, 649, + 587, 596, 353, 621, 633, 632, 452, 546, 0, 644, + 647, 576, 731, 0, 641, 655, 735, 654, 728, 462, + 0, 489, 652, 599, 0, 645, 618, 619, 0, 646, + 614, 650, 0, 588, 0, 557, 560, 589, 674, 675, + 676, 320, 559, 678, 679, 680, 681, 682, 683, 684, + 677, 529, 622, 598, 625, 538, 601, 600, 0, 0, + 636, 555, 637, 638, 446, 447, 448, 449, 760, 762, + 342, 558, 476, 774, 623, 0, 0, 0, 0, 0, + 0, 0, 0, 628, 629, 626, 740, 0, 685, 686, + 0, 0, 552, 553, 377, 0, 571, 385, 341, 461, + 379, 536, 409, 0, 564, 630, 565, 478, 479, 688, + 693, 689, 690, 692, 712, 453, 399, 405, 493, 411, + 429, 481, 535, 459, 486, 339, 525, 495, 434, 615, + 643, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 665, 664, - 663, 662, 661, 660, 659, 658, 0, 0, 607, 507, - 355, 307, 351, 352, 359, 724, 720, 725, 708, 711, - 710, 686, 0, 315, 587, 422, 470, 376, 652, 653, - 0, 706, 259, 260, 261, 262, 263, 264, 265, 266, + 0, 0, 0, 0, 0, 0, 0, 0, 670, 669, + 668, 667, 666, 665, 664, 663, 0, 0, 612, 512, + 355, 307, 351, 352, 359, 729, 725, 730, 713, 716, + 715, 691, 0, 315, 592, 427, 475, 376, 657, 658, + 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, - 280, 281, 282, 283, 284, 285, 655, 276, 277, 286, + 280, 281, 282, 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 298, 299, 0, 0, 0, 0, 309, 712, 713, - 714, 715, 716, 0, 0, 310, 311, 312, 0, 0, - 274, 275, 302, 498, 303, 304, 305, 306, 0, 0, - 537, 538, 539, 562, 0, 540, 522, 586, 386, 316, - 502, 529, 722, 0, 0, 0, 0, 0, 0, 0, - 637, 648, 682, 0, 694, 695, 697, 699, 698, 701, - 495, 496, 709, 0, 0, 703, 704, 705, 702, 426, - 482, 503, 489, 0, 728, 577, 578, 729, 690, 317, - 183, 223, 182, 214, 184, 0, 0, 0, 0, 0, - 0, 453, 754, 0, 592, 626, 615, 700, 580, 0, + 297, 298, 299, 0, 0, 0, 0, 309, 717, 718, + 719, 720, 721, 0, 0, 310, 311, 312, 0, 0, + 274, 275, 302, 503, 303, 304, 305, 306, 0, 0, + 542, 543, 544, 567, 0, 545, 527, 591, 386, 316, + 507, 534, 727, 0, 0, 0, 0, 0, 0, 0, + 642, 653, 687, 0, 699, 700, 702, 704, 703, 706, + 500, 501, 714, 0, 0, 708, 709, 710, 707, 431, + 487, 508, 494, 0, 733, 582, 583, 734, 695, 317, + 458, 0, 0, 597, 631, 620, 705, 585, 0, 1257, + 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, + 0, 426, 635, 616, 627, 617, 602, 603, 604, 611, + 381, 605, 606, 607, 577, 608, 578, 609, 610, 0, + 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, + 0, 0, 0, 337, 246, 579, 701, 581, 580, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 497, 526, + 0, 539, 0, 2894, 2895, 1238, 0, 0, 0, 0, + 0, 0, 324, 504, 523, 338, 491, 537, 343, 499, + 516, 333, 457, 488, 0, 0, 2888, 2891, 2892, 2893, + 2896, 0, 2901, 2897, 2898, 2899, 2900, 0, 0, 2884, + 2885, 2886, 2887, 1236, 2864, 2889, 0, 2865, 454, 2866, + 2867, 2868, 2869, 1240, 2870, 2871, 2872, 2873, 2874, 2881, + 2882, 2875, 2876, 2877, 2878, 2879, 2880, 2902, 2903, 2904, + 2905, 2906, 2907, 2908, 2909, 2911, 2910, 2912, 2913, 2914, + 2915, 2916, 2917, 2918, 2919, 1268, 1270, 1272, 1274, 1277, + 561, 562, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 694, 0, 0, + 698, 0, 533, 0, 0, 0, 0, 0, 0, 502, + 0, 0, 420, 0, 0, 0, 2883, 0, 485, 460, + 736, 0, 0, 483, 428, 518, 471, 524, 505, 532, + 477, 472, 318, 506, 365, 441, 334, 336, 726, 367, + 370, 374, 375, 450, 451, 465, 490, 509, 510, 511, + 364, 348, 484, 349, 384, 350, 319, 356, 354, 357, + 492, 358, 321, 466, 515, 0, 380, 480, 436, 322, + 435, 467, 514, 513, 335, 541, 548, 549, 639, 0, + 554, 737, 738, 739, 563, 0, 473, 331, 330, 0, + 0, 0, 360, 468, 344, 346, 347, 345, 463, 464, + 568, 569, 570, 572, 0, 573, 574, 0, 0, 0, + 0, 575, 640, 656, 624, 593, 556, 648, 590, 594, + 595, 401, 402, 403, 404, 659, 0, 0, 0, 547, + 421, 422, 0, 372, 371, 437, 323, 0, 0, 410, + 400, 474, 329, 368, 412, 406, 423, 424, 425, 378, + 313, 314, 732, 361, 456, 661, 696, 697, 586, 0, + 649, 587, 596, 353, 621, 633, 632, 452, 546, 0, + 644, 647, 576, 731, 0, 641, 655, 735, 654, 728, + 462, 0, 489, 652, 599, 0, 645, 618, 619, 0, + 646, 614, 650, 0, 588, 0, 557, 560, 589, 674, + 675, 676, 320, 559, 678, 679, 680, 681, 682, 683, + 684, 677, 529, 622, 598, 625, 538, 601, 600, 0, + 0, 636, 555, 637, 638, 446, 447, 448, 449, 382, + 662, 342, 558, 476, 0, 623, 0, 0, 0, 0, + 0, 0, 0, 0, 628, 629, 626, 740, 0, 685, + 686, 0, 0, 552, 553, 377, 0, 571, 385, 341, + 461, 379, 536, 409, 0, 564, 630, 565, 478, 479, + 688, 693, 689, 690, 692, 712, 453, 399, 405, 493, + 411, 429, 481, 535, 459, 486, 339, 525, 495, 434, + 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 670, + 669, 668, 667, 666, 665, 664, 663, 0, 0, 612, + 512, 355, 307, 351, 352, 359, 729, 725, 730, 713, + 716, 715, 691, 0, 315, 2890, 427, 475, 376, 657, + 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, + 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, + 279, 280, 281, 282, 283, 284, 285, 660, 276, 277, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 298, 299, 0, 0, 0, 0, 309, 717, + 718, 719, 720, 721, 0, 0, 310, 311, 312, 0, + 0, 274, 275, 302, 503, 303, 304, 305, 306, 0, + 0, 542, 543, 544, 567, 0, 545, 527, 591, 386, + 316, 507, 534, 727, 0, 0, 0, 0, 0, 0, + 0, 642, 653, 687, 0, 699, 700, 702, 704, 703, + 706, 500, 501, 714, 0, 0, 708, 709, 710, 707, + 431, 487, 508, 494, 0, 733, 582, 583, 734, 695, + 2863, 458, 0, 0, 597, 631, 620, 705, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, - 0, 0, 421, 630, 611, 622, 612, 597, 598, 599, - 606, 381, 600, 601, 602, 572, 603, 573, 604, 605, - 0, 629, 579, 491, 437, 0, 646, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 761, 0, 0, 0, - 0, 0, 0, 0, 760, 0, 0, 245, 0, 0, - 0, 0, 0, 0, 337, 246, 574, 696, 576, 575, + 0, 0, 426, 635, 616, 627, 617, 602, 603, 604, + 611, 381, 605, 606, 607, 577, 608, 578, 609, 610, + 0, 634, 584, 496, 442, 0, 651, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, + 0, 0, 0, 0, 337, 246, 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, + 2710, 2713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, + 526, 0, 539, 0, 407, 408, 0, 0, 0, 0, + 0, 0, 0, 324, 504, 523, 338, 491, 537, 343, + 499, 516, 333, 457, 488, 0, 0, 326, 521, 498, + 439, 325, 0, 482, 366, 383, 363, 455, 0, 0, + 520, 550, 362, 540, 0, 531, 328, 0, 530, 454, + 517, 522, 440, 433, 0, 327, 519, 438, 432, 413, + 373, 566, 414, 415, 416, 417, 418, 419, 387, 469, + 430, 470, 388, 444, 443, 445, 389, 390, 391, 392, + 393, 394, 395, 396, 397, 398, 0, 0, 0, 0, + 0, 561, 562, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 694, 0, + 0, 698, 2714, 533, 0, 0, 0, 2709, 0, 2708, + 502, 2706, 2711, 420, 0, 0, 0, 551, 0, 485, + 460, 736, 0, 0, 483, 428, 518, 471, 524, 505, + 532, 477, 472, 318, 506, 365, 441, 334, 336, 726, + 367, 370, 374, 375, 450, 451, 465, 490, 509, 510, + 511, 364, 348, 484, 349, 384, 350, 319, 356, 354, + 357, 492, 358, 321, 466, 515, 2712, 380, 480, 436, + 322, 435, 467, 514, 513, 335, 541, 548, 549, 639, + 0, 554, 737, 738, 739, 563, 0, 473, 331, 330, + 0, 0, 0, 360, 468, 344, 346, 347, 345, 463, + 464, 568, 569, 570, 572, 0, 573, 574, 0, 0, + 0, 0, 575, 640, 656, 624, 593, 556, 648, 590, + 594, 595, 401, 402, 403, 404, 659, 0, 0, 0, + 547, 421, 422, 0, 372, 371, 437, 323, 0, 0, + 410, 400, 474, 329, 368, 412, 406, 423, 424, 425, + 378, 313, 314, 732, 361, 456, 661, 696, 697, 586, + 0, 649, 587, 596, 353, 621, 633, 632, 452, 546, + 0, 644, 647, 576, 731, 0, 641, 655, 735, 654, + 728, 462, 0, 489, 652, 599, 0, 645, 618, 619, + 0, 646, 614, 650, 0, 588, 0, 557, 560, 589, + 674, 675, 676, 320, 559, 678, 679, 680, 681, 682, + 683, 684, 677, 529, 622, 598, 625, 538, 601, 600, + 0, 0, 636, 555, 637, 638, 446, 447, 448, 449, + 382, 662, 342, 558, 476, 0, 623, 0, 0, 0, + 0, 0, 0, 0, 0, 628, 629, 626, 740, 0, + 685, 686, 0, 0, 552, 553, 377, 0, 571, 385, + 341, 461, 379, 536, 409, 0, 564, 630, 565, 478, + 479, 688, 693, 689, 690, 692, 712, 453, 399, 405, + 493, 411, 429, 481, 535, 459, 486, 339, 525, 495, + 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, - 521, 0, 534, 0, 406, 407, 0, 0, 0, 0, - 0, 0, 0, 324, 499, 518, 338, 486, 532, 343, - 494, 511, 333, 452, 483, 0, 0, 326, 516, 493, - 434, 325, 0, 477, 366, 383, 363, 450, 0, 0, - 515, 545, 362, 535, 0, 526, 328, 0, 525, 449, - 512, 517, 435, 428, 0, 327, 514, 433, 427, 412, - 373, 561, 413, 414, 387, 464, 425, 465, 388, 439, - 438, 440, 389, 390, 391, 392, 393, 394, 395, 396, - 397, 398, 0, 0, 0, 0, 0, 556, 557, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 758, 759, 0, 689, 0, 0, 693, 0, 528, - 0, 0, 0, 0, 0, 0, 497, 0, 0, 415, - 0, 0, 0, 546, 0, 480, 455, 731, 0, 0, - 478, 423, 513, 466, 519, 500, 527, 472, 467, 318, - 501, 365, 436, 334, 336, 721, 367, 370, 374, 375, - 445, 446, 460, 485, 504, 505, 506, 364, 348, 479, - 349, 384, 350, 319, 356, 354, 357, 487, 358, 321, - 461, 510, 0, 380, 475, 431, 322, 430, 462, 509, - 508, 335, 536, 543, 544, 634, 0, 549, 732, 733, - 734, 558, 0, 468, 331, 330, 0, 0, 0, 360, - 463, 344, 346, 347, 345, 458, 459, 563, 564, 565, - 567, 0, 568, 569, 0, 0, 0, 0, 570, 635, - 651, 619, 588, 551, 643, 585, 589, 590, 401, 402, - 403, 654, 0, 0, 0, 542, 416, 417, 0, 372, - 371, 432, 323, 0, 0, 409, 400, 469, 329, 368, - 411, 405, 418, 419, 420, 378, 313, 314, 727, 361, - 451, 656, 691, 692, 581, 0, 644, 582, 591, 353, - 616, 628, 627, 447, 541, 0, 639, 642, 571, 726, - 0, 636, 650, 730, 649, 723, 457, 0, 484, 647, - 594, 0, 640, 613, 614, 0, 641, 609, 645, 0, - 583, 0, 552, 555, 584, 669, 670, 671, 320, 554, - 673, 674, 675, 676, 677, 678, 679, 672, 524, 617, - 593, 620, 533, 596, 595, 0, 0, 631, 550, 632, - 633, 441, 442, 443, 444, 755, 757, 342, 553, 471, - 769, 618, 0, 0, 0, 0, 0, 0, 0, 0, - 623, 624, 621, 735, 0, 680, 681, 0, 0, 547, - 548, 377, 0, 566, 385, 341, 456, 379, 531, 408, - 0, 559, 625, 560, 473, 474, 683, 688, 684, 685, - 687, 707, 448, 399, 404, 488, 410, 424, 476, 530, - 454, 481, 339, 520, 490, 429, 610, 638, 0, 0, - 0, 0, 0, 0, 0, 0, 71, 0, 0, 300, - 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 665, 664, 663, 662, 661, - 660, 659, 658, 0, 0, 607, 507, 355, 307, 351, - 352, 359, 724, 720, 725, 708, 711, 710, 686, 0, - 315, 587, 422, 470, 376, 652, 653, 0, 706, 259, - 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, - 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, - 283, 284, 285, 655, 276, 277, 286, 287, 288, 289, - 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, - 0, 0, 0, 0, 309, 712, 713, 714, 715, 716, - 0, 0, 310, 311, 312, 0, 0, 274, 275, 302, - 498, 303, 304, 305, 306, 0, 0, 537, 538, 539, - 562, 0, 540, 522, 586, 386, 316, 502, 529, 722, - 0, 0, 0, 0, 0, 0, 0, 637, 648, 682, - 0, 694, 695, 697, 699, 698, 701, 495, 496, 709, - 0, 0, 703, 704, 705, 702, 426, 482, 503, 489, - 0, 728, 577, 578, 729, 690, 317, 453, 0, 0, - 592, 626, 615, 700, 580, 0, 1247, 0, 0, 0, - 0, 0, 0, 0, 0, 369, 0, 0, 421, 630, - 611, 622, 612, 597, 598, 599, 606, 381, 600, 601, - 602, 572, 603, 573, 604, 605, 0, 629, 579, 491, - 437, 0, 646, 0, 0, 0, 0, 0, 0, 0, + 670, 669, 668, 667, 666, 665, 664, 663, 0, 0, + 612, 512, 355, 307, 351, 352, 359, 729, 725, 730, + 713, 716, 715, 691, 0, 315, 592, 427, 475, 376, + 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, + 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, + 278, 279, 280, 281, 282, 283, 284, 285, 660, 276, + 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, + 295, 296, 297, 298, 299, 0, 0, 0, 0, 309, + 717, 718, 719, 720, 721, 0, 0, 310, 311, 312, + 0, 0, 274, 275, 302, 503, 303, 304, 305, 306, + 0, 0, 542, 543, 544, 567, 0, 545, 527, 591, + 386, 316, 507, 534, 727, 0, 0, 0, 0, 0, + 0, 0, 642, 653, 687, 0, 699, 700, 702, 704, + 703, 706, 500, 501, 714, 0, 0, 708, 709, 710, + 707, 431, 487, 508, 494, 0, 733, 582, 583, 734, + 695, 317, 458, 0, 0, 597, 631, 620, 705, 585, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 369, 0, 0, 426, 635, 616, 627, 617, 602, 603, + 604, 611, 381, 605, 606, 607, 577, 608, 578, 609, + 610, 0, 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, - 337, 246, 574, 696, 576, 575, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, + 0, 0, 0, 0, 0, 337, 246, 579, 701, 581, + 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 340, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 492, 521, 0, 534, 0, - 2873, 2874, 1232, 0, 0, 0, 0, 0, 0, 324, - 499, 518, 338, 486, 532, 343, 494, 511, 333, 452, - 483, 0, 0, 2867, 2870, 2871, 2872, 2875, 0, 2880, - 2876, 2877, 2878, 2879, 0, 0, 2863, 2864, 2865, 2866, - 1230, 2847, 2868, 0, 2848, 449, 2849, 2850, 2851, 2852, - 1234, 2853, 2854, 2855, 2856, 2857, 2860, 2861, 2858, 2859, - 2881, 2882, 2883, 2884, 2885, 2886, 2887, 2888, 2890, 2889, - 2891, 2892, 2893, 2894, 2895, 2896, 2897, 2898, 1258, 1260, - 1262, 1264, 1267, 556, 557, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 689, 0, 0, 693, 0, 528, 0, 0, 0, 0, - 0, 0, 497, 0, 0, 415, 0, 0, 0, 2862, - 0, 480, 455, 731, 0, 0, 478, 423, 513, 466, - 519, 500, 527, 472, 467, 318, 501, 365, 436, 334, - 336, 721, 367, 370, 374, 375, 445, 446, 460, 485, - 504, 505, 506, 364, 348, 479, 349, 384, 350, 319, - 356, 354, 357, 487, 358, 321, 461, 510, 0, 380, - 475, 431, 322, 430, 462, 509, 508, 335, 536, 543, - 544, 634, 0, 549, 732, 733, 734, 558, 0, 468, - 331, 330, 0, 0, 0, 360, 463, 344, 346, 347, - 345, 458, 459, 563, 564, 565, 567, 0, 568, 569, - 0, 0, 0, 0, 570, 635, 651, 619, 588, 551, - 643, 585, 589, 590, 401, 402, 403, 654, 0, 0, - 0, 542, 416, 417, 0, 372, 371, 432, 323, 0, - 0, 409, 400, 469, 329, 368, 411, 405, 418, 419, - 420, 378, 313, 314, 727, 361, 451, 656, 691, 692, - 581, 0, 644, 582, 591, 353, 616, 628, 627, 447, - 541, 0, 639, 642, 571, 726, 0, 636, 650, 730, - 649, 723, 457, 0, 484, 647, 594, 0, 640, 613, - 614, 0, 641, 609, 645, 0, 583, 0, 552, 555, - 584, 669, 670, 671, 320, 554, 673, 674, 675, 676, - 677, 678, 679, 672, 524, 617, 593, 620, 533, 596, - 595, 0, 0, 631, 550, 632, 633, 441, 442, 443, - 444, 382, 657, 342, 553, 471, 0, 618, 0, 0, - 0, 0, 0, 0, 0, 0, 623, 624, 621, 735, - 0, 680, 681, 0, 0, 547, 548, 377, 0, 566, - 385, 341, 456, 379, 531, 408, 0, 559, 625, 560, - 473, 474, 683, 688, 684, 685, 687, 707, 448, 399, - 404, 488, 410, 424, 476, 530, 454, 481, 339, 520, - 490, 429, 610, 638, 0, 0, 0, 0, 0, 0, + 497, 526, 0, 539, 0, 407, 408, 0, 0, 0, + 0, 0, 0, 0, 324, 504, 523, 338, 491, 537, + 343, 499, 516, 333, 457, 488, 0, 0, 326, 521, + 498, 439, 325, 0, 482, 366, 383, 363, 455, 0, + 0, 520, 550, 362, 540, 0, 531, 328, 0, 530, + 454, 517, 522, 440, 433, 0, 327, 519, 438, 432, + 413, 373, 566, 414, 415, 416, 417, 418, 419, 387, + 469, 430, 470, 388, 444, 443, 445, 389, 390, 391, + 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, + 0, 0, 561, 562, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, + 0, 0, 698, 2730, 533, 0, 0, 0, 2736, 2733, + 2735, 502, 0, 2734, 420, 0, 0, 0, 551, 0, + 485, 460, 736, 0, 2728, 483, 428, 518, 471, 524, + 505, 532, 477, 472, 318, 506, 365, 441, 334, 336, + 726, 367, 370, 374, 375, 450, 451, 465, 490, 509, + 510, 511, 364, 348, 484, 349, 384, 350, 319, 356, + 354, 357, 492, 358, 321, 466, 515, 0, 380, 480, + 436, 322, 435, 467, 514, 513, 335, 541, 548, 549, + 639, 0, 554, 737, 738, 739, 563, 0, 473, 331, + 330, 0, 0, 0, 360, 468, 344, 346, 347, 345, + 463, 464, 568, 569, 570, 572, 0, 573, 574, 0, + 0, 0, 0, 575, 640, 656, 624, 593, 556, 648, + 590, 594, 595, 401, 402, 403, 404, 659, 0, 0, + 0, 547, 421, 422, 0, 372, 371, 437, 323, 0, + 0, 410, 400, 474, 329, 368, 412, 406, 423, 424, + 425, 378, 313, 314, 732, 361, 456, 661, 696, 697, + 586, 0, 649, 587, 596, 353, 621, 633, 632, 452, + 546, 0, 644, 647, 576, 731, 0, 641, 655, 735, + 654, 728, 462, 0, 489, 652, 599, 0, 645, 618, + 619, 0, 646, 614, 650, 0, 588, 0, 557, 560, + 589, 674, 675, 676, 320, 559, 678, 679, 680, 681, + 682, 683, 684, 677, 529, 622, 598, 625, 538, 601, + 600, 0, 0, 636, 555, 637, 638, 446, 447, 448, + 449, 382, 662, 342, 558, 476, 0, 623, 0, 0, + 0, 0, 0, 0, 0, 0, 628, 629, 626, 740, + 0, 685, 686, 0, 0, 552, 553, 377, 0, 571, + 385, 341, 461, 379, 536, 409, 0, 564, 630, 565, + 478, 479, 688, 693, 689, 690, 692, 712, 453, 399, + 405, 493, 411, 429, 481, 535, 459, 486, 339, 525, + 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 665, 664, 663, 662, 661, 660, 659, 658, 0, - 0, 607, 507, 355, 307, 351, 352, 359, 724, 720, - 725, 708, 711, 710, 686, 0, 315, 2869, 422, 470, - 376, 652, 653, 0, 706, 259, 260, 261, 262, 263, + 0, 670, 669, 668, 667, 666, 665, 664, 663, 0, + 0, 612, 512, 355, 307, 351, 352, 359, 729, 725, + 730, 713, 716, 715, 691, 0, 315, 592, 427, 475, + 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, - 273, 278, 279, 280, 281, 282, 283, 284, 285, 655, + 273, 278, 279, 280, 281, 282, 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, 0, - 309, 712, 713, 714, 715, 716, 0, 0, 310, 311, - 312, 0, 0, 274, 275, 302, 498, 303, 304, 305, - 306, 0, 0, 537, 538, 539, 562, 0, 540, 522, - 586, 386, 316, 502, 529, 722, 0, 0, 0, 0, - 0, 0, 0, 637, 648, 682, 0, 694, 695, 697, - 699, 698, 701, 495, 496, 709, 0, 0, 703, 704, - 705, 702, 426, 482, 503, 489, 0, 728, 577, 578, - 729, 690, 2846, 453, 0, 0, 592, 626, 615, 700, - 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 369, 0, 0, 421, 630, 611, 622, 612, 597, - 598, 599, 606, 381, 600, 601, 602, 572, 603, 573, - 604, 605, 0, 629, 579, 491, 437, 0, 646, 0, + 309, 717, 718, 719, 720, 721, 0, 0, 310, 311, + 312, 0, 0, 274, 275, 302, 503, 303, 304, 305, + 306, 0, 0, 542, 543, 544, 567, 0, 545, 527, + 591, 386, 316, 507, 534, 727, 0, 0, 0, 0, + 0, 0, 0, 642, 653, 687, 0, 699, 700, 702, + 704, 703, 706, 500, 501, 714, 0, 0, 708, 709, + 710, 707, 431, 487, 508, 494, 0, 733, 582, 583, + 734, 695, 317, 458, 0, 0, 597, 631, 620, 705, + 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 369, 0, 0, 426, 635, 616, 627, 617, 602, + 603, 604, 611, 381, 605, 606, 607, 577, 608, 578, + 609, 610, 0, 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, - 0, 0, 0, 0, 0, 0, 337, 246, 574, 696, - 576, 575, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 340, 2693, 2696, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 492, 521, 0, 534, 0, 406, 407, 0, 0, - 0, 0, 0, 0, 0, 324, 499, 518, 338, 486, - 532, 343, 494, 511, 333, 452, 483, 0, 0, 326, - 516, 493, 434, 325, 0, 477, 366, 383, 363, 450, - 0, 0, 515, 545, 362, 535, 0, 526, 328, 0, - 525, 449, 512, 517, 435, 428, 0, 327, 514, 433, - 427, 412, 373, 561, 413, 414, 387, 464, 425, 465, - 388, 439, 438, 440, 389, 390, 391, 392, 393, 394, - 395, 396, 397, 398, 0, 0, 0, 0, 0, 556, - 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 689, 0, 0, 693, - 2697, 528, 0, 0, 0, 2692, 0, 2691, 497, 2689, - 2694, 415, 0, 0, 0, 546, 0, 480, 455, 731, - 0, 0, 478, 423, 513, 466, 519, 500, 527, 472, - 467, 318, 501, 365, 436, 334, 336, 721, 367, 370, - 374, 375, 445, 446, 460, 485, 504, 505, 506, 364, - 348, 479, 349, 384, 350, 319, 356, 354, 357, 487, - 358, 321, 461, 510, 2695, 380, 475, 431, 322, 430, - 462, 509, 508, 335, 536, 543, 544, 634, 0, 549, - 732, 733, 734, 558, 0, 468, 331, 330, 0, 0, - 0, 360, 463, 344, 346, 347, 345, 458, 459, 563, - 564, 565, 567, 0, 568, 569, 0, 0, 0, 0, - 570, 635, 651, 619, 588, 551, 643, 585, 589, 590, - 401, 402, 403, 654, 0, 0, 0, 542, 416, 417, - 0, 372, 371, 432, 323, 0, 0, 409, 400, 469, - 329, 368, 411, 405, 418, 419, 420, 378, 313, 314, - 727, 361, 451, 656, 691, 692, 581, 0, 644, 582, - 591, 353, 616, 628, 627, 447, 541, 0, 639, 642, - 571, 726, 0, 636, 650, 730, 649, 723, 457, 0, - 484, 647, 594, 0, 640, 613, 614, 0, 641, 609, - 645, 0, 583, 0, 552, 555, 584, 669, 670, 671, - 320, 554, 673, 674, 675, 676, 677, 678, 679, 672, - 524, 617, 593, 620, 533, 596, 595, 0, 0, 631, - 550, 632, 633, 441, 442, 443, 444, 382, 657, 342, - 553, 471, 0, 618, 0, 0, 0, 0, 0, 0, - 0, 0, 623, 624, 621, 735, 0, 680, 681, 0, - 0, 547, 548, 377, 0, 566, 385, 341, 456, 379, - 531, 408, 0, 559, 625, 560, 473, 474, 683, 688, - 684, 685, 687, 707, 448, 399, 404, 488, 410, 424, - 476, 530, 454, 481, 339, 520, 490, 429, 610, 638, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 665, 664, 663, - 662, 661, 660, 659, 658, 0, 0, 607, 507, 355, - 307, 351, 352, 359, 724, 720, 725, 708, 711, 710, - 686, 0, 315, 587, 422, 470, 376, 652, 653, 0, - 706, 259, 260, 261, 262, 263, 264, 265, 266, 308, - 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, - 281, 282, 283, 284, 285, 655, 276, 277, 286, 287, - 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, - 298, 299, 0, 0, 0, 0, 309, 712, 713, 714, - 715, 716, 0, 0, 310, 311, 312, 0, 0, 274, - 275, 302, 498, 303, 304, 305, 306, 0, 0, 537, - 538, 539, 562, 0, 540, 522, 586, 386, 316, 502, - 529, 722, 0, 0, 0, 0, 0, 0, 0, 637, - 648, 682, 0, 694, 695, 697, 699, 698, 701, 495, - 496, 709, 0, 0, 703, 704, 705, 702, 426, 482, - 503, 489, 0, 728, 577, 578, 729, 690, 317, 453, - 0, 0, 592, 626, 615, 700, 580, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, - 421, 630, 611, 622, 612, 597, 598, 599, 606, 381, - 600, 601, 602, 572, 603, 573, 604, 605, 0, 629, - 579, 491, 437, 0, 646, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 337, 246, 579, 701, + 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 340, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, - 0, 0, 337, 246, 574, 696, 576, 575, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 340, 0, 2714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 497, 526, 0, 539, 0, 407, 408, 0, 0, + 0, 0, 0, 0, 0, 324, 504, 523, 338, 491, + 537, 343, 499, 516, 333, 457, 488, 0, 0, 326, + 521, 498, 439, 325, 0, 482, 366, 383, 363, 455, + 0, 0, 520, 550, 362, 540, 0, 531, 328, 0, + 530, 454, 517, 522, 440, 433, 0, 327, 519, 438, + 432, 413, 373, 566, 414, 415, 416, 417, 418, 419, + 387, 469, 430, 470, 388, 444, 443, 445, 389, 390, + 391, 392, 393, 394, 395, 396, 397, 398, 0, 0, + 0, 0, 0, 561, 562, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 694, 0, 0, 698, 2730, 533, 0, 0, 0, 2736, + 2733, 2735, 502, 0, 2734, 420, 0, 0, 0, 551, + 0, 485, 460, 736, 0, 0, 483, 428, 518, 471, + 524, 505, 532, 477, 472, 318, 506, 365, 441, 334, + 336, 726, 367, 370, 374, 375, 450, 451, 465, 490, + 509, 510, 511, 364, 348, 484, 349, 384, 350, 319, + 356, 354, 357, 492, 358, 321, 466, 515, 0, 380, + 480, 436, 322, 435, 467, 514, 513, 335, 541, 548, + 549, 639, 0, 554, 737, 738, 739, 563, 0, 473, + 331, 330, 0, 0, 0, 360, 468, 344, 346, 347, + 345, 463, 464, 568, 569, 570, 572, 0, 573, 574, + 0, 0, 0, 0, 575, 640, 656, 624, 593, 556, + 648, 590, 594, 595, 401, 402, 403, 404, 659, 0, + 0, 0, 547, 421, 422, 0, 372, 371, 437, 323, + 0, 0, 410, 400, 474, 329, 368, 412, 406, 423, + 424, 425, 378, 313, 314, 732, 361, 456, 661, 696, + 697, 586, 0, 649, 587, 596, 353, 621, 633, 632, + 452, 546, 0, 644, 647, 576, 731, 0, 641, 655, + 735, 654, 728, 462, 0, 489, 652, 599, 0, 645, + 618, 619, 0, 646, 614, 650, 0, 588, 0, 557, + 560, 589, 674, 675, 676, 320, 559, 678, 679, 680, + 681, 682, 683, 684, 677, 529, 622, 598, 625, 538, + 601, 600, 0, 0, 636, 555, 637, 638, 446, 447, + 448, 449, 382, 662, 342, 558, 476, 0, 623, 0, + 0, 0, 0, 0, 0, 0, 0, 628, 629, 626, + 740, 0, 685, 686, 0, 0, 552, 553, 377, 0, + 571, 385, 341, 461, 379, 536, 409, 0, 564, 630, + 565, 478, 479, 688, 693, 689, 690, 692, 712, 453, + 399, 405, 493, 411, 429, 481, 535, 459, 486, 339, + 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 492, 521, 0, - 534, 0, 406, 407, 0, 0, 0, 0, 0, 0, - 0, 324, 499, 518, 338, 486, 532, 343, 494, 511, - 333, 452, 483, 0, 0, 326, 516, 493, 434, 325, - 0, 477, 366, 383, 363, 450, 0, 0, 515, 545, - 362, 535, 0, 526, 328, 0, 525, 449, 512, 517, - 435, 428, 0, 327, 514, 433, 427, 412, 373, 561, - 413, 414, 387, 464, 425, 465, 388, 439, 438, 440, - 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, - 0, 0, 0, 0, 0, 556, 557, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 689, 0, 0, 693, 2713, 528, 0, 0, - 0, 2719, 2716, 2718, 497, 0, 2717, 415, 0, 0, - 0, 546, 0, 480, 455, 731, 0, 2711, 478, 423, - 513, 466, 519, 500, 527, 472, 467, 318, 501, 365, - 436, 334, 336, 721, 367, 370, 374, 375, 445, 446, - 460, 485, 504, 505, 506, 364, 348, 479, 349, 384, - 350, 319, 356, 354, 357, 487, 358, 321, 461, 510, - 0, 380, 475, 431, 322, 430, 462, 509, 508, 335, - 536, 543, 544, 634, 0, 549, 732, 733, 734, 558, - 0, 468, 331, 330, 0, 0, 0, 360, 463, 344, - 346, 347, 345, 458, 459, 563, 564, 565, 567, 0, - 568, 569, 0, 0, 0, 0, 570, 635, 651, 619, - 588, 551, 643, 585, 589, 590, 401, 402, 403, 654, - 0, 0, 0, 542, 416, 417, 0, 372, 371, 432, - 323, 0, 0, 409, 400, 469, 329, 368, 411, 405, - 418, 419, 420, 378, 313, 314, 727, 361, 451, 656, - 691, 692, 581, 0, 644, 582, 591, 353, 616, 628, - 627, 447, 541, 0, 639, 642, 571, 726, 0, 636, - 650, 730, 649, 723, 457, 0, 484, 647, 594, 0, - 640, 613, 614, 0, 641, 609, 645, 0, 583, 0, - 552, 555, 584, 669, 670, 671, 320, 554, 673, 674, - 675, 676, 677, 678, 679, 672, 524, 617, 593, 620, - 533, 596, 595, 0, 0, 631, 550, 632, 633, 441, - 442, 443, 444, 382, 657, 342, 553, 471, 0, 618, - 0, 0, 0, 0, 0, 0, 0, 0, 623, 624, - 621, 735, 0, 680, 681, 0, 0, 547, 548, 377, - 0, 566, 385, 341, 456, 379, 531, 408, 0, 559, - 625, 560, 473, 474, 683, 688, 684, 685, 687, 707, - 448, 399, 404, 488, 410, 424, 476, 530, 454, 481, - 339, 520, 490, 429, 610, 638, 0, 0, 0, 0, + 0, 0, 670, 669, 668, 667, 666, 665, 664, 663, + 0, 0, 612, 512, 355, 307, 351, 352, 359, 729, + 725, 730, 713, 716, 715, 691, 0, 315, 592, 427, + 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, + 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, + 272, 273, 278, 279, 280, 281, 282, 283, 284, 285, + 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, + 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, + 0, 309, 717, 718, 719, 720, 721, 0, 0, 310, + 311, 312, 0, 0, 274, 275, 302, 503, 303, 304, + 305, 306, 0, 0, 542, 543, 544, 567, 0, 545, + 527, 591, 386, 316, 507, 534, 727, 0, 0, 0, + 0, 0, 0, 0, 642, 653, 687, 0, 699, 700, + 702, 704, 703, 706, 500, 501, 714, 0, 0, 708, + 709, 710, 707, 431, 487, 508, 494, 0, 733, 582, + 583, 734, 695, 317, 458, 0, 0, 597, 631, 620, + 705, 585, 0, 0, 0, 0, 0, 2382, 0, 0, + 0, 0, 369, 0, 0, 426, 635, 616, 627, 617, + 602, 603, 604, 611, 381, 605, 606, 607, 577, 608, + 578, 609, 610, 0, 634, 584, 496, 442, 0, 651, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 245, 0, 0, 2383, 0, 0, 0, 337, 246, 579, + 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 340, 0, 0, 1395, 1396, 1397, 1394, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 497, 526, 0, 539, 0, 407, 408, 0, + 0, 0, 0, 0, 0, 0, 324, 504, 523, 338, + 491, 537, 343, 499, 516, 333, 457, 488, 0, 0, + 326, 521, 498, 439, 325, 0, 482, 366, 383, 363, + 455, 0, 0, 520, 550, 362, 540, 0, 531, 328, + 0, 530, 454, 517, 522, 440, 433, 0, 327, 519, + 438, 432, 413, 373, 566, 414, 415, 416, 417, 418, + 419, 387, 469, 430, 470, 388, 444, 443, 445, 389, + 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, + 0, 0, 0, 0, 561, 562, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 694, 0, 0, 698, 0, 533, 0, 0, 0, + 0, 0, 0, 502, 0, 0, 420, 0, 0, 0, + 551, 0, 485, 460, 736, 0, 0, 483, 428, 518, + 471, 524, 505, 532, 477, 472, 318, 506, 365, 441, + 334, 336, 726, 367, 370, 374, 375, 450, 451, 465, + 490, 509, 510, 511, 364, 348, 484, 349, 384, 350, + 319, 356, 354, 357, 492, 358, 321, 466, 515, 0, + 380, 480, 436, 322, 435, 467, 514, 513, 335, 541, + 548, 549, 639, 0, 554, 737, 738, 739, 563, 0, + 473, 331, 330, 0, 0, 0, 360, 468, 344, 346, + 347, 345, 463, 464, 568, 569, 570, 572, 0, 573, + 574, 0, 0, 0, 0, 575, 640, 656, 624, 593, + 556, 648, 590, 594, 595, 401, 402, 403, 404, 659, + 0, 0, 0, 547, 421, 422, 0, 372, 371, 437, + 323, 0, 0, 410, 400, 474, 329, 368, 412, 406, + 423, 424, 425, 378, 313, 314, 732, 361, 456, 661, + 696, 697, 586, 0, 649, 587, 596, 353, 621, 633, + 632, 452, 546, 0, 644, 647, 576, 731, 0, 641, + 655, 735, 654, 728, 462, 0, 489, 652, 599, 0, + 645, 618, 619, 0, 646, 614, 650, 0, 588, 0, + 557, 560, 589, 674, 675, 676, 320, 559, 678, 679, + 680, 681, 682, 683, 684, 677, 529, 622, 598, 625, + 538, 601, 600, 0, 0, 636, 555, 637, 638, 446, + 447, 448, 449, 382, 662, 342, 558, 476, 0, 623, + 0, 0, 0, 0, 0, 0, 0, 0, 628, 629, + 626, 740, 0, 685, 686, 0, 0, 552, 553, 377, + 0, 571, 385, 341, 461, 379, 536, 409, 0, 564, + 630, 565, 478, 479, 688, 693, 689, 690, 692, 712, + 453, 399, 405, 493, 411, 429, 481, 535, 459, 486, + 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 665, 664, 663, 662, 661, 660, 659, - 658, 0, 0, 607, 507, 355, 307, 351, 352, 359, - 724, 720, 725, 708, 711, 710, 686, 0, 315, 587, - 422, 470, 376, 652, 653, 0, 706, 259, 260, 261, + 0, 0, 0, 670, 669, 668, 667, 666, 665, 664, + 663, 0, 0, 612, 512, 355, 307, 351, 352, 359, + 729, 725, 730, 713, 716, 715, 691, 0, 315, 592, + 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, 283, 284, - 285, 655, 276, 277, 286, 287, 288, 289, 290, 291, + 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, 0, - 0, 0, 309, 712, 713, 714, 715, 716, 0, 0, - 310, 311, 312, 0, 0, 274, 275, 302, 498, 303, - 304, 305, 306, 0, 0, 537, 538, 539, 562, 0, - 540, 522, 586, 386, 316, 502, 529, 722, 0, 0, - 0, 0, 0, 0, 0, 637, 648, 682, 0, 694, - 695, 697, 699, 698, 701, 495, 496, 709, 0, 0, - 703, 704, 705, 702, 426, 482, 503, 489, 0, 728, - 577, 578, 729, 690, 317, 453, 0, 0, 592, 626, - 615, 700, 580, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 369, 0, 0, 421, 630, 611, 622, - 612, 597, 598, 599, 606, 381, 600, 601, 602, 572, - 603, 573, 604, 605, 0, 629, 579, 491, 437, 0, - 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 245, 0, 0, 0, 0, 0, 0, 337, 246, - 574, 696, 576, 575, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 340, 0, 2714, 0, 0, 0, 0, + 0, 0, 309, 717, 718, 719, 720, 721, 0, 0, + 310, 311, 312, 0, 0, 274, 275, 302, 503, 303, + 304, 305, 306, 0, 0, 542, 543, 544, 567, 0, + 545, 527, 591, 386, 316, 507, 534, 727, 0, 0, + 0, 0, 0, 0, 0, 642, 653, 687, 0, 699, + 700, 702, 704, 703, 706, 500, 501, 714, 0, 0, + 708, 709, 710, 707, 431, 487, 508, 494, 0, 733, + 582, 583, 734, 695, 317, 183, 223, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 458, 0, 0, 597, + 631, 620, 705, 585, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 369, 0, 0, 426, 635, 616, + 627, 617, 602, 603, 604, 611, 381, 605, 606, 607, + 577, 608, 578, 609, 610, 153, 634, 584, 496, 442, + 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, + 2968, 0, 245, 0, 0, 0, 0, 0, 0, 337, + 246, 579, 701, 581, 580, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 492, 521, 0, 534, 0, 406, 407, - 0, 0, 0, 0, 0, 0, 0, 324, 499, 518, - 338, 486, 532, 343, 494, 511, 333, 452, 483, 0, - 0, 326, 516, 493, 434, 325, 0, 477, 366, 383, - 363, 450, 0, 0, 515, 545, 362, 535, 0, 526, - 328, 0, 525, 449, 512, 517, 435, 428, 0, 327, - 514, 433, 427, 412, 373, 561, 413, 414, 387, 464, - 425, 465, 388, 439, 438, 440, 389, 390, 391, 392, - 393, 394, 395, 396, 397, 398, 0, 0, 0, 0, - 0, 556, 557, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 689, 0, - 0, 693, 2713, 528, 0, 0, 0, 2719, 2716, 2718, - 497, 0, 2717, 415, 0, 0, 0, 546, 0, 480, - 455, 731, 0, 0, 478, 423, 513, 466, 519, 500, - 527, 472, 467, 318, 501, 365, 436, 334, 336, 721, - 367, 370, 374, 375, 445, 446, 460, 485, 504, 505, - 506, 364, 348, 479, 349, 384, 350, 319, 356, 354, - 357, 487, 358, 321, 461, 510, 0, 380, 475, 431, - 322, 430, 462, 509, 508, 335, 536, 543, 544, 634, - 0, 549, 732, 733, 734, 558, 0, 468, 331, 330, - 0, 0, 0, 360, 463, 344, 346, 347, 345, 458, - 459, 563, 564, 565, 567, 0, 568, 569, 0, 0, - 0, 0, 570, 635, 651, 619, 588, 551, 643, 585, - 589, 590, 401, 402, 403, 654, 0, 0, 0, 542, - 416, 417, 0, 372, 371, 432, 323, 0, 0, 409, - 400, 469, 329, 368, 411, 405, 418, 419, 420, 378, - 313, 314, 727, 361, 451, 656, 691, 692, 581, 0, - 644, 582, 591, 353, 616, 628, 627, 447, 541, 0, - 639, 642, 571, 726, 0, 636, 650, 730, 649, 723, - 457, 0, 484, 647, 594, 0, 640, 613, 614, 0, - 641, 609, 645, 0, 583, 0, 552, 555, 584, 669, - 670, 671, 320, 554, 673, 674, 675, 676, 677, 678, - 679, 672, 524, 617, 593, 620, 533, 596, 595, 0, - 0, 631, 550, 632, 633, 441, 442, 443, 444, 382, - 657, 342, 553, 471, 0, 618, 0, 0, 0, 0, - 0, 0, 0, 0, 623, 624, 621, 735, 0, 680, - 681, 0, 0, 547, 548, 377, 0, 566, 385, 341, - 456, 379, 531, 408, 0, 559, 625, 560, 473, 474, - 683, 688, 684, 685, 687, 707, 448, 399, 404, 488, - 410, 424, 476, 530, 454, 481, 339, 520, 490, 429, - 610, 638, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, - 664, 663, 662, 661, 660, 659, 658, 0, 0, 607, - 507, 355, 307, 351, 352, 359, 724, 720, 725, 708, - 711, 710, 686, 0, 315, 587, 422, 470, 376, 652, - 653, 0, 706, 259, 260, 261, 262, 263, 264, 265, - 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, - 279, 280, 281, 282, 283, 284, 285, 655, 276, 277, - 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, - 296, 297, 298, 299, 0, 0, 0, 0, 309, 712, - 713, 714, 715, 716, 0, 0, 310, 311, 312, 0, - 0, 274, 275, 302, 498, 303, 304, 305, 306, 0, - 0, 537, 538, 539, 562, 0, 540, 522, 586, 386, - 316, 502, 529, 722, 0, 0, 0, 0, 0, 0, - 0, 637, 648, 682, 0, 694, 695, 697, 699, 698, - 701, 495, 496, 709, 0, 0, 703, 704, 705, 702, - 426, 482, 503, 489, 0, 728, 577, 578, 729, 690, - 317, 453, 0, 0, 592, 626, 615, 700, 580, 0, - 0, 0, 0, 0, 2366, 0, 0, 0, 0, 369, - 0, 0, 421, 630, 611, 622, 612, 597, 598, 599, - 606, 381, 600, 601, 602, 572, 603, 573, 604, 605, - 0, 629, 579, 491, 437, 0, 646, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, - 2367, 0, 0, 0, 337, 246, 574, 696, 576, 575, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, - 0, 0, 1385, 1386, 1387, 1384, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, - 521, 0, 534, 0, 406, 407, 0, 0, 0, 0, - 0, 0, 0, 324, 499, 518, 338, 486, 532, 343, - 494, 511, 333, 452, 483, 0, 0, 326, 516, 493, - 434, 325, 0, 477, 366, 383, 363, 450, 0, 0, - 515, 545, 362, 535, 0, 526, 328, 0, 525, 449, - 512, 517, 435, 428, 0, 327, 514, 433, 427, 412, - 373, 561, 413, 414, 387, 464, 425, 465, 388, 439, - 438, 440, 389, 390, 391, 392, 393, 394, 395, 396, - 397, 398, 0, 0, 0, 0, 0, 556, 557, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 689, 0, 0, 693, 0, 528, - 0, 0, 0, 0, 0, 0, 497, 0, 0, 415, - 0, 0, 0, 546, 0, 480, 455, 731, 0, 0, - 478, 423, 513, 466, 519, 500, 527, 472, 467, 318, - 501, 365, 436, 334, 336, 721, 367, 370, 374, 375, - 445, 446, 460, 485, 504, 505, 506, 364, 348, 479, - 349, 384, 350, 319, 356, 354, 357, 487, 358, 321, - 461, 510, 0, 380, 475, 431, 322, 430, 462, 509, - 508, 335, 536, 543, 544, 634, 0, 549, 732, 733, - 734, 558, 0, 468, 331, 330, 0, 0, 0, 360, - 463, 344, 346, 347, 345, 458, 459, 563, 564, 565, - 567, 0, 568, 569, 0, 0, 0, 0, 570, 635, - 651, 619, 588, 551, 643, 585, 589, 590, 401, 402, - 403, 654, 0, 0, 0, 542, 416, 417, 0, 372, - 371, 432, 323, 0, 0, 409, 400, 469, 329, 368, - 411, 405, 418, 419, 420, 378, 313, 314, 727, 361, - 451, 656, 691, 692, 581, 0, 644, 582, 591, 353, - 616, 628, 627, 447, 541, 0, 639, 642, 571, 726, - 0, 636, 650, 730, 649, 723, 457, 0, 484, 647, - 594, 0, 640, 613, 614, 0, 641, 609, 645, 0, - 583, 0, 552, 555, 584, 669, 670, 671, 320, 554, - 673, 674, 675, 676, 677, 678, 679, 672, 524, 617, - 593, 620, 533, 596, 595, 0, 0, 631, 550, 632, - 633, 441, 442, 443, 444, 382, 657, 342, 553, 471, - 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, - 623, 624, 621, 735, 0, 680, 681, 0, 0, 547, - 548, 377, 0, 566, 385, 341, 456, 379, 531, 408, - 0, 559, 625, 560, 473, 474, 683, 688, 684, 685, - 687, 707, 448, 399, 404, 488, 410, 424, 476, 530, - 454, 481, 339, 520, 490, 429, 610, 638, 0, 0, + 0, 0, 0, 0, 497, 526, 0, 539, 0, 407, + 408, 0, 0, 0, 0, 0, 0, 0, 324, 504, + 523, 338, 491, 537, 343, 499, 516, 333, 457, 488, + 0, 0, 326, 521, 498, 439, 325, 0, 482, 366, + 383, 363, 455, 0, 0, 520, 550, 362, 540, 0, + 531, 328, 0, 530, 454, 517, 522, 440, 433, 0, + 327, 519, 438, 432, 413, 373, 566, 414, 415, 416, + 417, 418, 419, 387, 469, 430, 470, 388, 444, 443, + 445, 389, 390, 391, 392, 393, 394, 395, 396, 397, + 398, 0, 0, 0, 0, 0, 561, 562, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 694, 0, 0, 698, 0, 533, 0, + 0, 0, 0, 0, 0, 502, 0, 0, 420, 0, + 0, 0, 551, 0, 485, 460, 736, 0, 0, 483, + 428, 518, 471, 524, 505, 532, 477, 472, 318, 506, + 365, 441, 334, 336, 726, 367, 370, 374, 375, 450, + 451, 465, 490, 509, 510, 511, 364, 348, 484, 349, + 384, 350, 319, 356, 354, 357, 492, 358, 321, 466, + 515, 0, 380, 480, 436, 322, 435, 467, 514, 513, + 335, 541, 548, 549, 639, 0, 554, 737, 738, 739, + 563, 0, 473, 331, 330, 0, 0, 0, 360, 468, + 344, 346, 347, 345, 463, 464, 568, 569, 570, 572, + 0, 573, 574, 0, 0, 0, 0, 575, 640, 656, + 624, 593, 556, 648, 590, 594, 595, 401, 402, 403, + 404, 659, 0, 0, 0, 547, 421, 422, 0, 372, + 371, 437, 323, 0, 0, 410, 400, 474, 329, 368, + 412, 406, 423, 424, 425, 378, 313, 314, 732, 361, + 456, 661, 696, 697, 586, 0, 649, 587, 596, 353, + 621, 633, 632, 452, 546, 0, 644, 647, 576, 731, + 0, 641, 655, 735, 654, 728, 462, 0, 489, 652, + 599, 0, 645, 618, 619, 0, 646, 614, 650, 0, + 588, 0, 557, 560, 589, 674, 675, 676, 320, 559, + 678, 679, 680, 681, 682, 683, 684, 677, 529, 622, + 598, 625, 538, 601, 600, 0, 0, 636, 555, 637, + 638, 446, 447, 448, 449, 382, 662, 342, 558, 476, + 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, + 628, 629, 626, 740, 0, 685, 686, 0, 0, 552, + 553, 377, 0, 571, 385, 341, 461, 379, 536, 409, + 0, 564, 630, 565, 478, 479, 688, 693, 689, 690, + 692, 712, 453, 399, 405, 493, 411, 429, 481, 535, + 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 665, 664, 663, 662, 661, - 660, 659, 658, 0, 0, 607, 507, 355, 307, 351, - 352, 359, 724, 720, 725, 708, 711, 710, 686, 0, - 315, 587, 422, 470, 376, 652, 653, 0, 706, 259, + 0, 0, 0, 0, 0, 670, 669, 668, 667, 666, + 665, 664, 663, 0, 0, 612, 512, 355, 307, 351, + 352, 359, 729, 725, 730, 713, 716, 715, 691, 0, + 315, 592, 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, - 283, 284, 285, 655, 276, 277, 286, 287, 288, 289, + 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, - 0, 0, 0, 0, 309, 712, 713, 714, 715, 716, + 0, 0, 0, 0, 309, 717, 718, 719, 720, 721, 0, 0, 310, 311, 312, 0, 0, 274, 275, 302, - 498, 303, 304, 305, 306, 0, 0, 537, 538, 539, - 562, 0, 540, 522, 586, 386, 316, 502, 529, 722, - 0, 0, 0, 0, 0, 0, 0, 637, 648, 682, - 0, 694, 695, 697, 699, 698, 701, 495, 496, 709, - 0, 0, 703, 704, 705, 702, 426, 482, 503, 489, - 0, 728, 577, 578, 729, 690, 317, 183, 223, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 453, 0, - 0, 592, 626, 615, 700, 580, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 369, 0, 0, 421, - 630, 611, 622, 612, 597, 598, 599, 606, 381, 600, - 601, 602, 572, 603, 573, 604, 605, 153, 629, 579, - 491, 437, 0, 646, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 219, 2947, 0, 245, 0, 0, 0, 0, 0, - 0, 337, 246, 574, 696, 576, 575, 0, 0, 0, + 503, 303, 304, 305, 306, 0, 0, 542, 543, 544, + 567, 0, 545, 527, 591, 386, 316, 507, 534, 727, + 0, 0, 0, 0, 0, 0, 0, 642, 653, 687, + 0, 699, 700, 702, 704, 703, 706, 500, 501, 714, + 0, 0, 708, 709, 710, 707, 431, 487, 508, 494, + 0, 733, 582, 583, 734, 695, 317, 183, 223, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 458, 0, + 0, 597, 631, 620, 705, 585, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 369, 0, 0, 426, + 635, 616, 627, 617, 602, 603, 604, 611, 381, 605, + 606, 607, 577, 608, 578, 609, 610, 153, 634, 584, + 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 219, 2651, 0, 245, 0, 0, 0, 0, 0, + 0, 337, 246, 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 492, 521, 0, 534, - 0, 406, 407, 0, 0, 0, 0, 0, 0, 0, - 324, 499, 518, 338, 486, 532, 343, 494, 511, 333, - 452, 483, 0, 0, 326, 516, 493, 434, 325, 0, - 477, 366, 383, 363, 450, 0, 0, 515, 545, 362, - 535, 0, 526, 328, 0, 525, 449, 512, 517, 435, - 428, 0, 327, 514, 433, 427, 412, 373, 561, 413, - 414, 387, 464, 425, 465, 388, 439, 438, 440, 389, - 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, - 0, 0, 0, 0, 556, 557, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 689, 0, 0, 693, 0, 528, 0, 0, 0, - 0, 0, 0, 497, 0, 0, 415, 0, 0, 0, - 546, 0, 480, 455, 731, 0, 0, 478, 423, 513, - 466, 519, 500, 527, 472, 467, 318, 501, 365, 436, - 334, 336, 721, 367, 370, 374, 375, 445, 446, 460, - 485, 504, 505, 506, 364, 348, 479, 349, 384, 350, - 319, 356, 354, 357, 487, 358, 321, 461, 510, 0, - 380, 475, 431, 322, 430, 462, 509, 508, 335, 536, - 543, 544, 634, 0, 549, 732, 733, 734, 558, 0, - 468, 331, 330, 0, 0, 0, 360, 463, 344, 346, - 347, 345, 458, 459, 563, 564, 565, 567, 0, 568, - 569, 0, 0, 0, 0, 570, 635, 651, 619, 588, - 551, 643, 585, 589, 590, 401, 402, 403, 654, 0, - 0, 0, 542, 416, 417, 0, 372, 371, 432, 323, - 0, 0, 409, 400, 469, 329, 368, 411, 405, 418, - 419, 420, 378, 313, 314, 727, 361, 451, 656, 691, - 692, 581, 0, 644, 582, 591, 353, 616, 628, 627, - 447, 541, 0, 639, 642, 571, 726, 0, 636, 650, - 730, 649, 723, 457, 0, 484, 647, 594, 0, 640, - 613, 614, 0, 641, 609, 645, 0, 583, 0, 552, - 555, 584, 669, 670, 671, 320, 554, 673, 674, 675, - 676, 677, 678, 679, 672, 524, 617, 593, 620, 533, - 596, 595, 0, 0, 631, 550, 632, 633, 441, 442, - 443, 444, 382, 657, 342, 553, 471, 0, 618, 0, - 0, 0, 0, 0, 0, 0, 0, 623, 624, 621, - 735, 0, 680, 681, 0, 0, 547, 548, 377, 0, - 566, 385, 341, 456, 379, 531, 408, 0, 559, 625, - 560, 473, 474, 683, 688, 684, 685, 687, 707, 448, - 399, 404, 488, 410, 424, 476, 530, 454, 481, 339, - 520, 490, 429, 610, 638, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, + 0, 0, 0, 0, 0, 0, 497, 526, 0, 539, + 0, 407, 408, 0, 0, 0, 0, 0, 0, 0, + 324, 504, 523, 338, 491, 537, 343, 499, 516, 333, + 457, 488, 0, 0, 326, 521, 498, 439, 325, 0, + 482, 366, 383, 363, 455, 0, 0, 520, 550, 362, + 540, 0, 531, 328, 0, 530, 454, 517, 522, 440, + 433, 0, 327, 519, 438, 432, 413, 373, 566, 414, + 415, 416, 417, 418, 419, 387, 469, 430, 470, 388, + 444, 443, 445, 389, 390, 391, 392, 393, 394, 395, + 396, 397, 398, 0, 0, 0, 0, 0, 561, 562, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 694, 0, 0, 698, 0, + 533, 0, 0, 0, 0, 0, 0, 502, 0, 0, + 420, 0, 0, 0, 551, 0, 485, 460, 736, 0, + 0, 483, 428, 518, 471, 524, 505, 532, 477, 472, + 318, 506, 365, 441, 334, 336, 726, 367, 370, 374, + 375, 450, 451, 465, 490, 509, 510, 511, 364, 348, + 484, 349, 384, 350, 319, 356, 354, 357, 492, 358, + 321, 466, 515, 0, 380, 480, 436, 322, 435, 467, + 514, 513, 335, 541, 548, 549, 639, 0, 554, 737, + 738, 739, 563, 0, 473, 331, 330, 0, 0, 0, + 360, 468, 344, 346, 347, 345, 463, 464, 568, 569, + 570, 572, 0, 573, 574, 0, 0, 0, 0, 575, + 640, 656, 624, 593, 556, 648, 590, 594, 595, 401, + 402, 403, 404, 659, 0, 0, 0, 547, 421, 422, + 0, 372, 371, 437, 323, 0, 0, 410, 400, 474, + 329, 368, 412, 406, 423, 424, 425, 378, 313, 314, + 732, 361, 456, 661, 696, 697, 586, 0, 649, 587, + 596, 353, 621, 633, 632, 452, 546, 0, 644, 647, + 576, 731, 0, 641, 655, 735, 654, 728, 462, 0, + 489, 652, 599, 0, 645, 618, 619, 0, 646, 614, + 650, 0, 588, 0, 557, 560, 589, 674, 675, 676, + 320, 559, 678, 679, 680, 681, 682, 683, 684, 677, + 529, 622, 598, 625, 538, 601, 600, 0, 0, 636, + 555, 637, 638, 446, 447, 448, 449, 382, 662, 342, + 558, 476, 0, 623, 0, 0, 0, 0, 0, 0, + 0, 0, 628, 629, 626, 740, 0, 685, 686, 0, + 0, 552, 553, 377, 0, 571, 385, 341, 461, 379, + 536, 409, 0, 564, 630, 565, 478, 479, 688, 693, + 689, 690, 692, 712, 453, 399, 405, 493, 411, 429, + 481, 535, 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 665, 664, 663, 662, 661, 660, 659, 658, - 0, 0, 607, 507, 355, 307, 351, 352, 359, 724, - 720, 725, 708, 711, 710, 686, 0, 315, 587, 422, - 470, 376, 652, 653, 0, 706, 259, 260, 261, 262, - 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, - 272, 273, 278, 279, 280, 281, 282, 283, 284, 285, - 655, 276, 277, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, - 0, 309, 712, 713, 714, 715, 716, 0, 0, 310, - 311, 312, 0, 0, 274, 275, 302, 498, 303, 304, - 305, 306, 0, 0, 537, 538, 539, 562, 0, 540, - 522, 586, 386, 316, 502, 529, 722, 0, 0, 0, - 0, 0, 0, 0, 637, 648, 682, 0, 694, 695, - 697, 699, 698, 701, 495, 496, 709, 0, 0, 703, - 704, 705, 702, 426, 482, 503, 489, 0, 728, 577, - 578, 729, 690, 317, 183, 223, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 453, 0, 0, 592, 626, - 615, 700, 580, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 369, 0, 0, 421, 630, 611, 622, - 612, 597, 598, 599, 606, 381, 600, 601, 602, 572, - 603, 573, 604, 605, 153, 629, 579, 491, 437, 0, - 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 219, 2634, - 0, 245, 0, 0, 0, 0, 0, 0, 337, 246, - 574, 696, 576, 575, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, + 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 670, 669, 668, + 667, 666, 665, 664, 663, 0, 0, 612, 512, 355, + 307, 351, 352, 359, 729, 725, 730, 713, 716, 715, + 691, 0, 315, 592, 427, 475, 376, 657, 658, 0, + 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, + 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, + 281, 282, 283, 284, 285, 660, 276, 277, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 298, 299, 0, 0, 0, 0, 309, 717, 718, 719, + 720, 721, 0, 0, 310, 311, 312, 0, 0, 274, + 275, 302, 503, 303, 304, 305, 306, 0, 0, 542, + 543, 544, 567, 0, 545, 527, 591, 386, 316, 507, + 534, 727, 0, 0, 0, 0, 0, 0, 0, 642, + 653, 687, 0, 699, 700, 702, 704, 703, 706, 500, + 501, 714, 0, 0, 708, 709, 710, 707, 431, 487, + 508, 494, 0, 733, 582, 583, 734, 695, 317, 458, + 0, 0, 597, 631, 620, 705, 585, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 369, 1161, 0, + 426, 635, 616, 627, 617, 602, 603, 604, 611, 381, + 605, 606, 607, 577, 608, 578, 609, 610, 0, 634, + 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 245, 1168, 1169, 0, 0, + 0, 0, 337, 246, 579, 701, 581, 580, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1172, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 497, 526, 0, + 539, 0, 407, 408, 0, 0, 0, 0, 0, 0, + 0, 324, 504, 1155, 338, 491, 537, 343, 499, 516, + 333, 457, 488, 0, 0, 326, 521, 498, 439, 325, + 0, 482, 366, 383, 363, 455, 0, 0, 520, 550, + 362, 540, 1140, 531, 328, 1139, 530, 454, 517, 522, + 440, 433, 0, 327, 519, 438, 432, 413, 373, 566, + 414, 415, 416, 417, 418, 419, 387, 469, 430, 470, + 388, 444, 443, 445, 389, 390, 391, 392, 393, 394, + 395, 396, 397, 398, 0, 0, 0, 0, 0, 561, + 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 694, 0, 0, 698, + 0, 533, 0, 0, 0, 0, 0, 0, 502, 0, + 0, 420, 0, 0, 0, 551, 0, 485, 460, 736, + 0, 0, 483, 428, 518, 471, 524, 505, 532, 1159, + 472, 318, 506, 365, 441, 334, 336, 726, 367, 370, + 374, 375, 450, 451, 465, 490, 509, 510, 511, 364, + 348, 484, 349, 384, 350, 319, 356, 354, 357, 492, + 358, 321, 466, 515, 0, 380, 480, 436, 322, 435, + 467, 514, 513, 335, 541, 548, 549, 639, 0, 554, + 737, 738, 739, 563, 0, 473, 331, 330, 0, 0, + 0, 360, 468, 344, 346, 347, 345, 463, 464, 568, + 569, 570, 572, 0, 573, 574, 0, 0, 0, 0, + 575, 640, 656, 624, 593, 556, 648, 590, 594, 595, + 401, 402, 403, 404, 659, 0, 0, 0, 547, 421, + 422, 0, 372, 371, 437, 323, 0, 0, 410, 400, + 474, 329, 368, 412, 406, 423, 424, 425, 378, 313, + 314, 732, 361, 456, 661, 696, 697, 586, 0, 649, + 587, 596, 353, 621, 633, 632, 452, 546, 0, 644, + 647, 576, 731, 0, 641, 655, 735, 654, 728, 462, + 0, 489, 652, 599, 0, 645, 618, 619, 0, 646, + 614, 650, 0, 588, 0, 557, 560, 589, 674, 675, + 676, 320, 559, 678, 679, 680, 681, 682, 683, 1160, + 677, 529, 622, 598, 625, 538, 601, 600, 0, 0, + 636, 1163, 637, 638, 446, 447, 448, 449, 382, 662, + 1158, 558, 476, 0, 623, 0, 0, 0, 0, 0, + 0, 0, 0, 628, 629, 626, 740, 0, 685, 686, + 0, 0, 552, 553, 377, 0, 571, 385, 341, 461, + 379, 536, 409, 0, 564, 630, 565, 478, 479, 688, + 693, 689, 690, 692, 712, 1170, 1156, 1166, 1157, 411, + 429, 481, 535, 459, 486, 339, 525, 495, 1167, 615, + 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 670, 669, + 668, 667, 666, 665, 664, 663, 0, 0, 612, 512, + 355, 307, 351, 352, 359, 729, 725, 730, 713, 716, + 715, 691, 0, 315, 592, 427, 475, 376, 657, 658, + 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, + 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, + 280, 281, 282, 283, 284, 285, 660, 276, 277, 286, + 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, + 297, 298, 299, 0, 0, 0, 0, 309, 717, 718, + 719, 720, 721, 0, 0, 310, 311, 312, 0, 0, + 274, 275, 302, 503, 303, 304, 305, 306, 0, 0, + 542, 543, 544, 567, 0, 545, 527, 591, 386, 316, + 507, 534, 727, 0, 0, 0, 0, 0, 0, 0, + 642, 653, 687, 0, 699, 700, 702, 704, 703, 706, + 500, 501, 714, 0, 0, 708, 709, 710, 707, 1154, + 487, 508, 494, 0, 733, 582, 583, 734, 695, 317, + 183, 223, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 458, 0, 0, 597, 631, 620, 705, 585, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, + 0, 0, 426, 635, 616, 627, 617, 602, 603, 604, + 611, 381, 605, 606, 607, 577, 608, 578, 609, 610, + 153, 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 2308, 0, 0, 245, 0, 0, + 0, 0, 0, 0, 337, 246, 579, 701, 581, 580, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 492, 521, 0, 534, 0, 406, 407, - 0, 0, 0, 0, 0, 0, 0, 324, 499, 518, - 338, 486, 532, 343, 494, 511, 333, 452, 483, 0, - 0, 326, 516, 493, 434, 325, 0, 477, 366, 383, - 363, 450, 0, 0, 515, 545, 362, 535, 0, 526, - 328, 0, 525, 449, 512, 517, 435, 428, 0, 327, - 514, 433, 427, 412, 373, 561, 413, 414, 387, 464, - 425, 465, 388, 439, 438, 440, 389, 390, 391, 392, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, + 526, 0, 539, 0, 407, 408, 0, 0, 0, 0, + 0, 0, 0, 324, 504, 523, 338, 491, 537, 343, + 499, 516, 333, 457, 488, 0, 0, 326, 521, 498, + 439, 325, 0, 482, 366, 383, 363, 455, 0, 0, + 520, 550, 362, 540, 0, 531, 328, 0, 530, 454, + 517, 522, 440, 433, 0, 327, 519, 438, 432, 413, + 373, 566, 414, 415, 416, 417, 418, 419, 387, 469, + 430, 470, 388, 444, 443, 445, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, 0, - 0, 556, 557, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 689, 0, - 0, 693, 0, 528, 0, 0, 0, 0, 0, 0, - 497, 0, 0, 415, 0, 0, 0, 546, 0, 480, - 455, 731, 0, 0, 478, 423, 513, 466, 519, 500, - 527, 472, 467, 318, 501, 365, 436, 334, 336, 721, - 367, 370, 374, 375, 445, 446, 460, 485, 504, 505, - 506, 364, 348, 479, 349, 384, 350, 319, 356, 354, - 357, 487, 358, 321, 461, 510, 0, 380, 475, 431, - 322, 430, 462, 509, 508, 335, 536, 543, 544, 634, - 0, 549, 732, 733, 734, 558, 0, 468, 331, 330, - 0, 0, 0, 360, 463, 344, 346, 347, 345, 458, - 459, 563, 564, 565, 567, 0, 568, 569, 0, 0, - 0, 0, 570, 635, 651, 619, 588, 551, 643, 585, - 589, 590, 401, 402, 403, 654, 0, 0, 0, 542, - 416, 417, 0, 372, 371, 432, 323, 0, 0, 409, - 400, 469, 329, 368, 411, 405, 418, 419, 420, 378, - 313, 314, 727, 361, 451, 656, 691, 692, 581, 0, - 644, 582, 591, 353, 616, 628, 627, 447, 541, 0, - 639, 642, 571, 726, 0, 636, 650, 730, 649, 723, - 457, 0, 484, 647, 594, 0, 640, 613, 614, 0, - 641, 609, 645, 0, 583, 0, 552, 555, 584, 669, - 670, 671, 320, 554, 673, 674, 675, 676, 677, 678, - 679, 672, 524, 617, 593, 620, 533, 596, 595, 0, - 0, 631, 550, 632, 633, 441, 442, 443, 444, 382, - 657, 342, 553, 471, 0, 618, 0, 0, 0, 0, - 0, 0, 0, 0, 623, 624, 621, 735, 0, 680, - 681, 0, 0, 547, 548, 377, 0, 566, 385, 341, - 456, 379, 531, 408, 0, 559, 625, 560, 473, 474, - 683, 688, 684, 685, 687, 707, 448, 399, 404, 488, - 410, 424, 476, 530, 454, 481, 339, 520, 490, 429, - 610, 638, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, - 664, 663, 662, 661, 660, 659, 658, 0, 0, 607, - 507, 355, 307, 351, 352, 359, 724, 720, 725, 708, - 711, 710, 686, 0, 315, 587, 422, 470, 376, 652, - 653, 0, 706, 259, 260, 261, 262, 263, 264, 265, - 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, - 279, 280, 281, 282, 283, 284, 285, 655, 276, 277, - 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, - 296, 297, 298, 299, 0, 0, 0, 0, 309, 712, - 713, 714, 715, 716, 0, 0, 310, 311, 312, 0, - 0, 274, 275, 302, 498, 303, 304, 305, 306, 0, - 0, 537, 538, 539, 562, 0, 540, 522, 586, 386, - 316, 502, 529, 722, 0, 0, 0, 0, 0, 0, - 0, 637, 648, 682, 0, 694, 695, 697, 699, 698, - 701, 495, 496, 709, 0, 0, 703, 704, 705, 702, - 426, 482, 503, 489, 0, 728, 577, 578, 729, 690, - 317, 453, 0, 0, 592, 626, 615, 700, 580, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, - 1155, 0, 421, 630, 611, 622, 612, 597, 598, 599, - 606, 381, 600, 601, 602, 572, 603, 573, 604, 605, - 0, 629, 579, 491, 437, 0, 646, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 245, 1162, 1163, - 0, 0, 0, 0, 337, 246, 574, 696, 576, 575, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1166, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, - 521, 0, 534, 0, 406, 407, 0, 0, 0, 0, - 0, 0, 0, 324, 499, 1149, 338, 486, 532, 343, - 494, 511, 333, 452, 483, 0, 0, 326, 516, 493, - 434, 325, 0, 477, 366, 383, 363, 450, 0, 0, - 515, 545, 362, 535, 1134, 526, 328, 1133, 525, 449, - 512, 517, 435, 428, 0, 327, 514, 433, 427, 412, - 373, 561, 413, 414, 387, 464, 425, 465, 388, 439, - 438, 440, 389, 390, 391, 392, 393, 394, 395, 396, - 397, 398, 0, 0, 0, 0, 0, 556, 557, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 689, 0, 0, 693, 0, 528, - 0, 0, 0, 0, 0, 0, 497, 0, 0, 415, - 0, 0, 0, 546, 0, 480, 455, 731, 0, 0, - 478, 423, 513, 466, 519, 500, 527, 1153, 467, 318, - 501, 365, 436, 334, 336, 721, 367, 370, 374, 375, - 445, 446, 460, 485, 504, 505, 506, 364, 348, 479, - 349, 384, 350, 319, 356, 354, 357, 487, 358, 321, - 461, 510, 0, 380, 475, 431, 322, 430, 462, 509, - 508, 335, 536, 543, 544, 634, 0, 549, 732, 733, - 734, 558, 0, 468, 331, 330, 0, 0, 0, 360, - 463, 344, 346, 347, 345, 458, 459, 563, 564, 565, - 567, 0, 568, 569, 0, 0, 0, 0, 570, 635, - 651, 619, 588, 551, 643, 585, 589, 590, 401, 402, - 403, 654, 0, 0, 0, 542, 416, 417, 0, 372, - 371, 432, 323, 0, 0, 409, 400, 469, 329, 368, - 411, 405, 418, 419, 420, 378, 313, 314, 727, 361, - 451, 656, 691, 692, 581, 0, 644, 582, 591, 353, - 616, 628, 627, 447, 541, 0, 639, 642, 571, 726, - 0, 636, 650, 730, 649, 723, 457, 0, 484, 647, - 594, 0, 640, 613, 614, 0, 641, 609, 645, 0, - 583, 0, 552, 555, 584, 669, 670, 671, 320, 554, - 673, 674, 675, 676, 677, 678, 1154, 672, 524, 617, - 593, 620, 533, 596, 595, 0, 0, 631, 1157, 632, - 633, 441, 442, 443, 444, 382, 657, 1152, 553, 471, - 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, - 623, 624, 621, 735, 0, 680, 681, 0, 0, 547, - 548, 377, 0, 566, 385, 341, 456, 379, 531, 408, - 0, 559, 625, 560, 473, 474, 683, 688, 684, 685, - 687, 707, 1164, 1150, 1160, 1151, 410, 424, 476, 530, - 454, 481, 339, 520, 490, 1161, 610, 638, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, - 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 665, 664, 663, 662, 661, - 660, 659, 658, 0, 0, 607, 507, 355, 307, 351, - 352, 359, 724, 720, 725, 708, 711, 710, 686, 0, - 315, 587, 422, 470, 376, 652, 653, 0, 706, 259, - 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, - 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, - 283, 284, 285, 655, 276, 277, 286, 287, 288, 289, - 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, - 0, 0, 0, 0, 309, 712, 713, 714, 715, 716, - 0, 0, 310, 311, 312, 0, 0, 274, 275, 302, - 498, 303, 304, 305, 306, 0, 0, 537, 538, 539, - 562, 0, 540, 522, 586, 386, 316, 502, 529, 722, - 0, 0, 0, 0, 0, 0, 0, 637, 648, 682, - 0, 694, 695, 697, 699, 698, 701, 495, 496, 709, - 0, 0, 703, 704, 705, 702, 1148, 482, 503, 489, - 0, 728, 577, 578, 729, 690, 317, 183, 223, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 453, 0, - 0, 592, 626, 615, 700, 580, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 369, 0, 0, 421, - 630, 611, 622, 612, 597, 598, 599, 606, 381, 600, - 601, 602, 572, 603, 573, 604, 605, 153, 629, 579, - 491, 437, 0, 646, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 2292, 0, 0, 245, 0, 0, 0, 0, 0, - 0, 337, 246, 574, 696, 576, 575, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, + 0, 561, 562, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 694, 0, + 0, 698, 0, 533, 0, 0, 0, 0, 0, 0, + 502, 0, 0, 420, 0, 0, 0, 551, 0, 485, + 460, 736, 0, 0, 483, 428, 518, 471, 524, 505, + 532, 477, 472, 318, 506, 365, 441, 334, 336, 726, + 367, 370, 374, 375, 450, 451, 465, 490, 509, 510, + 511, 364, 348, 484, 349, 384, 350, 319, 356, 354, + 357, 492, 358, 321, 466, 515, 0, 380, 480, 436, + 322, 435, 467, 514, 513, 335, 541, 548, 549, 639, + 0, 554, 737, 738, 739, 563, 0, 473, 331, 330, + 0, 0, 0, 360, 468, 344, 346, 347, 345, 463, + 464, 568, 569, 570, 572, 0, 573, 574, 0, 0, + 0, 0, 575, 640, 656, 624, 593, 556, 648, 590, + 594, 595, 401, 402, 403, 404, 659, 0, 0, 0, + 547, 421, 422, 0, 372, 371, 437, 323, 0, 0, + 410, 400, 474, 329, 368, 412, 406, 423, 424, 425, + 378, 313, 314, 732, 361, 456, 661, 696, 697, 586, + 0, 649, 587, 596, 353, 621, 633, 632, 452, 546, + 0, 644, 647, 576, 731, 0, 641, 655, 735, 654, + 728, 462, 0, 489, 652, 599, 0, 645, 618, 619, + 0, 646, 614, 650, 0, 588, 0, 557, 560, 589, + 674, 675, 676, 320, 559, 678, 679, 680, 681, 682, + 683, 684, 677, 529, 622, 598, 625, 538, 601, 600, + 0, 0, 636, 555, 637, 638, 446, 447, 448, 449, + 382, 662, 342, 558, 476, 0, 623, 0, 0, 0, + 0, 0, 0, 0, 0, 628, 629, 626, 740, 0, + 685, 686, 0, 0, 552, 553, 377, 0, 571, 385, + 341, 461, 379, 536, 409, 0, 564, 630, 565, 478, + 479, 688, 693, 689, 690, 692, 712, 453, 399, 405, + 493, 411, 429, 481, 535, 459, 486, 339, 525, 495, + 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 670, 669, 668, 667, 666, 665, 664, 663, 0, 0, + 612, 512, 355, 307, 351, 352, 359, 729, 725, 730, + 713, 716, 715, 691, 0, 315, 592, 427, 475, 376, + 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, + 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, + 278, 279, 280, 281, 282, 283, 284, 285, 660, 276, + 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, + 295, 296, 297, 298, 299, 0, 0, 0, 0, 309, + 717, 718, 719, 720, 721, 0, 0, 310, 311, 312, + 0, 0, 274, 275, 302, 503, 303, 304, 305, 306, + 0, 0, 542, 543, 544, 567, 0, 545, 527, 591, + 386, 316, 507, 534, 727, 0, 0, 0, 0, 0, + 0, 0, 642, 653, 687, 0, 699, 700, 702, 704, + 703, 706, 500, 501, 714, 0, 0, 708, 709, 710, + 707, 431, 487, 508, 494, 0, 733, 582, 583, 734, + 695, 317, 458, 0, 0, 597, 631, 620, 705, 585, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 369, 0, 0, 426, 635, 616, 627, 617, 602, 603, + 604, 611, 381, 605, 606, 607, 577, 608, 578, 609, + 610, 0, 634, 584, 496, 442, 0, 651, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 245, 1168, + 1169, 0, 0, 0, 0, 337, 246, 579, 701, 581, + 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 492, 521, 0, 534, - 0, 406, 407, 0, 0, 0, 0, 0, 0, 0, - 324, 499, 518, 338, 486, 532, 343, 494, 511, 333, - 452, 483, 0, 0, 326, 516, 493, 434, 325, 0, - 477, 366, 383, 363, 450, 0, 0, 515, 545, 362, - 535, 0, 526, 328, 0, 525, 449, 512, 517, 435, - 428, 0, 327, 514, 433, 427, 412, 373, 561, 413, - 414, 387, 464, 425, 465, 388, 439, 438, 440, 389, - 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, - 0, 0, 0, 0, 556, 557, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 689, 0, 0, 693, 0, 528, 0, 0, 0, - 0, 0, 0, 497, 0, 0, 415, 0, 0, 0, - 546, 0, 480, 455, 731, 0, 0, 478, 423, 513, - 466, 519, 500, 527, 472, 467, 318, 501, 365, 436, - 334, 336, 721, 367, 370, 374, 375, 445, 446, 460, - 485, 504, 505, 506, 364, 348, 479, 349, 384, 350, - 319, 356, 354, 357, 487, 358, 321, 461, 510, 0, - 380, 475, 431, 322, 430, 462, 509, 508, 335, 536, - 543, 544, 634, 0, 549, 732, 733, 734, 558, 0, - 468, 331, 330, 0, 0, 0, 360, 463, 344, 346, - 347, 345, 458, 459, 563, 564, 565, 567, 0, 568, - 569, 0, 0, 0, 0, 570, 635, 651, 619, 588, - 551, 643, 585, 589, 590, 401, 402, 403, 654, 0, - 0, 0, 542, 416, 417, 0, 372, 371, 432, 323, - 0, 0, 409, 400, 469, 329, 368, 411, 405, 418, - 419, 420, 378, 313, 314, 727, 361, 451, 656, 691, - 692, 581, 0, 644, 582, 591, 353, 616, 628, 627, - 447, 541, 0, 639, 642, 571, 726, 0, 636, 650, - 730, 649, 723, 457, 0, 484, 647, 594, 0, 640, - 613, 614, 0, 641, 609, 645, 0, 583, 0, 552, - 555, 584, 669, 670, 671, 320, 554, 673, 674, 675, - 676, 677, 678, 679, 672, 524, 617, 593, 620, 533, - 596, 595, 0, 0, 631, 550, 632, 633, 441, 442, - 443, 444, 382, 657, 342, 553, 471, 0, 618, 0, - 0, 0, 0, 0, 0, 0, 0, 623, 624, 621, - 735, 0, 680, 681, 0, 0, 547, 548, 377, 0, - 566, 385, 341, 456, 379, 531, 408, 0, 559, 625, - 560, 473, 474, 683, 688, 684, 685, 687, 707, 448, - 399, 404, 488, 410, 424, 476, 530, 454, 481, 339, - 520, 490, 429, 610, 638, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 497, 526, 0, 539, 0, 407, 408, 0, 0, 0, + 0, 0, 0, 0, 324, 504, 523, 338, 491, 537, + 343, 499, 516, 333, 457, 488, 0, 0, 326, 521, + 498, 439, 325, 0, 482, 366, 383, 363, 455, 0, + 0, 520, 550, 362, 540, 1140, 531, 328, 1139, 530, + 454, 517, 522, 440, 433, 0, 327, 519, 438, 432, + 413, 373, 566, 414, 415, 416, 417, 418, 419, 387, + 469, 430, 470, 388, 444, 443, 445, 389, 390, 391, + 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, + 0, 0, 561, 562, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, + 0, 0, 698, 0, 533, 0, 0, 0, 0, 0, + 0, 502, 0, 0, 420, 0, 0, 0, 551, 0, + 485, 460, 736, 0, 0, 483, 428, 518, 471, 524, + 505, 532, 477, 472, 318, 506, 365, 441, 334, 336, + 726, 367, 370, 374, 375, 450, 451, 465, 490, 509, + 510, 511, 364, 348, 484, 349, 384, 350, 319, 356, + 354, 357, 492, 358, 321, 466, 515, 0, 380, 480, + 436, 322, 435, 467, 514, 513, 335, 541, 548, 549, + 639, 0, 554, 737, 738, 739, 563, 0, 473, 331, + 330, 0, 0, 0, 360, 468, 344, 346, 347, 345, + 463, 464, 568, 569, 570, 572, 0, 573, 574, 0, + 0, 0, 0, 575, 640, 656, 624, 593, 556, 648, + 590, 594, 595, 401, 402, 403, 404, 659, 0, 0, + 0, 547, 421, 422, 0, 372, 371, 437, 323, 0, + 0, 410, 400, 474, 329, 368, 412, 406, 423, 424, + 425, 378, 313, 314, 732, 361, 456, 661, 696, 697, + 586, 0, 649, 587, 596, 353, 621, 633, 632, 452, + 546, 0, 644, 647, 576, 731, 0, 641, 655, 735, + 654, 728, 462, 0, 489, 652, 599, 0, 645, 618, + 619, 0, 646, 614, 650, 0, 588, 0, 557, 560, + 589, 674, 675, 676, 320, 559, 678, 679, 680, 681, + 682, 683, 684, 677, 529, 622, 598, 625, 538, 601, + 600, 0, 0, 636, 555, 637, 638, 446, 447, 448, + 449, 382, 662, 342, 558, 476, 0, 623, 0, 0, + 0, 0, 0, 0, 0, 0, 628, 629, 626, 740, + 0, 685, 686, 0, 0, 552, 553, 377, 0, 571, + 385, 341, 461, 379, 536, 409, 0, 564, 630, 565, + 478, 479, 688, 693, 689, 690, 692, 712, 1170, 2329, + 1166, 2330, 411, 429, 481, 535, 459, 486, 339, 525, + 495, 1167, 615, 643, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 670, 669, 668, 667, 666, 665, 664, 663, 0, + 0, 612, 512, 355, 307, 351, 352, 359, 729, 725, + 730, 713, 716, 715, 691, 0, 315, 592, 427, 475, + 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, + 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, + 273, 278, 279, 280, 281, 282, 283, 284, 285, 660, + 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 298, 299, 0, 0, 0, 0, + 309, 717, 718, 719, 720, 721, 0, 0, 310, 311, + 312, 0, 0, 274, 275, 302, 503, 303, 304, 305, + 306, 0, 0, 542, 543, 544, 567, 0, 545, 527, + 591, 386, 316, 507, 534, 727, 0, 0, 0, 0, + 0, 0, 0, 642, 653, 687, 0, 699, 700, 702, + 704, 703, 706, 500, 501, 714, 0, 0, 708, 709, + 710, 707, 431, 487, 508, 494, 0, 733, 582, 583, + 734, 695, 317, 458, 0, 0, 597, 631, 620, 705, + 585, 0, 0, 3353, 0, 0, 0, 0, 0, 0, + 0, 369, 0, 0, 426, 635, 616, 627, 617, 602, + 603, 604, 611, 381, 605, 606, 607, 577, 608, 578, + 609, 610, 0, 634, 584, 496, 442, 0, 651, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, + 0, 0, 0, 0, 0, 0, 337, 246, 579, 701, + 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 497, 526, 0, 539, 0, 407, 408, 0, 0, + 0, 0, 0, 0, 0, 324, 504, 523, 338, 491, + 537, 343, 499, 516, 333, 457, 488, 0, 0, 326, + 521, 498, 439, 325, 0, 482, 366, 383, 363, 455, + 0, 0, 520, 550, 362, 540, 0, 531, 328, 0, + 530, 454, 517, 522, 440, 433, 0, 327, 519, 438, + 432, 413, 373, 566, 414, 415, 416, 417, 418, 419, + 387, 469, 430, 470, 388, 444, 443, 445, 389, 390, + 391, 392, 393, 394, 395, 396, 397, 398, 0, 0, + 0, 0, 0, 561, 562, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3356, 0, 0, 0, 0, 3355, + 694, 0, 0, 698, 0, 533, 0, 0, 0, 0, + 0, 0, 502, 0, 0, 420, 0, 0, 0, 551, + 0, 485, 460, 736, 0, 0, 483, 428, 518, 471, + 524, 505, 532, 477, 472, 318, 506, 365, 441, 334, + 336, 726, 367, 370, 374, 375, 450, 451, 465, 490, + 509, 510, 511, 364, 348, 484, 349, 384, 350, 319, + 356, 354, 357, 492, 358, 321, 466, 515, 0, 380, + 480, 436, 322, 435, 467, 514, 513, 335, 541, 548, + 549, 639, 0, 554, 737, 738, 739, 563, 0, 473, + 331, 330, 0, 0, 0, 360, 468, 344, 346, 347, + 345, 463, 464, 568, 569, 570, 572, 0, 573, 574, + 0, 0, 0, 0, 575, 640, 656, 624, 593, 556, + 648, 590, 594, 595, 401, 402, 403, 404, 659, 0, + 0, 0, 547, 421, 422, 0, 372, 371, 437, 323, + 0, 0, 410, 400, 474, 329, 368, 412, 406, 423, + 424, 425, 378, 313, 314, 732, 361, 456, 661, 696, + 697, 586, 0, 649, 587, 596, 353, 621, 633, 632, + 452, 546, 0, 644, 647, 576, 731, 0, 641, 655, + 735, 654, 728, 462, 0, 489, 652, 599, 0, 645, + 618, 619, 0, 646, 614, 650, 0, 588, 0, 557, + 560, 589, 674, 675, 676, 320, 559, 678, 679, 680, + 681, 682, 683, 684, 677, 529, 622, 598, 625, 538, + 601, 600, 0, 0, 636, 555, 637, 638, 446, 447, + 448, 449, 382, 662, 342, 558, 476, 0, 623, 0, + 0, 0, 0, 0, 0, 0, 0, 628, 629, 626, + 740, 0, 685, 686, 0, 0, 552, 553, 377, 0, + 571, 385, 341, 461, 379, 536, 409, 0, 564, 630, + 565, 478, 479, 688, 693, 689, 690, 692, 712, 453, + 399, 405, 493, 411, 429, 481, 535, 459, 486, 339, + 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 665, 664, 663, 662, 661, 660, 659, 658, - 0, 0, 607, 507, 355, 307, 351, 352, 359, 724, - 720, 725, 708, 711, 710, 686, 0, 315, 587, 422, - 470, 376, 652, 653, 0, 706, 259, 260, 261, 262, + 0, 0, 670, 669, 668, 667, 666, 665, 664, 663, + 0, 0, 612, 512, 355, 307, 351, 352, 359, 729, + 725, 730, 713, 716, 715, 691, 0, 315, 592, 427, + 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, 283, 284, 285, - 655, 276, 277, 286, 287, 288, 289, 290, 291, 292, + 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, - 0, 309, 712, 713, 714, 715, 716, 0, 0, 310, - 311, 312, 0, 0, 274, 275, 302, 498, 303, 304, - 305, 306, 0, 0, 537, 538, 539, 562, 0, 540, - 522, 586, 386, 316, 502, 529, 722, 0, 0, 0, - 0, 0, 0, 0, 637, 648, 682, 0, 694, 695, - 697, 699, 698, 701, 495, 496, 709, 0, 0, 703, - 704, 705, 702, 426, 482, 503, 489, 0, 728, 577, - 578, 729, 690, 317, 453, 0, 0, 592, 626, 615, - 700, 580, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 369, 0, 0, 421, 630, 611, 622, 612, - 597, 598, 599, 606, 381, 600, 601, 602, 572, 603, - 573, 604, 605, 0, 629, 579, 491, 437, 0, 646, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 1162, 1163, 0, 0, 0, 0, 337, 246, 574, - 696, 576, 575, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1166, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 492, 521, 0, 534, 0, 406, 407, 0, - 0, 0, 0, 0, 0, 0, 324, 499, 518, 338, - 486, 532, 343, 494, 511, 333, 452, 483, 0, 0, - 326, 516, 493, 434, 325, 0, 477, 366, 383, 363, - 450, 0, 0, 515, 545, 362, 535, 1134, 526, 328, - 1133, 525, 449, 512, 517, 435, 428, 0, 327, 514, - 433, 427, 412, 373, 561, 413, 414, 387, 464, 425, - 465, 388, 439, 438, 440, 389, 390, 391, 392, 393, - 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, - 556, 557, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 689, 0, 0, - 693, 0, 528, 0, 0, 0, 0, 0, 0, 497, - 0, 0, 415, 0, 0, 0, 546, 0, 480, 455, - 731, 0, 0, 478, 423, 513, 466, 519, 500, 527, - 472, 467, 318, 501, 365, 436, 334, 336, 721, 367, - 370, 374, 375, 445, 446, 460, 485, 504, 505, 506, - 364, 348, 479, 349, 384, 350, 319, 356, 354, 357, - 487, 358, 321, 461, 510, 0, 380, 475, 431, 322, - 430, 462, 509, 508, 335, 536, 543, 544, 634, 0, - 549, 732, 733, 734, 558, 0, 468, 331, 330, 0, - 0, 0, 360, 463, 344, 346, 347, 345, 458, 459, - 563, 564, 565, 567, 0, 568, 569, 0, 0, 0, - 0, 570, 635, 651, 619, 588, 551, 643, 585, 589, - 590, 401, 402, 403, 654, 0, 0, 0, 542, 416, - 417, 0, 372, 371, 432, 323, 0, 0, 409, 400, - 469, 329, 368, 411, 405, 418, 419, 420, 378, 313, - 314, 727, 361, 451, 656, 691, 692, 581, 0, 644, - 582, 591, 353, 616, 628, 627, 447, 541, 0, 639, - 642, 571, 726, 0, 636, 650, 730, 649, 723, 457, - 0, 484, 647, 594, 0, 640, 613, 614, 0, 641, - 609, 645, 0, 583, 0, 552, 555, 584, 669, 670, - 671, 320, 554, 673, 674, 675, 676, 677, 678, 679, - 672, 524, 617, 593, 620, 533, 596, 595, 0, 0, - 631, 550, 632, 633, 441, 442, 443, 444, 382, 657, - 342, 553, 471, 0, 618, 0, 0, 0, 0, 0, - 0, 0, 0, 623, 624, 621, 735, 0, 680, 681, - 0, 0, 547, 548, 377, 0, 566, 385, 341, 456, - 379, 531, 408, 0, 559, 625, 560, 473, 474, 683, - 688, 684, 685, 687, 707, 1164, 2313, 1160, 2314, 410, - 424, 476, 530, 454, 481, 339, 520, 490, 1161, 610, - 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 665, 664, - 663, 662, 661, 660, 659, 658, 0, 0, 607, 507, - 355, 307, 351, 352, 359, 724, 720, 725, 708, 711, - 710, 686, 0, 315, 587, 422, 470, 376, 652, 653, - 0, 706, 259, 260, 261, 262, 263, 264, 265, 266, - 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, - 280, 281, 282, 283, 284, 285, 655, 276, 277, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 298, 299, 0, 0, 0, 0, 309, 712, 713, - 714, 715, 716, 0, 0, 310, 311, 312, 0, 0, - 274, 275, 302, 498, 303, 304, 305, 306, 0, 0, - 537, 538, 539, 562, 0, 540, 522, 586, 386, 316, - 502, 529, 722, 0, 0, 0, 0, 0, 0, 0, - 637, 648, 682, 0, 694, 695, 697, 699, 698, 701, - 495, 496, 709, 0, 0, 703, 704, 705, 702, 426, - 482, 503, 489, 0, 728, 577, 578, 729, 690, 317, - 453, 0, 0, 592, 626, 615, 700, 580, 0, 0, - 3330, 0, 0, 0, 0, 0, 0, 0, 369, 0, - 0, 421, 630, 611, 622, 612, 597, 598, 599, 606, - 381, 600, 601, 602, 572, 603, 573, 604, 605, 0, - 629, 579, 491, 437, 0, 646, 0, 0, 0, 0, + 0, 309, 717, 718, 719, 720, 721, 0, 0, 310, + 311, 312, 0, 0, 274, 275, 302, 503, 303, 304, + 305, 306, 0, 0, 542, 543, 544, 567, 0, 545, + 527, 591, 386, 316, 507, 534, 727, 0, 0, 0, + 0, 0, 0, 0, 642, 653, 687, 0, 699, 700, + 702, 704, 703, 706, 500, 501, 714, 0, 0, 708, + 709, 710, 707, 431, 487, 508, 494, 0, 733, 582, + 583, 734, 695, 317, 458, 0, 0, 597, 631, 620, + 705, 585, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 369, 1736, 0, 426, 635, 616, 627, 617, + 602, 603, 604, 611, 381, 605, 606, 607, 577, 608, + 578, 609, 610, 0, 634, 584, 496, 442, 0, 651, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 245, 0, 0, 1734, 0, 0, 0, 337, 246, 579, + 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 497, 526, 0, 539, 0, 407, 408, 1732, + 0, 0, 0, 0, 0, 0, 324, 504, 523, 338, + 491, 537, 343, 499, 516, 333, 457, 488, 0, 0, + 326, 521, 498, 439, 325, 0, 482, 366, 383, 363, + 455, 0, 0, 520, 550, 362, 540, 0, 531, 328, + 0, 530, 454, 517, 522, 440, 433, 0, 327, 519, + 438, 432, 413, 373, 566, 414, 415, 416, 417, 418, + 419, 387, 469, 430, 470, 388, 444, 443, 445, 389, + 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, + 0, 0, 0, 0, 561, 562, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 694, 0, 0, 698, 0, 533, 0, 0, 0, + 0, 0, 0, 502, 0, 0, 420, 0, 0, 0, + 551, 0, 485, 460, 736, 0, 0, 483, 428, 518, + 471, 524, 505, 532, 477, 472, 318, 506, 365, 441, + 334, 336, 726, 367, 370, 374, 375, 450, 451, 465, + 490, 509, 510, 511, 364, 348, 484, 349, 384, 350, + 319, 356, 354, 357, 492, 358, 321, 466, 515, 0, + 380, 480, 436, 322, 435, 467, 514, 513, 335, 541, + 548, 549, 639, 0, 554, 737, 738, 739, 563, 0, + 473, 331, 330, 0, 0, 0, 360, 468, 344, 346, + 347, 345, 463, 464, 568, 569, 570, 572, 0, 573, + 574, 0, 0, 0, 0, 575, 640, 656, 624, 593, + 556, 648, 590, 594, 595, 401, 402, 403, 404, 659, + 0, 0, 0, 547, 421, 422, 0, 372, 371, 437, + 323, 0, 0, 410, 400, 474, 329, 368, 412, 406, + 423, 424, 425, 378, 313, 314, 732, 361, 456, 661, + 696, 697, 586, 0, 649, 587, 596, 353, 621, 633, + 632, 452, 546, 0, 644, 647, 576, 731, 0, 641, + 655, 735, 654, 728, 462, 0, 489, 652, 599, 0, + 645, 618, 619, 0, 646, 614, 650, 0, 588, 0, + 557, 560, 589, 674, 675, 676, 320, 559, 678, 679, + 680, 681, 682, 683, 684, 677, 529, 622, 598, 625, + 538, 601, 600, 0, 0, 636, 555, 637, 638, 446, + 447, 448, 449, 382, 662, 342, 558, 476, 0, 623, + 0, 0, 0, 0, 0, 0, 0, 0, 628, 629, + 626, 740, 0, 685, 686, 0, 0, 552, 553, 377, + 0, 571, 385, 341, 461, 379, 536, 409, 0, 564, + 630, 565, 478, 479, 688, 693, 689, 690, 692, 712, + 453, 399, 405, 493, 411, 429, 481, 535, 459, 486, + 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, - 0, 0, 0, 337, 246, 574, 696, 576, 575, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, + 0, 0, 0, 670, 669, 668, 667, 666, 665, 664, + 663, 0, 0, 612, 512, 355, 307, 351, 352, 359, + 729, 725, 730, 713, 716, 715, 691, 0, 315, 592, + 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, + 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, + 271, 272, 273, 278, 279, 280, 281, 282, 283, 284, + 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 298, 299, 0, 0, + 0, 0, 309, 717, 718, 719, 720, 721, 0, 0, + 310, 311, 312, 0, 0, 274, 275, 302, 503, 303, + 304, 305, 306, 0, 0, 542, 543, 544, 567, 0, + 545, 527, 591, 386, 316, 507, 534, 727, 0, 0, + 0, 0, 0, 0, 0, 642, 653, 687, 0, 699, + 700, 702, 704, 703, 706, 500, 501, 714, 0, 0, + 708, 709, 710, 707, 431, 487, 508, 494, 0, 733, + 582, 583, 734, 695, 317, 458, 0, 0, 597, 631, + 620, 705, 585, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 369, 1730, 0, 426, 635, 616, 627, + 617, 602, 603, 604, 611, 381, 605, 606, 607, 577, + 608, 578, 609, 610, 0, 634, 584, 496, 442, 0, + 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 245, 0, 0, 1734, 0, 0, 0, 337, 246, + 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 492, 521, - 0, 534, 0, 406, 407, 0, 0, 0, 0, 0, - 0, 0, 324, 499, 518, 338, 486, 532, 343, 494, - 511, 333, 452, 483, 0, 0, 326, 516, 493, 434, - 325, 0, 477, 366, 383, 363, 450, 0, 0, 515, - 545, 362, 535, 0, 526, 328, 0, 525, 449, 512, - 517, 435, 428, 0, 327, 514, 433, 427, 412, 373, - 561, 413, 414, 387, 464, 425, 465, 388, 439, 438, - 440, 389, 390, 391, 392, 393, 394, 395, 396, 397, - 398, 0, 0, 0, 0, 0, 556, 557, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 3333, 0, 0, - 0, 0, 3332, 689, 0, 0, 693, 0, 528, 0, - 0, 0, 0, 0, 0, 497, 0, 0, 415, 0, - 0, 0, 546, 0, 480, 455, 731, 0, 0, 478, - 423, 513, 466, 519, 500, 527, 472, 467, 318, 501, - 365, 436, 334, 336, 721, 367, 370, 374, 375, 445, - 446, 460, 485, 504, 505, 506, 364, 348, 479, 349, - 384, 350, 319, 356, 354, 357, 487, 358, 321, 461, - 510, 0, 380, 475, 431, 322, 430, 462, 509, 508, - 335, 536, 543, 544, 634, 0, 549, 732, 733, 734, - 558, 0, 468, 331, 330, 0, 0, 0, 360, 463, - 344, 346, 347, 345, 458, 459, 563, 564, 565, 567, - 0, 568, 569, 0, 0, 0, 0, 570, 635, 651, - 619, 588, 551, 643, 585, 589, 590, 401, 402, 403, - 654, 0, 0, 0, 542, 416, 417, 0, 372, 371, - 432, 323, 0, 0, 409, 400, 469, 329, 368, 411, - 405, 418, 419, 420, 378, 313, 314, 727, 361, 451, - 656, 691, 692, 581, 0, 644, 582, 591, 353, 616, - 628, 627, 447, 541, 0, 639, 642, 571, 726, 0, - 636, 650, 730, 649, 723, 457, 0, 484, 647, 594, - 0, 640, 613, 614, 0, 641, 609, 645, 0, 583, - 0, 552, 555, 584, 669, 670, 671, 320, 554, 673, - 674, 675, 676, 677, 678, 679, 672, 524, 617, 593, - 620, 533, 596, 595, 0, 0, 631, 550, 632, 633, - 441, 442, 443, 444, 382, 657, 342, 553, 471, 0, - 618, 0, 0, 0, 0, 0, 0, 0, 0, 623, - 624, 621, 735, 0, 680, 681, 0, 0, 547, 548, - 377, 0, 566, 385, 341, 456, 379, 531, 408, 0, - 559, 625, 560, 473, 474, 683, 688, 684, 685, 687, - 707, 448, 399, 404, 488, 410, 424, 476, 530, 454, - 481, 339, 520, 490, 429, 610, 638, 0, 0, 0, + 0, 0, 0, 497, 526, 0, 539, 0, 407, 408, + 1732, 0, 0, 0, 0, 0, 0, 324, 504, 523, + 338, 491, 537, 343, 499, 516, 333, 457, 488, 0, + 0, 326, 521, 498, 439, 325, 0, 482, 366, 383, + 363, 455, 0, 0, 520, 550, 362, 540, 0, 531, + 328, 0, 530, 454, 517, 522, 440, 433, 0, 327, + 519, 438, 432, 413, 373, 566, 414, 415, 416, 417, + 418, 419, 387, 469, 430, 470, 388, 444, 443, 445, + 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, + 0, 0, 0, 0, 0, 561, 562, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 694, 0, 0, 698, 0, 533, 0, 0, + 0, 0, 0, 0, 502, 0, 0, 420, 0, 0, + 0, 551, 0, 485, 460, 736, 0, 0, 483, 428, + 518, 471, 524, 505, 532, 477, 472, 318, 506, 365, + 441, 334, 336, 726, 367, 370, 374, 375, 450, 451, + 465, 490, 509, 510, 511, 364, 348, 484, 349, 384, + 350, 319, 356, 354, 357, 492, 358, 321, 466, 515, + 0, 380, 480, 436, 322, 435, 467, 514, 513, 335, + 541, 548, 549, 639, 0, 554, 737, 738, 739, 563, + 0, 473, 331, 330, 0, 0, 0, 360, 468, 344, + 346, 347, 345, 463, 464, 568, 569, 570, 572, 0, + 573, 574, 0, 0, 0, 0, 575, 640, 656, 624, + 593, 556, 648, 590, 594, 595, 401, 402, 403, 404, + 659, 0, 0, 0, 547, 421, 422, 0, 372, 371, + 437, 323, 0, 0, 410, 400, 474, 329, 368, 412, + 406, 423, 424, 425, 378, 313, 314, 732, 361, 456, + 661, 696, 697, 586, 0, 649, 587, 596, 353, 621, + 633, 632, 452, 546, 0, 644, 647, 576, 731, 0, + 641, 655, 735, 654, 728, 462, 0, 489, 652, 599, + 0, 645, 618, 619, 0, 646, 614, 650, 0, 588, + 0, 557, 560, 589, 674, 675, 676, 320, 559, 678, + 679, 680, 681, 682, 683, 684, 677, 529, 622, 598, + 625, 538, 601, 600, 0, 0, 636, 555, 637, 638, + 446, 447, 448, 449, 382, 662, 342, 558, 476, 0, + 623, 0, 0, 0, 0, 0, 0, 0, 0, 628, + 629, 626, 740, 0, 685, 686, 0, 0, 552, 553, + 377, 0, 571, 385, 341, 461, 379, 536, 409, 0, + 564, 630, 565, 478, 479, 688, 693, 689, 690, 692, + 712, 453, 399, 405, 493, 411, 429, 481, 535, 459, + 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 665, 664, 663, 662, 661, 660, - 659, 658, 0, 0, 607, 507, 355, 307, 351, 352, - 359, 724, 720, 725, 708, 711, 710, 686, 0, 315, - 587, 422, 470, 376, 652, 653, 0, 706, 259, 260, + 0, 0, 0, 0, 670, 669, 668, 667, 666, 665, + 664, 663, 0, 0, 612, 512, 355, 307, 351, 352, + 359, 729, 725, 730, 713, 716, 715, 691, 0, 315, + 592, 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, 283, - 284, 285, 655, 276, 277, 286, 287, 288, 289, 290, + 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, - 0, 0, 0, 309, 712, 713, 714, 715, 716, 0, - 0, 310, 311, 312, 0, 0, 274, 275, 302, 498, - 303, 304, 305, 306, 0, 0, 537, 538, 539, 562, - 0, 540, 522, 586, 386, 316, 502, 529, 722, 0, - 0, 0, 0, 0, 0, 0, 637, 648, 682, 0, - 694, 695, 697, 699, 698, 701, 495, 496, 709, 0, - 0, 703, 704, 705, 702, 426, 482, 503, 489, 0, - 728, 577, 578, 729, 690, 317, 453, 0, 0, 592, - 626, 615, 700, 580, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 369, 1725, 0, 421, 630, 611, - 622, 612, 597, 598, 599, 606, 381, 600, 601, 602, - 572, 603, 573, 604, 605, 0, 629, 579, 491, 437, - 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 245, 0, 0, 1723, 0, 0, 0, 337, - 246, 574, 696, 576, 575, 0, 0, 0, 0, 0, + 0, 0, 0, 309, 717, 718, 719, 720, 721, 0, + 0, 310, 311, 312, 0, 0, 274, 275, 302, 503, + 303, 304, 305, 306, 0, 0, 542, 543, 544, 567, + 0, 545, 527, 591, 386, 316, 507, 534, 727, 0, + 0, 0, 0, 0, 0, 0, 642, 653, 687, 0, + 699, 700, 702, 704, 703, 706, 500, 501, 714, 0, + 0, 708, 709, 710, 707, 431, 487, 508, 494, 0, + 733, 582, 583, 734, 695, 317, 458, 0, 0, 597, + 631, 620, 705, 585, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 369, 0, 0, 426, 635, 616, + 627, 617, 602, 603, 604, 611, 381, 605, 606, 607, + 577, 608, 578, 609, 610, 0, 634, 584, 496, 442, + 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4699, 0, 245, 950, 0, 0, 0, 0, 0, 337, + 246, 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 492, 521, 0, 534, 0, 406, - 407, 1721, 0, 0, 0, 0, 0, 0, 324, 499, - 518, 338, 486, 532, 343, 494, 511, 333, 452, 483, - 0, 0, 326, 516, 493, 434, 325, 0, 477, 366, - 383, 363, 450, 0, 0, 515, 545, 362, 535, 0, - 526, 328, 0, 525, 449, 512, 517, 435, 428, 0, - 327, 514, 433, 427, 412, 373, 561, 413, 414, 387, - 464, 425, 465, 388, 439, 438, 440, 389, 390, 391, - 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, - 0, 0, 556, 557, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, - 0, 0, 693, 0, 528, 0, 0, 0, 0, 0, - 0, 497, 0, 0, 415, 0, 0, 0, 546, 0, - 480, 455, 731, 0, 0, 478, 423, 513, 466, 519, - 500, 527, 472, 467, 318, 501, 365, 436, 334, 336, - 721, 367, 370, 374, 375, 445, 446, 460, 485, 504, - 505, 506, 364, 348, 479, 349, 384, 350, 319, 356, - 354, 357, 487, 358, 321, 461, 510, 0, 380, 475, - 431, 322, 430, 462, 509, 508, 335, 536, 543, 544, - 634, 0, 549, 732, 733, 734, 558, 0, 468, 331, - 330, 0, 0, 0, 360, 463, 344, 346, 347, 345, - 458, 459, 563, 564, 565, 567, 0, 568, 569, 0, - 0, 0, 0, 570, 635, 651, 619, 588, 551, 643, - 585, 589, 590, 401, 402, 403, 654, 0, 0, 0, - 542, 416, 417, 0, 372, 371, 432, 323, 0, 0, - 409, 400, 469, 329, 368, 411, 405, 418, 419, 420, - 378, 313, 314, 727, 361, 451, 656, 691, 692, 581, - 0, 644, 582, 591, 353, 616, 628, 627, 447, 541, - 0, 639, 642, 571, 726, 0, 636, 650, 730, 649, - 723, 457, 0, 484, 647, 594, 0, 640, 613, 614, - 0, 641, 609, 645, 0, 583, 0, 552, 555, 584, - 669, 670, 671, 320, 554, 673, 674, 675, 676, 677, - 678, 679, 672, 524, 617, 593, 620, 533, 596, 595, - 0, 0, 631, 550, 632, 633, 441, 442, 443, 444, - 382, 657, 342, 553, 471, 0, 618, 0, 0, 0, - 0, 0, 0, 0, 0, 623, 624, 621, 735, 0, - 680, 681, 0, 0, 547, 548, 377, 0, 566, 385, - 341, 456, 379, 531, 408, 0, 559, 625, 560, 473, - 474, 683, 688, 684, 685, 687, 707, 448, 399, 404, - 488, 410, 424, 476, 530, 454, 481, 339, 520, 490, - 429, 610, 638, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 665, 664, 663, 662, 661, 660, 659, 658, 0, 0, - 607, 507, 355, 307, 351, 352, 359, 724, 720, 725, - 708, 711, 710, 686, 0, 315, 587, 422, 470, 376, - 652, 653, 0, 706, 259, 260, 261, 262, 263, 264, - 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, - 278, 279, 280, 281, 282, 283, 284, 285, 655, 276, - 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 298, 299, 0, 0, 0, 0, 309, - 712, 713, 714, 715, 716, 0, 0, 310, 311, 312, - 0, 0, 274, 275, 302, 498, 303, 304, 305, 306, - 0, 0, 537, 538, 539, 562, 0, 540, 522, 586, - 386, 316, 502, 529, 722, 0, 0, 0, 0, 0, - 0, 0, 637, 648, 682, 0, 694, 695, 697, 699, - 698, 701, 495, 496, 709, 0, 0, 703, 704, 705, - 702, 426, 482, 503, 489, 0, 728, 577, 578, 729, - 690, 317, 453, 0, 0, 592, 626, 615, 700, 580, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 369, 1719, 0, 421, 630, 611, 622, 612, 597, 598, - 599, 606, 381, 600, 601, 602, 572, 603, 573, 604, - 605, 0, 629, 579, 491, 437, 0, 646, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 1723, 0, 0, 0, 337, 246, 574, 696, 576, - 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 497, 526, 0, 539, 0, 407, + 408, 0, 0, 0, 0, 0, 0, 0, 324, 504, + 523, 338, 491, 537, 343, 499, 516, 333, 457, 488, + 0, 0, 326, 521, 498, 439, 325, 0, 482, 366, + 383, 363, 455, 0, 0, 520, 550, 362, 540, 0, + 531, 328, 0, 530, 454, 517, 522, 440, 433, 0, + 327, 519, 438, 432, 413, 373, 566, 414, 415, 416, + 417, 418, 419, 387, 469, 430, 470, 388, 444, 443, + 445, 389, 390, 391, 392, 393, 394, 395, 396, 397, + 398, 0, 0, 0, 0, 0, 561, 562, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 694, 0, 0, 698, 0, 533, 0, + 0, 0, 0, 0, 0, 502, 0, 0, 420, 0, + 0, 0, 551, 0, 485, 460, 736, 0, 0, 483, + 428, 518, 471, 524, 505, 532, 477, 472, 318, 506, + 365, 441, 334, 336, 726, 367, 370, 374, 375, 450, + 451, 465, 490, 509, 510, 511, 364, 348, 484, 349, + 384, 350, 319, 356, 354, 357, 492, 358, 321, 466, + 515, 0, 380, 480, 436, 322, 435, 467, 514, 513, + 335, 541, 548, 549, 639, 0, 554, 737, 738, 739, + 563, 0, 473, 331, 330, 0, 0, 0, 360, 468, + 344, 346, 347, 345, 463, 464, 568, 569, 570, 572, + 0, 573, 574, 0, 0, 0, 0, 575, 640, 656, + 624, 593, 556, 648, 590, 594, 595, 401, 402, 403, + 404, 659, 0, 0, 0, 547, 421, 422, 0, 372, + 371, 437, 323, 0, 0, 410, 400, 474, 329, 368, + 412, 406, 423, 424, 425, 378, 313, 314, 732, 361, + 456, 661, 696, 697, 586, 0, 649, 587, 596, 353, + 621, 633, 632, 452, 546, 0, 644, 647, 576, 731, + 0, 641, 655, 735, 654, 728, 462, 0, 489, 652, + 599, 0, 645, 618, 619, 0, 646, 614, 650, 0, + 588, 0, 557, 560, 589, 674, 675, 676, 320, 559, + 678, 679, 680, 681, 682, 683, 684, 677, 529, 622, + 598, 625, 538, 601, 600, 0, 0, 636, 555, 637, + 638, 446, 447, 448, 449, 382, 662, 342, 558, 476, + 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, + 628, 629, 626, 740, 0, 685, 686, 0, 0, 552, + 553, 377, 0, 571, 385, 341, 461, 379, 536, 409, + 0, 564, 630, 565, 478, 479, 688, 693, 689, 690, + 692, 712, 453, 399, 405, 493, 411, 429, 481, 535, + 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, + 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 670, 669, 668, 667, 666, + 665, 664, 663, 0, 0, 612, 512, 355, 307, 351, + 352, 359, 729, 725, 730, 713, 716, 715, 691, 0, + 315, 592, 427, 475, 376, 657, 658, 0, 711, 259, + 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, + 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, + 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, + 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, + 0, 0, 0, 0, 309, 717, 718, 719, 720, 721, + 0, 0, 310, 311, 312, 0, 0, 274, 275, 302, + 503, 303, 304, 305, 306, 0, 0, 542, 543, 544, + 567, 0, 545, 527, 591, 386, 316, 507, 534, 727, + 0, 0, 0, 0, 0, 0, 0, 642, 653, 687, + 0, 699, 700, 702, 704, 703, 706, 500, 501, 714, + 0, 0, 708, 709, 710, 707, 431, 487, 508, 494, + 0, 733, 582, 583, 734, 695, 317, 458, 0, 0, + 597, 631, 620, 705, 585, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 369, 0, 0, 426, 635, + 616, 627, 617, 602, 603, 604, 611, 381, 605, 606, + 607, 577, 608, 578, 609, 610, 0, 634, 584, 496, + 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 245, 0, 0, 1734, 0, 0, 0, + 337, 246, 579, 701, 581, 580, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 492, 521, 0, 534, 0, 406, 407, 1721, 0, 0, - 0, 0, 0, 0, 324, 499, 518, 338, 486, 532, - 343, 494, 511, 333, 452, 483, 0, 0, 326, 516, - 493, 434, 325, 0, 477, 366, 383, 363, 450, 0, - 0, 515, 545, 362, 535, 0, 526, 328, 0, 525, - 449, 512, 517, 435, 428, 0, 327, 514, 433, 427, - 412, 373, 561, 413, 414, 387, 464, 425, 465, 388, - 439, 438, 440, 389, 390, 391, 392, 393, 394, 395, - 396, 397, 398, 0, 0, 0, 0, 0, 556, 557, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 689, 0, 0, 693, 0, - 528, 0, 0, 0, 0, 0, 0, 497, 0, 0, - 415, 0, 0, 0, 546, 0, 480, 455, 731, 0, - 0, 478, 423, 513, 466, 519, 500, 527, 472, 467, - 318, 501, 365, 436, 334, 336, 721, 367, 370, 374, - 375, 445, 446, 460, 485, 504, 505, 506, 364, 348, - 479, 349, 384, 350, 319, 356, 354, 357, 487, 358, - 321, 461, 510, 0, 380, 475, 431, 322, 430, 462, - 509, 508, 335, 536, 543, 544, 634, 0, 549, 732, - 733, 734, 558, 0, 468, 331, 330, 0, 0, 0, - 360, 463, 344, 346, 347, 345, 458, 459, 563, 564, - 565, 567, 0, 568, 569, 0, 0, 0, 0, 570, - 635, 651, 619, 588, 551, 643, 585, 589, 590, 401, - 402, 403, 654, 0, 0, 0, 542, 416, 417, 0, - 372, 371, 432, 323, 0, 0, 409, 400, 469, 329, - 368, 411, 405, 418, 419, 420, 378, 313, 314, 727, - 361, 451, 656, 691, 692, 581, 0, 644, 582, 591, - 353, 616, 628, 627, 447, 541, 0, 639, 642, 571, - 726, 0, 636, 650, 730, 649, 723, 457, 0, 484, - 647, 594, 0, 640, 613, 614, 0, 641, 609, 645, - 0, 583, 0, 552, 555, 584, 669, 670, 671, 320, - 554, 673, 674, 675, 676, 677, 678, 679, 672, 524, - 617, 593, 620, 533, 596, 595, 0, 0, 631, 550, - 632, 633, 441, 442, 443, 444, 382, 657, 342, 553, - 471, 0, 618, 0, 0, 0, 0, 0, 0, 0, - 0, 623, 624, 621, 735, 0, 680, 681, 0, 0, - 547, 548, 377, 0, 566, 385, 341, 456, 379, 531, - 408, 0, 559, 625, 560, 473, 474, 683, 688, 684, - 685, 687, 707, 448, 399, 404, 488, 410, 424, 476, - 530, 454, 481, 339, 520, 490, 429, 610, 638, 0, + 0, 0, 0, 0, 0, 497, 526, 0, 539, 0, + 407, 408, 1732, 0, 0, 0, 0, 0, 0, 324, + 504, 523, 338, 491, 537, 343, 499, 516, 333, 457, + 488, 0, 0, 326, 521, 498, 439, 325, 0, 482, + 366, 383, 363, 455, 0, 0, 520, 550, 362, 540, + 0, 531, 328, 0, 530, 454, 517, 522, 440, 433, + 0, 327, 519, 438, 432, 413, 373, 566, 414, 415, + 416, 417, 418, 419, 387, 469, 430, 470, 388, 444, + 443, 445, 389, 390, 391, 392, 393, 394, 395, 396, + 397, 398, 0, 0, 0, 0, 0, 561, 562, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 694, 0, 0, 698, 0, 533, + 0, 0, 0, 0, 0, 0, 502, 0, 0, 420, + 0, 0, 0, 551, 0, 485, 460, 736, 0, 0, + 483, 428, 518, 471, 524, 505, 532, 477, 472, 318, + 506, 365, 441, 334, 336, 726, 367, 370, 374, 375, + 450, 451, 465, 490, 509, 510, 511, 364, 348, 484, + 349, 384, 350, 319, 356, 354, 357, 492, 358, 321, + 466, 515, 0, 380, 480, 436, 322, 435, 467, 514, + 513, 335, 541, 548, 549, 639, 0, 554, 737, 738, + 739, 563, 0, 473, 331, 330, 0, 0, 0, 360, + 468, 344, 346, 347, 345, 463, 464, 568, 569, 570, + 572, 0, 573, 574, 0, 0, 0, 0, 575, 640, + 656, 624, 593, 556, 648, 590, 594, 595, 401, 402, + 403, 404, 659, 0, 0, 0, 547, 421, 422, 0, + 372, 371, 437, 323, 0, 0, 410, 400, 474, 329, + 368, 412, 406, 423, 424, 425, 378, 313, 314, 732, + 361, 456, 661, 696, 697, 586, 0, 649, 587, 596, + 353, 621, 633, 632, 452, 546, 0, 644, 647, 576, + 731, 0, 641, 655, 735, 654, 728, 462, 0, 489, + 652, 599, 0, 645, 618, 619, 0, 646, 614, 650, + 0, 588, 0, 557, 560, 589, 674, 675, 676, 320, + 559, 678, 679, 680, 681, 682, 683, 684, 677, 529, + 622, 598, 625, 538, 601, 600, 0, 0, 636, 555, + 637, 638, 446, 447, 448, 449, 382, 662, 342, 558, + 476, 0, 623, 0, 0, 0, 0, 0, 0, 0, + 0, 628, 629, 626, 740, 0, 685, 686, 0, 0, + 552, 553, 377, 0, 571, 385, 341, 461, 379, 536, + 409, 0, 564, 630, 565, 478, 479, 688, 693, 689, + 690, 692, 712, 453, 399, 405, 493, 411, 429, 481, + 535, 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 665, 664, 663, 662, - 661, 660, 659, 658, 0, 0, 607, 507, 355, 307, - 351, 352, 359, 724, 720, 725, 708, 711, 710, 686, - 0, 315, 587, 422, 470, 376, 652, 653, 0, 706, + 0, 0, 0, 0, 0, 0, 670, 669, 668, 667, + 666, 665, 664, 663, 0, 0, 612, 512, 355, 307, + 351, 352, 359, 729, 725, 730, 713, 716, 715, 691, + 0, 315, 592, 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, - 282, 283, 284, 285, 655, 276, 277, 286, 287, 288, + 282, 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, - 299, 0, 0, 0, 0, 309, 712, 713, 714, 715, - 716, 0, 0, 310, 311, 312, 0, 0, 274, 275, - 302, 498, 303, 304, 305, 306, 0, 0, 537, 538, - 539, 562, 0, 540, 522, 586, 386, 316, 502, 529, - 722, 0, 0, 0, 0, 0, 0, 0, 637, 648, - 682, 0, 694, 695, 697, 699, 698, 701, 495, 496, - 709, 0, 0, 703, 704, 705, 702, 426, 482, 503, - 489, 0, 728, 577, 578, 729, 690, 317, 453, 0, - 0, 592, 626, 615, 700, 580, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 369, 0, 0, 421, - 630, 611, 622, 612, 597, 598, 599, 606, 381, 600, - 601, 602, 572, 603, 573, 604, 605, 0, 629, 579, - 491, 437, 0, 646, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 4666, 0, 245, 944, 0, 0, 0, 0, - 0, 337, 246, 574, 696, 576, 575, 0, 0, 0, + 299, 0, 0, 0, 0, 309, 717, 718, 719, 720, + 721, 0, 0, 310, 311, 312, 0, 0, 274, 275, + 302, 503, 303, 304, 305, 306, 0, 0, 542, 543, + 544, 567, 0, 545, 527, 591, 386, 316, 507, 534, + 727, 0, 0, 0, 0, 0, 0, 0, 642, 653, + 687, 0, 699, 700, 702, 704, 703, 706, 500, 501, + 714, 0, 0, 708, 709, 710, 707, 431, 487, 508, + 494, 0, 733, 582, 583, 734, 695, 317, 458, 0, + 0, 597, 631, 620, 705, 585, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 369, 0, 0, 426, + 635, 616, 627, 617, 602, 603, 604, 611, 381, 605, + 606, 607, 577, 608, 578, 609, 610, 0, 634, 584, + 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 245, 0, 0, 1734, 0, 0, + 0, 337, 246, 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 492, 521, 0, 534, - 0, 406, 407, 0, 0, 0, 0, 0, 0, 0, - 324, 499, 518, 338, 486, 532, 343, 494, 511, 333, - 452, 483, 0, 0, 326, 516, 493, 434, 325, 0, - 477, 366, 383, 363, 450, 0, 0, 515, 545, 362, - 535, 0, 526, 328, 0, 525, 449, 512, 517, 435, - 428, 0, 327, 514, 433, 427, 412, 373, 561, 413, - 414, 387, 464, 425, 465, 388, 439, 438, 440, 389, - 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, - 0, 0, 0, 0, 556, 557, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 689, 0, 0, 693, 0, 528, 0, 0, 0, - 0, 0, 0, 497, 0, 0, 415, 0, 0, 0, - 546, 0, 480, 455, 731, 0, 0, 478, 423, 513, - 466, 519, 500, 527, 472, 467, 318, 501, 365, 436, - 334, 336, 721, 367, 370, 374, 375, 445, 446, 460, - 485, 504, 505, 506, 364, 348, 479, 349, 384, 350, - 319, 356, 354, 357, 487, 358, 321, 461, 510, 0, - 380, 475, 431, 322, 430, 462, 509, 508, 335, 536, - 543, 544, 634, 0, 549, 732, 733, 734, 558, 0, - 468, 331, 330, 0, 0, 0, 360, 463, 344, 346, - 347, 345, 458, 459, 563, 564, 565, 567, 0, 568, - 569, 0, 0, 0, 0, 570, 635, 651, 619, 588, - 551, 643, 585, 589, 590, 401, 402, 403, 654, 0, - 0, 0, 542, 416, 417, 0, 372, 371, 432, 323, - 0, 0, 409, 400, 469, 329, 368, 411, 405, 418, - 419, 420, 378, 313, 314, 727, 361, 451, 656, 691, - 692, 581, 0, 644, 582, 591, 353, 616, 628, 627, - 447, 541, 0, 639, 642, 571, 726, 0, 636, 650, - 730, 649, 723, 457, 0, 484, 647, 594, 0, 640, - 613, 614, 0, 641, 609, 645, 0, 583, 0, 552, - 555, 584, 669, 670, 671, 320, 554, 673, 674, 675, - 676, 677, 678, 679, 672, 524, 617, 593, 620, 533, - 596, 595, 0, 0, 631, 550, 632, 633, 441, 442, - 443, 444, 382, 657, 342, 553, 471, 0, 618, 0, - 0, 0, 0, 0, 0, 0, 0, 623, 624, 621, - 735, 0, 680, 681, 0, 0, 547, 548, 377, 0, - 566, 385, 341, 456, 379, 531, 408, 0, 559, 625, - 560, 473, 474, 683, 688, 684, 685, 687, 707, 448, - 399, 404, 488, 410, 424, 476, 530, 454, 481, 339, - 520, 490, 429, 610, 638, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 665, 664, 663, 662, 661, 660, 659, 658, - 0, 0, 607, 507, 355, 307, 351, 352, 359, 724, - 720, 725, 708, 711, 710, 686, 0, 315, 587, 422, - 470, 376, 652, 653, 0, 706, 259, 260, 261, 262, - 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, - 272, 273, 278, 279, 280, 281, 282, 283, 284, 285, - 655, 276, 277, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, - 0, 309, 712, 713, 714, 715, 716, 0, 0, 310, - 311, 312, 0, 0, 274, 275, 302, 498, 303, 304, - 305, 306, 0, 0, 537, 538, 539, 562, 0, 540, - 522, 586, 386, 316, 502, 529, 722, 0, 0, 0, - 0, 0, 0, 0, 637, 648, 682, 0, 694, 695, - 697, 699, 698, 701, 495, 496, 709, 0, 0, 703, - 704, 705, 702, 426, 482, 503, 489, 0, 728, 577, - 578, 729, 690, 317, 453, 0, 0, 592, 626, 615, - 700, 580, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 369, 0, 0, 421, 630, 611, 622, 612, - 597, 598, 599, 606, 381, 600, 601, 602, 572, 603, - 573, 604, 605, 0, 629, 579, 491, 437, 0, 646, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 1723, 0, 0, 0, 337, 246, 574, - 696, 576, 575, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 497, 526, 0, 539, + 0, 407, 408, 1951, 0, 0, 0, 0, 0, 0, + 324, 504, 523, 338, 491, 537, 343, 499, 516, 333, + 457, 488, 0, 0, 326, 521, 498, 439, 325, 0, + 482, 366, 383, 363, 455, 0, 0, 520, 550, 362, + 540, 0, 531, 328, 0, 530, 454, 517, 522, 440, + 433, 0, 327, 519, 438, 432, 413, 373, 566, 414, + 415, 416, 417, 418, 419, 387, 469, 430, 470, 388, + 444, 443, 445, 389, 390, 391, 392, 393, 394, 395, + 396, 397, 398, 0, 0, 0, 0, 0, 561, 562, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 694, 0, 0, 698, 0, + 533, 0, 0, 0, 0, 0, 0, 502, 0, 0, + 420, 0, 0, 0, 551, 0, 485, 460, 736, 0, + 0, 483, 428, 518, 471, 524, 505, 532, 477, 472, + 318, 506, 365, 441, 334, 336, 726, 367, 370, 374, + 375, 450, 451, 465, 490, 509, 510, 511, 364, 348, + 484, 349, 384, 350, 319, 356, 354, 357, 492, 358, + 321, 466, 515, 0, 380, 480, 436, 322, 435, 467, + 514, 513, 335, 541, 548, 549, 639, 0, 554, 737, + 738, 739, 563, 0, 473, 331, 330, 0, 0, 0, + 360, 468, 344, 346, 347, 345, 463, 464, 568, 569, + 570, 572, 0, 573, 574, 0, 0, 0, 0, 575, + 640, 656, 624, 593, 556, 648, 590, 594, 595, 401, + 402, 403, 404, 659, 0, 0, 0, 547, 421, 422, + 0, 372, 371, 437, 323, 0, 0, 410, 400, 474, + 329, 368, 412, 406, 423, 424, 425, 378, 313, 314, + 732, 361, 456, 661, 696, 697, 586, 0, 649, 587, + 596, 353, 621, 633, 632, 452, 546, 0, 644, 647, + 576, 731, 0, 641, 655, 735, 654, 728, 462, 0, + 489, 652, 599, 0, 645, 618, 619, 0, 646, 614, + 650, 0, 588, 0, 557, 560, 589, 674, 675, 676, + 320, 559, 678, 679, 680, 681, 682, 683, 684, 677, + 529, 622, 598, 625, 538, 601, 600, 0, 0, 636, + 555, 637, 638, 446, 447, 448, 449, 382, 662, 342, + 558, 476, 0, 623, 0, 0, 0, 0, 0, 0, + 0, 0, 628, 629, 626, 740, 0, 685, 686, 0, + 0, 552, 553, 377, 0, 571, 385, 341, 461, 379, + 536, 409, 0, 564, 630, 565, 478, 479, 688, 693, + 689, 690, 692, 712, 453, 399, 405, 493, 411, 429, + 481, 535, 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 492, 521, 0, 534, 0, 406, 407, 1721, - 0, 0, 0, 0, 0, 0, 324, 499, 518, 338, - 486, 532, 343, 494, 511, 333, 452, 483, 0, 0, - 326, 516, 493, 434, 325, 0, 477, 366, 383, 363, - 450, 0, 0, 515, 545, 362, 535, 0, 526, 328, - 0, 525, 449, 512, 517, 435, 428, 0, 327, 514, - 433, 427, 412, 373, 561, 413, 414, 387, 464, 425, - 465, 388, 439, 438, 440, 389, 390, 391, 392, 393, - 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, - 556, 557, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 689, 0, 0, - 693, 0, 528, 0, 0, 0, 0, 0, 0, 497, - 0, 0, 415, 0, 0, 0, 546, 0, 480, 455, - 731, 0, 0, 478, 423, 513, 466, 519, 500, 527, - 472, 467, 318, 501, 365, 436, 334, 336, 721, 367, - 370, 374, 375, 445, 446, 460, 485, 504, 505, 506, - 364, 348, 479, 349, 384, 350, 319, 356, 354, 357, - 487, 358, 321, 461, 510, 0, 380, 475, 431, 322, - 430, 462, 509, 508, 335, 536, 543, 544, 634, 0, - 549, 732, 733, 734, 558, 0, 468, 331, 330, 0, - 0, 0, 360, 463, 344, 346, 347, 345, 458, 459, - 563, 564, 565, 567, 0, 568, 569, 0, 0, 0, - 0, 570, 635, 651, 619, 588, 551, 643, 585, 589, - 590, 401, 402, 403, 654, 0, 0, 0, 542, 416, - 417, 0, 372, 371, 432, 323, 0, 0, 409, 400, - 469, 329, 368, 411, 405, 418, 419, 420, 378, 313, - 314, 727, 361, 451, 656, 691, 692, 581, 0, 644, - 582, 591, 353, 616, 628, 627, 447, 541, 0, 639, - 642, 571, 726, 0, 636, 650, 730, 649, 723, 457, - 0, 484, 647, 594, 0, 640, 613, 614, 0, 641, - 609, 645, 0, 583, 0, 552, 555, 584, 669, 670, - 671, 320, 554, 673, 674, 675, 676, 677, 678, 679, - 672, 524, 617, 593, 620, 533, 596, 595, 0, 0, - 631, 550, 632, 633, 441, 442, 443, 444, 382, 657, - 342, 553, 471, 0, 618, 0, 0, 0, 0, 0, - 0, 0, 0, 623, 624, 621, 735, 0, 680, 681, - 0, 0, 547, 548, 377, 0, 566, 385, 341, 456, - 379, 531, 408, 0, 559, 625, 560, 473, 474, 683, - 688, 684, 685, 687, 707, 448, 399, 404, 488, 410, - 424, 476, 530, 454, 481, 339, 520, 490, 429, 610, - 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 670, 669, 668, + 667, 666, 665, 664, 663, 0, 0, 612, 512, 355, + 307, 351, 352, 359, 729, 725, 730, 713, 716, 715, + 691, 0, 315, 592, 427, 475, 376, 657, 658, 0, + 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, + 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, + 281, 282, 283, 284, 285, 660, 276, 277, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 298, 299, 0, 0, 0, 0, 309, 717, 718, 719, + 720, 721, 0, 0, 310, 311, 312, 0, 0, 274, + 275, 302, 503, 303, 304, 305, 306, 0, 0, 542, + 543, 544, 567, 0, 545, 527, 591, 386, 316, 507, + 534, 727, 0, 0, 0, 0, 0, 0, 0, 642, + 653, 687, 0, 699, 700, 702, 704, 703, 706, 500, + 501, 714, 0, 0, 708, 709, 710, 707, 431, 487, + 508, 494, 0, 733, 582, 583, 734, 695, 317, 458, + 0, 0, 597, 631, 620, 705, 585, 0, 0, 0, + 0, 0, 2823, 0, 0, 0, 0, 369, 0, 0, + 426, 635, 616, 627, 617, 602, 603, 604, 611, 381, + 605, 606, 607, 577, 608, 578, 609, 610, 0, 634, + 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 245, 0, 0, 2825, 0, + 0, 0, 337, 246, 579, 701, 581, 580, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 497, 526, 0, + 539, 0, 407, 408, 0, 0, 0, 0, 0, 0, + 0, 324, 504, 523, 338, 491, 537, 343, 499, 516, + 333, 457, 488, 0, 0, 326, 521, 498, 439, 325, + 0, 482, 366, 383, 363, 455, 0, 0, 520, 550, + 362, 540, 0, 531, 328, 0, 530, 454, 517, 522, + 440, 433, 0, 327, 519, 438, 432, 413, 373, 566, + 414, 415, 416, 417, 418, 419, 387, 469, 430, 470, + 388, 444, 443, 445, 389, 390, 391, 392, 393, 394, + 395, 396, 397, 398, 0, 0, 0, 0, 0, 561, + 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 694, 0, 0, 698, + 0, 533, 0, 0, 0, 0, 0, 0, 502, 0, + 0, 420, 0, 0, 0, 551, 0, 485, 460, 736, + 0, 0, 483, 428, 518, 471, 524, 505, 532, 477, + 472, 318, 506, 365, 441, 334, 336, 726, 367, 370, + 374, 375, 450, 451, 465, 490, 509, 510, 511, 364, + 348, 484, 349, 384, 350, 319, 356, 354, 357, 492, + 358, 321, 466, 515, 0, 380, 480, 436, 322, 435, + 467, 514, 513, 335, 541, 548, 549, 639, 0, 554, + 737, 738, 739, 563, 0, 473, 331, 330, 0, 0, + 0, 360, 468, 344, 346, 347, 345, 463, 464, 568, + 569, 570, 572, 0, 573, 574, 0, 0, 0, 0, + 575, 640, 656, 624, 593, 556, 648, 590, 594, 595, + 401, 402, 403, 404, 659, 0, 0, 0, 547, 421, + 422, 0, 372, 371, 437, 323, 0, 0, 410, 400, + 474, 329, 368, 412, 406, 423, 424, 425, 378, 313, + 314, 732, 361, 456, 661, 696, 697, 586, 0, 649, + 587, 596, 353, 621, 633, 632, 452, 546, 0, 644, + 647, 576, 731, 0, 641, 655, 735, 654, 728, 462, + 0, 489, 652, 599, 0, 645, 618, 619, 0, 646, + 614, 650, 0, 588, 0, 557, 560, 589, 674, 675, + 676, 320, 559, 678, 679, 680, 681, 682, 683, 684, + 677, 529, 622, 598, 625, 538, 601, 600, 0, 0, + 636, 555, 637, 638, 446, 447, 448, 449, 382, 662, + 342, 558, 476, 0, 623, 0, 0, 0, 0, 0, + 0, 0, 0, 628, 629, 626, 740, 0, 685, 686, + 0, 0, 552, 553, 377, 0, 571, 385, 341, 461, + 379, 536, 409, 0, 564, 630, 565, 478, 479, 688, + 693, 689, 690, 692, 712, 453, 399, 405, 493, 411, + 429, 481, 535, 459, 486, 339, 525, 495, 434, 615, + 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 665, 664, - 663, 662, 661, 660, 659, 658, 0, 0, 607, 507, - 355, 307, 351, 352, 359, 724, 720, 725, 708, 711, - 710, 686, 0, 315, 587, 422, 470, 376, 652, 653, - 0, 706, 259, 260, 261, 262, 263, 264, 265, 266, + 0, 0, 0, 0, 0, 0, 0, 0, 670, 669, + 668, 667, 666, 665, 664, 663, 0, 0, 612, 512, + 355, 307, 351, 352, 359, 729, 725, 730, 713, 716, + 715, 691, 0, 315, 592, 427, 475, 376, 657, 658, + 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, - 280, 281, 282, 283, 284, 285, 655, 276, 277, 286, + 280, 281, 282, 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 298, 299, 0, 0, 0, 0, 309, 712, 713, - 714, 715, 716, 0, 0, 310, 311, 312, 0, 0, - 274, 275, 302, 498, 303, 304, 305, 306, 0, 0, - 537, 538, 539, 562, 0, 540, 522, 586, 386, 316, - 502, 529, 722, 0, 0, 0, 0, 0, 0, 0, - 637, 648, 682, 0, 694, 695, 697, 699, 698, 701, - 495, 496, 709, 0, 0, 703, 704, 705, 702, 426, - 482, 503, 489, 0, 728, 577, 578, 729, 690, 317, - 453, 0, 0, 592, 626, 615, 700, 580, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, - 0, 421, 630, 611, 622, 612, 597, 598, 599, 606, - 381, 600, 601, 602, 572, 603, 573, 604, 605, 0, - 629, 579, 491, 437, 0, 646, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 245, 0, 0, 1723, - 0, 0, 0, 337, 246, 574, 696, 576, 575, 0, + 297, 298, 299, 0, 0, 0, 0, 309, 717, 718, + 719, 720, 721, 0, 0, 310, 311, 312, 0, 0, + 274, 275, 302, 503, 303, 304, 305, 306, 0, 0, + 542, 543, 544, 567, 0, 545, 527, 591, 386, 316, + 507, 534, 727, 0, 0, 0, 0, 0, 0, 0, + 642, 653, 687, 0, 699, 700, 702, 704, 703, 706, + 500, 501, 714, 0, 0, 708, 709, 710, 707, 431, + 487, 508, 494, 0, 733, 582, 583, 734, 695, 317, + 458, 0, 0, 597, 631, 620, 705, 585, 0, 0, + 0, 0, 0, 2382, 0, 0, 0, 0, 369, 0, + 0, 426, 635, 616, 627, 617, 602, 603, 604, 611, + 381, 605, 606, 607, 577, 608, 578, 609, 610, 0, + 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 245, 0, 0, 2383, + 0, 0, 0, 337, 246, 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 492, 521, - 0, 534, 0, 406, 407, 1936, 0, 0, 0, 0, - 0, 0, 324, 499, 518, 338, 486, 532, 343, 494, - 511, 333, 452, 483, 0, 0, 326, 516, 493, 434, - 325, 0, 477, 366, 383, 363, 450, 0, 0, 515, - 545, 362, 535, 0, 526, 328, 0, 525, 449, 512, - 517, 435, 428, 0, 327, 514, 433, 427, 412, 373, - 561, 413, 414, 387, 464, 425, 465, 388, 439, 438, - 440, 389, 390, 391, 392, 393, 394, 395, 396, 397, - 398, 0, 0, 0, 0, 0, 556, 557, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 689, 0, 0, 693, 0, 528, 0, - 0, 0, 0, 0, 0, 497, 0, 0, 415, 0, - 0, 0, 546, 0, 480, 455, 731, 0, 0, 478, - 423, 513, 466, 519, 500, 527, 472, 467, 318, 501, - 365, 436, 334, 336, 721, 367, 370, 374, 375, 445, - 446, 460, 485, 504, 505, 506, 364, 348, 479, 349, - 384, 350, 319, 356, 354, 357, 487, 358, 321, 461, - 510, 0, 380, 475, 431, 322, 430, 462, 509, 508, - 335, 536, 543, 544, 634, 0, 549, 732, 733, 734, - 558, 0, 468, 331, 330, 0, 0, 0, 360, 463, - 344, 346, 347, 345, 458, 459, 563, 564, 565, 567, - 0, 568, 569, 0, 0, 0, 0, 570, 635, 651, - 619, 588, 551, 643, 585, 589, 590, 401, 402, 403, - 654, 0, 0, 0, 542, 416, 417, 0, 372, 371, - 432, 323, 0, 0, 409, 400, 469, 329, 368, 411, - 405, 418, 419, 420, 378, 313, 314, 727, 361, 451, - 656, 691, 692, 581, 0, 644, 582, 591, 353, 616, - 628, 627, 447, 541, 0, 639, 642, 571, 726, 0, - 636, 650, 730, 649, 723, 457, 0, 484, 647, 594, - 0, 640, 613, 614, 0, 641, 609, 645, 0, 583, - 0, 552, 555, 584, 669, 670, 671, 320, 554, 673, - 674, 675, 676, 677, 678, 679, 672, 524, 617, 593, - 620, 533, 596, 595, 0, 0, 631, 550, 632, 633, - 441, 442, 443, 444, 382, 657, 342, 553, 471, 0, - 618, 0, 0, 0, 0, 0, 0, 0, 0, 623, - 624, 621, 735, 0, 680, 681, 0, 0, 547, 548, - 377, 0, 566, 385, 341, 456, 379, 531, 408, 0, - 559, 625, 560, 473, 474, 683, 688, 684, 685, 687, - 707, 448, 399, 404, 488, 410, 424, 476, 530, 454, - 481, 339, 520, 490, 429, 610, 638, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, + 0, 0, 0, 0, 0, 0, 0, 0, 497, 526, + 0, 539, 0, 407, 408, 0, 0, 0, 0, 0, + 0, 0, 324, 504, 523, 338, 491, 537, 343, 499, + 516, 333, 457, 488, 0, 0, 326, 521, 498, 439, + 325, 0, 482, 366, 383, 363, 455, 0, 0, 520, + 550, 362, 540, 0, 531, 328, 0, 530, 454, 517, + 522, 440, 433, 0, 327, 519, 438, 432, 413, 373, + 566, 414, 415, 416, 417, 418, 419, 387, 469, 430, + 470, 388, 444, 443, 445, 389, 390, 391, 392, 393, + 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, + 561, 562, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 694, 0, 0, + 698, 0, 533, 0, 0, 0, 0, 0, 0, 502, + 0, 0, 420, 0, 0, 0, 551, 0, 485, 460, + 736, 0, 0, 483, 428, 518, 471, 524, 505, 532, + 477, 472, 318, 506, 365, 441, 334, 336, 726, 367, + 370, 374, 375, 450, 451, 465, 490, 509, 510, 511, + 364, 348, 484, 349, 384, 350, 319, 356, 354, 357, + 492, 358, 321, 466, 515, 0, 380, 480, 436, 322, + 435, 467, 514, 513, 335, 541, 548, 549, 639, 0, + 554, 737, 738, 739, 563, 0, 473, 331, 330, 0, + 0, 0, 360, 468, 344, 346, 347, 345, 463, 464, + 568, 569, 570, 572, 0, 573, 574, 0, 0, 0, + 0, 575, 640, 656, 624, 593, 556, 648, 590, 594, + 595, 401, 402, 403, 404, 659, 0, 0, 0, 547, + 421, 422, 0, 372, 371, 437, 323, 0, 0, 410, + 400, 474, 329, 368, 412, 406, 423, 424, 425, 378, + 313, 314, 732, 361, 456, 661, 696, 697, 586, 0, + 649, 587, 596, 353, 621, 633, 632, 452, 546, 0, + 644, 647, 576, 731, 0, 641, 655, 735, 654, 728, + 462, 0, 489, 652, 599, 0, 645, 618, 619, 0, + 646, 614, 650, 0, 588, 0, 557, 560, 589, 674, + 675, 676, 320, 559, 678, 679, 680, 681, 682, 683, + 684, 677, 529, 622, 598, 625, 538, 601, 600, 0, + 0, 636, 555, 637, 638, 446, 447, 448, 449, 382, + 662, 342, 558, 476, 0, 623, 0, 0, 0, 0, + 0, 0, 0, 0, 628, 629, 626, 740, 0, 685, + 686, 0, 0, 552, 553, 377, 0, 571, 385, 341, + 461, 379, 536, 409, 0, 564, 630, 565, 478, 479, + 688, 693, 689, 690, 692, 712, 453, 399, 405, 493, + 411, 429, 481, 535, 459, 486, 339, 525, 495, 434, + 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 670, + 669, 668, 667, 666, 665, 664, 663, 0, 0, 612, + 512, 355, 307, 351, 352, 359, 729, 725, 730, 713, + 716, 715, 691, 0, 315, 592, 427, 475, 376, 657, + 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, + 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, + 279, 280, 281, 282, 283, 284, 285, 660, 276, 277, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 298, 299, 0, 0, 0, 0, 309, 717, + 718, 719, 720, 721, 0, 0, 310, 311, 312, 0, + 0, 274, 275, 302, 503, 303, 304, 305, 306, 0, + 0, 542, 543, 544, 567, 0, 545, 527, 591, 386, + 316, 507, 534, 727, 0, 0, 0, 0, 0, 0, + 0, 642, 653, 687, 0, 699, 700, 702, 704, 703, + 706, 500, 501, 714, 0, 0, 708, 709, 710, 707, + 431, 487, 508, 494, 0, 733, 582, 583, 734, 695, + 317, 458, 0, 0, 597, 631, 620, 705, 585, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, + 0, 0, 426, 635, 616, 627, 617, 602, 603, 604, + 611, 381, 605, 606, 607, 577, 608, 578, 609, 610, + 0, 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 665, 664, 663, 662, 661, 660, - 659, 658, 0, 0, 607, 507, 355, 307, 351, 352, - 359, 724, 720, 725, 708, 711, 710, 686, 0, 315, - 587, 422, 470, 376, 652, 653, 0, 706, 259, 260, - 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, - 270, 271, 272, 273, 278, 279, 280, 281, 282, 283, - 284, 285, 655, 276, 277, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, - 0, 0, 0, 309, 712, 713, 714, 715, 716, 0, - 0, 310, 311, 312, 0, 0, 274, 275, 302, 498, - 303, 304, 305, 306, 0, 0, 537, 538, 539, 562, - 0, 540, 522, 586, 386, 316, 502, 529, 722, 0, - 0, 0, 0, 0, 0, 0, 637, 648, 682, 0, - 694, 695, 697, 699, 698, 701, 495, 496, 709, 0, - 0, 703, 704, 705, 702, 426, 482, 503, 489, 0, - 728, 577, 578, 729, 690, 317, 453, 0, 0, 592, - 626, 615, 700, 580, 0, 0, 0, 0, 0, 2806, - 0, 0, 0, 0, 369, 0, 0, 421, 630, 611, - 622, 612, 597, 598, 599, 606, 381, 600, 601, 602, - 572, 603, 573, 604, 605, 0, 629, 579, 491, 437, - 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 245, 0, 0, 2808, 0, 0, 0, 337, - 246, 574, 696, 576, 575, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, + 3591, 3593, 0, 0, 337, 246, 579, 701, 581, 580, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 492, 521, 0, 534, 0, 406, - 407, 0, 0, 0, 0, 0, 0, 0, 324, 499, - 518, 338, 486, 532, 343, 494, 511, 333, 452, 483, - 0, 0, 326, 516, 493, 434, 325, 0, 477, 366, - 383, 363, 450, 0, 0, 515, 545, 362, 535, 0, - 526, 328, 0, 525, 449, 512, 517, 435, 428, 0, - 327, 514, 433, 427, 412, 373, 561, 413, 414, 387, - 464, 425, 465, 388, 439, 438, 440, 389, 390, 391, - 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, - 0, 0, 556, 557, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, - 0, 0, 693, 0, 528, 0, 0, 0, 0, 0, - 0, 497, 0, 0, 415, 0, 0, 0, 546, 0, - 480, 455, 731, 0, 0, 478, 423, 513, 466, 519, - 500, 527, 472, 467, 318, 501, 365, 436, 334, 336, - 721, 367, 370, 374, 375, 445, 446, 460, 485, 504, - 505, 506, 364, 348, 479, 349, 384, 350, 319, 356, - 354, 357, 487, 358, 321, 461, 510, 0, 380, 475, - 431, 322, 430, 462, 509, 508, 335, 536, 543, 544, - 634, 0, 549, 732, 733, 734, 558, 0, 468, 331, - 330, 0, 0, 0, 360, 463, 344, 346, 347, 345, - 458, 459, 563, 564, 565, 567, 0, 568, 569, 0, - 0, 0, 0, 570, 635, 651, 619, 588, 551, 643, - 585, 589, 590, 401, 402, 403, 654, 0, 0, 0, - 542, 416, 417, 0, 372, 371, 432, 323, 0, 0, - 409, 400, 469, 329, 368, 411, 405, 418, 419, 420, - 378, 313, 314, 727, 361, 451, 656, 691, 692, 581, - 0, 644, 582, 591, 353, 616, 628, 627, 447, 541, - 0, 639, 642, 571, 726, 0, 636, 650, 730, 649, - 723, 457, 0, 484, 647, 594, 0, 640, 613, 614, - 0, 641, 609, 645, 0, 583, 0, 552, 555, 584, - 669, 670, 671, 320, 554, 673, 674, 675, 676, 677, - 678, 679, 672, 524, 617, 593, 620, 533, 596, 595, - 0, 0, 631, 550, 632, 633, 441, 442, 443, 444, - 382, 657, 342, 553, 471, 0, 618, 0, 0, 0, - 0, 0, 0, 0, 0, 623, 624, 621, 735, 0, - 680, 681, 0, 0, 547, 548, 377, 0, 566, 385, - 341, 456, 379, 531, 408, 0, 559, 625, 560, 473, - 474, 683, 688, 684, 685, 687, 707, 448, 399, 404, - 488, 410, 424, 476, 530, 454, 481, 339, 520, 490, - 429, 610, 638, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, + 526, 0, 539, 0, 407, 408, 0, 0, 0, 0, + 0, 0, 0, 324, 504, 523, 338, 491, 537, 343, + 499, 516, 333, 457, 488, 0, 0, 326, 521, 498, + 439, 325, 0, 482, 366, 383, 363, 455, 0, 0, + 520, 550, 362, 540, 0, 531, 328, 0, 530, 454, + 517, 522, 440, 433, 0, 327, 519, 438, 432, 413, + 373, 566, 414, 415, 416, 417, 418, 419, 387, 469, + 430, 470, 388, 444, 443, 445, 389, 390, 391, 392, + 393, 394, 395, 396, 397, 398, 0, 0, 0, 0, + 0, 561, 562, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 694, 0, + 0, 698, 0, 533, 0, 0, 0, 0, 0, 0, + 502, 0, 0, 420, 0, 0, 0, 551, 0, 485, + 460, 736, 0, 0, 483, 428, 518, 471, 524, 505, + 532, 477, 472, 318, 506, 365, 441, 334, 336, 726, + 367, 370, 374, 375, 450, 451, 465, 490, 509, 510, + 511, 364, 348, 484, 349, 384, 350, 319, 356, 354, + 357, 492, 358, 321, 466, 515, 0, 380, 480, 436, + 322, 435, 467, 514, 513, 335, 541, 548, 549, 639, + 0, 554, 737, 738, 739, 563, 0, 473, 331, 330, + 0, 0, 0, 360, 468, 344, 346, 347, 345, 463, + 464, 568, 569, 570, 572, 0, 573, 574, 0, 0, + 0, 0, 575, 640, 656, 624, 593, 556, 648, 590, + 594, 595, 401, 402, 403, 404, 659, 0, 0, 0, + 547, 421, 422, 0, 372, 371, 437, 323, 0, 0, + 410, 400, 474, 329, 368, 412, 406, 423, 424, 425, + 378, 313, 314, 732, 361, 456, 661, 696, 697, 586, + 0, 649, 587, 596, 353, 621, 633, 632, 452, 546, + 0, 644, 647, 576, 731, 0, 641, 655, 735, 654, + 728, 462, 0, 489, 652, 599, 0, 645, 618, 619, + 0, 646, 614, 650, 0, 588, 0, 557, 560, 589, + 674, 675, 676, 320, 559, 678, 679, 680, 681, 682, + 683, 684, 677, 529, 622, 598, 625, 538, 601, 600, + 0, 0, 636, 555, 637, 638, 446, 447, 448, 449, + 382, 662, 342, 558, 476, 0, 623, 0, 0, 0, + 0, 0, 0, 0, 0, 628, 629, 626, 740, 0, + 685, 686, 0, 0, 552, 553, 377, 0, 571, 385, + 341, 461, 379, 536, 409, 0, 564, 630, 565, 478, + 479, 688, 693, 689, 690, 692, 712, 453, 399, 405, + 493, 411, 429, 481, 535, 459, 486, 339, 525, 495, + 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 665, 664, 663, 662, 661, 660, 659, 658, 0, 0, - 607, 507, 355, 307, 351, 352, 359, 724, 720, 725, - 708, 711, 710, 686, 0, 315, 587, 422, 470, 376, - 652, 653, 0, 706, 259, 260, 261, 262, 263, 264, + 670, 669, 668, 667, 666, 665, 664, 663, 0, 0, + 612, 512, 355, 307, 351, 352, 359, 729, 725, 730, + 713, 716, 715, 691, 0, 315, 592, 427, 475, 376, + 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, - 278, 279, 280, 281, 282, 283, 284, 285, 655, 276, + 278, 279, 280, 281, 282, 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, 0, 309, - 712, 713, 714, 715, 716, 0, 0, 310, 311, 312, - 0, 0, 274, 275, 302, 498, 303, 304, 305, 306, - 0, 0, 537, 538, 539, 562, 0, 540, 522, 586, - 386, 316, 502, 529, 722, 0, 0, 0, 0, 0, - 0, 0, 637, 648, 682, 0, 694, 695, 697, 699, - 698, 701, 495, 496, 709, 0, 0, 703, 704, 705, - 702, 426, 482, 503, 489, 0, 728, 577, 578, 729, - 690, 317, 453, 0, 0, 592, 626, 615, 700, 580, - 0, 0, 0, 0, 0, 2366, 0, 0, 0, 0, - 369, 0, 0, 421, 630, 611, 622, 612, 597, 598, - 599, 606, 381, 600, 601, 602, 572, 603, 573, 604, - 605, 0, 629, 579, 491, 437, 0, 646, 0, 0, + 717, 718, 719, 720, 721, 0, 0, 310, 311, 312, + 0, 0, 274, 275, 302, 503, 303, 304, 305, 306, + 0, 0, 542, 543, 544, 567, 0, 545, 527, 591, + 386, 316, 507, 534, 727, 0, 0, 0, 0, 0, + 0, 0, 642, 653, 687, 0, 699, 700, 702, 704, + 703, 706, 500, 501, 714, 0, 0, 708, 709, 710, + 707, 431, 487, 508, 494, 0, 733, 582, 583, 734, + 695, 317, 458, 0, 0, 597, 631, 620, 705, 585, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 369, 2846, 0, 426, 635, 616, 627, 617, 602, 603, + 604, 611, 381, 605, 606, 607, 577, 608, 578, 609, + 610, 0, 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 2367, 0, 0, 0, 337, 246, 574, 696, 576, - 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1734, 0, 0, 0, 337, 246, 579, 701, 581, + 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 492, 521, 0, 534, 0, 406, 407, 0, 0, 0, - 0, 0, 0, 0, 324, 499, 518, 338, 486, 532, - 343, 494, 511, 333, 452, 483, 0, 0, 326, 516, - 493, 434, 325, 0, 477, 366, 383, 363, 450, 0, - 0, 515, 545, 362, 535, 0, 526, 328, 0, 525, - 449, 512, 517, 435, 428, 0, 327, 514, 433, 427, - 412, 373, 561, 413, 414, 387, 464, 425, 465, 388, - 439, 438, 440, 389, 390, 391, 392, 393, 394, 395, - 396, 397, 398, 0, 0, 0, 0, 0, 556, 557, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 689, 0, 0, 693, 0, - 528, 0, 0, 0, 0, 0, 0, 497, 0, 0, - 415, 0, 0, 0, 546, 0, 480, 455, 731, 0, - 0, 478, 423, 513, 466, 519, 500, 527, 472, 467, - 318, 501, 365, 436, 334, 336, 721, 367, 370, 374, - 375, 445, 446, 460, 485, 504, 505, 506, 364, 348, - 479, 349, 384, 350, 319, 356, 354, 357, 487, 358, - 321, 461, 510, 0, 380, 475, 431, 322, 430, 462, - 509, 508, 335, 536, 543, 544, 634, 0, 549, 732, - 733, 734, 558, 0, 468, 331, 330, 0, 0, 0, - 360, 463, 344, 346, 347, 345, 458, 459, 563, 564, - 565, 567, 0, 568, 569, 0, 0, 0, 0, 570, - 635, 651, 619, 588, 551, 643, 585, 589, 590, 401, - 402, 403, 654, 0, 0, 0, 542, 416, 417, 0, - 372, 371, 432, 323, 0, 0, 409, 400, 469, 329, - 368, 411, 405, 418, 419, 420, 378, 313, 314, 727, - 361, 451, 656, 691, 692, 581, 0, 644, 582, 591, - 353, 616, 628, 627, 447, 541, 0, 639, 642, 571, - 726, 0, 636, 650, 730, 649, 723, 457, 0, 484, - 647, 594, 0, 640, 613, 614, 0, 641, 609, 645, - 0, 583, 0, 552, 555, 584, 669, 670, 671, 320, - 554, 673, 674, 675, 676, 677, 678, 679, 672, 524, - 617, 593, 620, 533, 596, 595, 0, 0, 631, 550, - 632, 633, 441, 442, 443, 444, 382, 657, 342, 553, - 471, 0, 618, 0, 0, 0, 0, 0, 0, 0, - 0, 623, 624, 621, 735, 0, 680, 681, 0, 0, - 547, 548, 377, 0, 566, 385, 341, 456, 379, 531, - 408, 0, 559, 625, 560, 473, 474, 683, 688, 684, - 685, 687, 707, 448, 399, 404, 488, 410, 424, 476, - 530, 454, 481, 339, 520, 490, 429, 610, 638, 0, + 497, 526, 0, 539, 0, 407, 408, 0, 0, 0, + 0, 0, 0, 0, 324, 504, 523, 338, 491, 537, + 343, 499, 516, 333, 457, 488, 0, 0, 326, 521, + 498, 439, 325, 0, 482, 366, 383, 363, 455, 0, + 0, 520, 550, 362, 540, 0, 531, 328, 0, 530, + 454, 517, 522, 440, 433, 0, 327, 519, 438, 432, + 413, 373, 566, 414, 415, 416, 417, 418, 419, 387, + 469, 430, 470, 388, 444, 443, 445, 389, 390, 391, + 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, + 0, 0, 561, 562, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, + 0, 0, 698, 0, 533, 0, 0, 0, 0, 0, + 0, 502, 0, 0, 420, 0, 0, 0, 551, 0, + 485, 460, 736, 0, 0, 483, 428, 518, 471, 524, + 505, 532, 477, 472, 318, 506, 365, 441, 334, 336, + 726, 367, 370, 374, 375, 450, 451, 465, 490, 509, + 510, 511, 364, 348, 484, 349, 384, 350, 319, 356, + 354, 357, 492, 358, 321, 466, 515, 0, 380, 480, + 436, 322, 435, 467, 514, 513, 335, 541, 548, 549, + 639, 0, 554, 737, 738, 739, 563, 0, 473, 331, + 330, 0, 0, 0, 360, 468, 344, 346, 347, 345, + 463, 464, 568, 569, 570, 572, 0, 573, 574, 0, + 0, 0, 0, 575, 640, 656, 624, 593, 556, 648, + 590, 594, 595, 401, 402, 403, 404, 659, 0, 0, + 0, 547, 421, 422, 0, 372, 371, 437, 323, 0, + 0, 410, 400, 474, 329, 368, 412, 406, 423, 424, + 425, 378, 313, 314, 732, 361, 456, 661, 696, 697, + 586, 0, 649, 587, 596, 353, 621, 633, 632, 452, + 546, 0, 644, 647, 576, 731, 0, 641, 655, 735, + 654, 728, 462, 0, 489, 652, 599, 0, 645, 618, + 619, 0, 646, 614, 650, 0, 588, 0, 557, 560, + 589, 674, 675, 676, 320, 559, 678, 679, 680, 681, + 682, 683, 684, 677, 529, 622, 598, 625, 538, 601, + 600, 0, 0, 636, 555, 637, 638, 446, 447, 448, + 449, 382, 662, 342, 558, 476, 0, 623, 0, 0, + 0, 0, 0, 0, 0, 0, 628, 629, 626, 740, + 0, 685, 686, 0, 0, 552, 553, 377, 0, 571, + 385, 341, 461, 379, 536, 409, 0, 564, 630, 565, + 478, 479, 688, 693, 689, 690, 692, 712, 453, 399, + 405, 493, 411, 429, 481, 535, 459, 486, 339, 525, + 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 665, 664, 663, 662, - 661, 660, 659, 658, 0, 0, 607, 507, 355, 307, - 351, 352, 359, 724, 720, 725, 708, 711, 710, 686, - 0, 315, 587, 422, 470, 376, 652, 653, 0, 706, - 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, - 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, - 282, 283, 284, 285, 655, 276, 277, 286, 287, 288, - 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, - 299, 0, 0, 0, 0, 309, 712, 713, 714, 715, - 716, 0, 0, 310, 311, 312, 0, 0, 274, 275, - 302, 498, 303, 304, 305, 306, 0, 0, 537, 538, - 539, 562, 0, 540, 522, 586, 386, 316, 502, 529, - 722, 0, 0, 0, 0, 0, 0, 0, 637, 648, - 682, 0, 694, 695, 697, 699, 698, 701, 495, 496, - 709, 0, 0, 703, 704, 705, 702, 426, 482, 503, - 489, 0, 728, 577, 578, 729, 690, 317, 453, 0, - 0, 592, 626, 615, 700, 580, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 369, 0, 0, 421, - 630, 611, 622, 612, 597, 598, 599, 606, 381, 600, - 601, 602, 572, 603, 573, 604, 605, 0, 629, 579, - 491, 437, 0, 646, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 245, 0, 0, 3567, 3569, 0, - 0, 337, 246, 574, 696, 576, 575, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, + 0, 670, 669, 668, 667, 666, 665, 664, 663, 0, + 0, 612, 512, 355, 307, 351, 352, 359, 729, 725, + 730, 713, 716, 715, 691, 0, 315, 592, 427, 475, + 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, + 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, + 273, 278, 279, 280, 281, 282, 283, 284, 285, 660, + 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 298, 299, 0, 0, 0, 0, + 309, 717, 718, 719, 720, 721, 0, 0, 310, 311, + 312, 0, 0, 274, 275, 302, 503, 303, 304, 305, + 306, 0, 0, 542, 543, 544, 567, 0, 545, 527, + 591, 386, 316, 507, 534, 727, 0, 0, 0, 0, + 0, 0, 0, 642, 653, 687, 0, 699, 700, 702, + 704, 703, 706, 500, 501, 714, 0, 0, 708, 709, + 710, 707, 431, 487, 508, 494, 0, 733, 582, 583, + 734, 695, 317, 458, 0, 0, 597, 631, 620, 705, + 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1075, 369, 0, 0, 426, 635, 616, 627, 617, 602, + 603, 604, 611, 381, 605, 606, 607, 577, 608, 578, + 609, 610, 0, 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, + 0, 0, 0, 0, 0, 0, 337, 246, 579, 701, + 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 492, 521, 0, 534, - 0, 406, 407, 0, 0, 0, 0, 0, 0, 0, - 324, 499, 518, 338, 486, 532, 343, 494, 511, 333, - 452, 483, 0, 0, 326, 516, 493, 434, 325, 0, - 477, 366, 383, 363, 450, 0, 0, 515, 545, 362, - 535, 0, 526, 328, 0, 525, 449, 512, 517, 435, - 428, 0, 327, 514, 433, 427, 412, 373, 561, 413, - 414, 387, 464, 425, 465, 388, 439, 438, 440, 389, - 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, - 0, 0, 0, 0, 556, 557, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 689, 0, 0, 693, 0, 528, 0, 0, 0, - 0, 0, 0, 497, 0, 0, 415, 0, 0, 0, - 546, 0, 480, 455, 731, 0, 0, 478, 423, 513, - 466, 519, 500, 527, 472, 467, 318, 501, 365, 436, - 334, 336, 721, 367, 370, 374, 375, 445, 446, 460, - 485, 504, 505, 506, 364, 348, 479, 349, 384, 350, - 319, 356, 354, 357, 487, 358, 321, 461, 510, 0, - 380, 475, 431, 322, 430, 462, 509, 508, 335, 536, - 543, 544, 634, 0, 549, 732, 733, 734, 558, 0, - 468, 331, 330, 0, 0, 0, 360, 463, 344, 346, - 347, 345, 458, 459, 563, 564, 565, 567, 0, 568, - 569, 0, 0, 0, 0, 570, 635, 651, 619, 588, - 551, 643, 585, 589, 590, 401, 402, 403, 654, 0, - 0, 0, 542, 416, 417, 0, 372, 371, 432, 323, - 0, 0, 409, 400, 469, 329, 368, 411, 405, 418, - 419, 420, 378, 313, 314, 727, 361, 451, 656, 691, - 692, 581, 0, 644, 582, 591, 353, 616, 628, 627, - 447, 541, 0, 639, 642, 571, 726, 0, 636, 650, - 730, 649, 723, 457, 0, 484, 647, 594, 0, 640, - 613, 614, 0, 641, 609, 645, 0, 583, 0, 552, - 555, 584, 669, 670, 671, 320, 554, 673, 674, 675, - 676, 677, 678, 679, 672, 524, 617, 593, 620, 533, - 596, 595, 0, 0, 631, 550, 632, 633, 441, 442, - 443, 444, 382, 657, 342, 553, 471, 0, 618, 0, - 0, 0, 0, 0, 0, 0, 0, 623, 624, 621, - 735, 0, 680, 681, 0, 0, 547, 548, 377, 0, - 566, 385, 341, 456, 379, 531, 408, 0, 559, 625, - 560, 473, 474, 683, 688, 684, 685, 687, 707, 448, - 399, 404, 488, 410, 424, 476, 530, 454, 481, 339, - 520, 490, 429, 610, 638, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 497, 526, 0, 539, 0, 407, 408, 0, 0, + 0, 0, 0, 0, 0, 324, 504, 523, 338, 491, + 537, 343, 499, 516, 333, 457, 488, 0, 0, 326, + 521, 498, 439, 325, 0, 482, 366, 383, 363, 455, + 0, 0, 520, 550, 362, 540, 0, 531, 328, 0, + 530, 454, 517, 522, 440, 433, 0, 327, 519, 438, + 432, 413, 373, 566, 414, 415, 416, 417, 418, 419, + 387, 469, 430, 470, 388, 444, 443, 445, 389, 390, + 391, 392, 393, 394, 395, 396, 397, 398, 0, 0, + 0, 0, 0, 561, 562, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 694, 0, 0, 698, 0, 533, 0, 1074, 0, 0, + 0, 0, 502, 0, 0, 420, 0, 0, 0, 551, + 0, 485, 460, 736, 0, 0, 483, 428, 518, 471, + 524, 505, 532, 477, 472, 318, 506, 365, 441, 334, + 336, 726, 367, 370, 374, 375, 450, 451, 465, 490, + 509, 510, 511, 364, 348, 484, 349, 384, 350, 319, + 356, 354, 357, 492, 358, 321, 466, 515, 0, 380, + 480, 436, 322, 435, 467, 514, 513, 335, 541, 548, + 549, 639, 0, 554, 737, 738, 739, 563, 0, 473, + 331, 330, 0, 0, 0, 360, 468, 344, 346, 347, + 345, 463, 464, 568, 569, 570, 572, 0, 573, 574, + 0, 0, 0, 0, 575, 640, 656, 624, 593, 556, + 648, 590, 594, 595, 401, 402, 403, 404, 659, 0, + 0, 0, 547, 421, 422, 0, 372, 371, 437, 323, + 0, 0, 410, 400, 474, 329, 368, 412, 406, 423, + 424, 425, 378, 313, 314, 732, 361, 456, 661, 696, + 697, 586, 0, 649, 587, 596, 353, 621, 633, 632, + 452, 546, 0, 644, 647, 576, 731, 0, 641, 655, + 735, 654, 728, 462, 0, 489, 652, 599, 0, 645, + 618, 619, 0, 646, 614, 650, 0, 588, 0, 557, + 560, 589, 674, 675, 676, 320, 559, 678, 679, 680, + 681, 682, 683, 684, 677, 529, 622, 598, 625, 538, + 601, 600, 0, 0, 636, 555, 637, 638, 446, 447, + 448, 449, 382, 662, 342, 558, 476, 0, 623, 0, + 0, 0, 0, 0, 0, 0, 0, 628, 629, 626, + 740, 0, 685, 686, 0, 0, 552, 553, 377, 0, + 571, 385, 341, 461, 379, 536, 409, 0, 564, 630, + 565, 478, 479, 688, 693, 689, 690, 692, 712, 453, + 399, 405, 493, 411, 429, 481, 535, 459, 486, 339, + 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 665, 664, 663, 662, 661, 660, 659, 658, - 0, 0, 607, 507, 355, 307, 351, 352, 359, 724, - 720, 725, 708, 711, 710, 686, 0, 315, 587, 422, - 470, 376, 652, 653, 0, 706, 259, 260, 261, 262, + 0, 0, 670, 669, 668, 667, 666, 665, 664, 663, + 0, 0, 612, 512, 355, 307, 351, 352, 359, 729, + 725, 730, 713, 716, 715, 691, 0, 315, 592, 427, + 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, 283, 284, 285, - 655, 276, 277, 286, 287, 288, 289, 290, 291, 292, + 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, - 0, 309, 712, 713, 714, 715, 716, 0, 0, 310, - 311, 312, 0, 0, 274, 275, 302, 498, 303, 304, - 305, 306, 0, 0, 537, 538, 539, 562, 0, 540, - 522, 586, 386, 316, 502, 529, 722, 0, 0, 0, - 0, 0, 0, 0, 637, 648, 682, 0, 694, 695, - 697, 699, 698, 701, 495, 496, 709, 0, 0, 703, - 704, 705, 702, 426, 482, 503, 489, 0, 728, 577, - 578, 729, 690, 317, 453, 0, 0, 592, 626, 615, - 700, 580, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 369, 2829, 0, 421, 630, 611, 622, 612, - 597, 598, 599, 606, 381, 600, 601, 602, 572, 603, - 573, 604, 605, 0, 629, 579, 491, 437, 0, 646, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 1723, 0, 0, 0, 337, 246, 574, - 696, 576, 575, 0, 0, 0, 0, 0, 0, 0, + 0, 309, 717, 718, 719, 720, 721, 0, 0, 310, + 311, 312, 0, 0, 274, 275, 302, 503, 303, 304, + 305, 306, 0, 0, 542, 543, 544, 567, 0, 545, + 527, 591, 386, 316, 507, 534, 727, 0, 0, 0, + 0, 0, 0, 0, 642, 653, 687, 0, 699, 700, + 702, 704, 703, 706, 500, 501, 714, 0, 0, 708, + 709, 710, 707, 431, 487, 508, 494, 0, 733, 582, + 583, 734, 695, 317, 458, 0, 0, 597, 631, 620, + 705, 585, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 369, 0, 0, 426, 635, 616, 627, 617, + 602, 603, 604, 611, 381, 605, 606, 607, 577, 608, + 578, 609, 610, 0, 634, 584, 496, 442, 0, 651, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 245, 950, 0, 0, 0, 0, 0, 337, 246, 579, + 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 492, 521, 0, 534, 0, 406, 407, 0, - 0, 0, 0, 0, 0, 0, 324, 499, 518, 338, - 486, 532, 343, 494, 511, 333, 452, 483, 0, 0, - 326, 516, 493, 434, 325, 0, 477, 366, 383, 363, - 450, 0, 0, 515, 545, 362, 535, 0, 526, 328, - 0, 525, 449, 512, 517, 435, 428, 0, 327, 514, - 433, 427, 412, 373, 561, 413, 414, 387, 464, 425, - 465, 388, 439, 438, 440, 389, 390, 391, 392, 393, - 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, - 556, 557, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 689, 0, 0, - 693, 0, 528, 0, 0, 0, 0, 0, 0, 497, - 0, 0, 415, 0, 0, 0, 546, 0, 480, 455, - 731, 0, 0, 478, 423, 513, 466, 519, 500, 527, - 472, 467, 318, 501, 365, 436, 334, 336, 721, 367, - 370, 374, 375, 445, 446, 460, 485, 504, 505, 506, - 364, 348, 479, 349, 384, 350, 319, 356, 354, 357, - 487, 358, 321, 461, 510, 0, 380, 475, 431, 322, - 430, 462, 509, 508, 335, 536, 543, 544, 634, 0, - 549, 732, 733, 734, 558, 0, 468, 331, 330, 0, - 0, 0, 360, 463, 344, 346, 347, 345, 458, 459, - 563, 564, 565, 567, 0, 568, 569, 0, 0, 0, - 0, 570, 635, 651, 619, 588, 551, 643, 585, 589, - 590, 401, 402, 403, 654, 0, 0, 0, 542, 416, - 417, 0, 372, 371, 432, 323, 0, 0, 409, 400, - 469, 329, 368, 411, 405, 418, 419, 420, 378, 313, - 314, 727, 361, 451, 656, 691, 692, 581, 0, 644, - 582, 591, 353, 616, 628, 627, 447, 541, 0, 639, - 642, 571, 726, 0, 636, 650, 730, 649, 723, 457, - 0, 484, 647, 594, 0, 640, 613, 614, 0, 641, - 609, 645, 0, 583, 0, 552, 555, 584, 669, 670, - 671, 320, 554, 673, 674, 675, 676, 677, 678, 679, - 672, 524, 617, 593, 620, 533, 596, 595, 0, 0, - 631, 550, 632, 633, 441, 442, 443, 444, 382, 657, - 342, 553, 471, 0, 618, 0, 0, 0, 0, 0, - 0, 0, 0, 623, 624, 621, 735, 0, 680, 681, - 0, 0, 547, 548, 377, 0, 566, 385, 341, 456, - 379, 531, 408, 0, 559, 625, 560, 473, 474, 683, - 688, 684, 685, 687, 707, 448, 399, 404, 488, 410, - 424, 476, 530, 454, 481, 339, 520, 490, 429, 610, - 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 665, 664, - 663, 662, 661, 660, 659, 658, 0, 0, 607, 507, - 355, 307, 351, 352, 359, 724, 720, 725, 708, 711, - 710, 686, 0, 315, 587, 422, 470, 376, 652, 653, - 0, 706, 259, 260, 261, 262, 263, 264, 265, 266, - 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, - 280, 281, 282, 283, 284, 285, 655, 276, 277, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 298, 299, 0, 0, 0, 0, 309, 712, 713, - 714, 715, 716, 0, 0, 310, 311, 312, 0, 0, - 274, 275, 302, 498, 303, 304, 305, 306, 0, 0, - 537, 538, 539, 562, 0, 540, 522, 586, 386, 316, - 502, 529, 722, 0, 0, 0, 0, 0, 0, 0, - 637, 648, 682, 0, 694, 695, 697, 699, 698, 701, - 495, 496, 709, 0, 0, 703, 704, 705, 702, 426, - 482, 503, 489, 0, 728, 577, 578, 729, 690, 317, - 453, 0, 0, 592, 626, 615, 700, 580, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1069, 369, 0, - 0, 421, 630, 611, 622, 612, 597, 598, 599, 606, - 381, 600, 601, 602, 572, 603, 573, 604, 605, 0, - 629, 579, 491, 437, 0, 646, 0, 0, 0, 0, + 0, 0, 497, 526, 0, 539, 0, 407, 408, 0, + 0, 0, 0, 0, 0, 0, 324, 504, 523, 338, + 491, 537, 343, 499, 516, 333, 457, 488, 0, 0, + 326, 521, 498, 439, 325, 0, 482, 366, 383, 363, + 455, 0, 0, 520, 550, 362, 540, 0, 531, 328, + 0, 530, 454, 517, 522, 440, 433, 0, 327, 519, + 438, 432, 413, 373, 566, 414, 415, 416, 417, 418, + 419, 387, 469, 430, 470, 388, 444, 443, 445, 389, + 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, + 0, 0, 0, 0, 561, 562, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 694, 0, 0, 698, 0, 533, 0, 0, 0, + 0, 0, 0, 502, 0, 0, 420, 0, 0, 0, + 551, 0, 485, 460, 736, 0, 0, 483, 428, 518, + 471, 524, 505, 532, 477, 472, 318, 506, 365, 441, + 334, 336, 726, 367, 370, 374, 375, 450, 451, 465, + 490, 509, 510, 511, 364, 348, 484, 349, 384, 350, + 319, 356, 354, 357, 492, 358, 321, 466, 515, 0, + 380, 480, 436, 322, 435, 467, 514, 513, 335, 541, + 548, 549, 639, 0, 554, 737, 738, 739, 563, 0, + 473, 331, 330, 0, 0, 0, 360, 468, 344, 346, + 347, 345, 463, 464, 568, 569, 570, 572, 0, 573, + 574, 0, 0, 0, 0, 575, 640, 656, 624, 593, + 556, 648, 590, 594, 595, 401, 402, 403, 404, 659, + 0, 0, 0, 547, 421, 422, 0, 372, 371, 437, + 323, 0, 0, 410, 400, 474, 329, 368, 412, 406, + 423, 424, 425, 378, 313, 314, 732, 361, 456, 661, + 696, 697, 586, 0, 649, 587, 596, 353, 621, 633, + 632, 452, 546, 0, 644, 647, 576, 731, 0, 641, + 655, 735, 654, 728, 462, 0, 489, 652, 599, 0, + 645, 618, 619, 0, 646, 614, 650, 0, 588, 0, + 557, 560, 589, 674, 675, 676, 320, 559, 678, 679, + 680, 681, 682, 683, 684, 677, 529, 622, 598, 625, + 538, 601, 600, 0, 0, 636, 555, 637, 638, 446, + 447, 448, 449, 382, 662, 342, 558, 476, 0, 623, + 0, 0, 0, 0, 0, 0, 0, 0, 628, 629, + 626, 740, 0, 685, 686, 0, 0, 552, 553, 377, + 0, 571, 385, 341, 461, 379, 536, 409, 0, 564, + 630, 565, 478, 479, 688, 693, 689, 690, 692, 712, + 453, 399, 405, 493, 411, 429, 481, 535, 459, 486, + 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, - 0, 0, 0, 337, 246, 574, 696, 576, 575, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, + 0, 0, 0, 670, 669, 668, 667, 666, 665, 664, + 663, 0, 0, 612, 512, 355, 307, 351, 352, 359, + 729, 725, 730, 713, 716, 715, 691, 0, 315, 592, + 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, + 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, + 271, 272, 273, 278, 279, 280, 281, 282, 283, 284, + 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 298, 299, 0, 0, + 0, 0, 309, 717, 718, 719, 720, 721, 0, 0, + 310, 311, 312, 0, 0, 274, 275, 302, 503, 303, + 304, 305, 306, 0, 0, 542, 543, 544, 567, 0, + 545, 527, 591, 386, 316, 507, 534, 727, 0, 0, + 0, 0, 0, 0, 0, 642, 653, 687, 0, 699, + 700, 702, 704, 703, 706, 500, 501, 714, 0, 0, + 708, 709, 710, 707, 431, 487, 508, 494, 0, 733, + 582, 583, 734, 695, 317, 458, 0, 0, 597, 631, + 620, 705, 585, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 369, 0, 0, 426, 635, 616, 627, + 617, 602, 603, 604, 611, 381, 605, 606, 607, 577, + 608, 578, 609, 610, 0, 634, 584, 496, 442, 0, + 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 4675, 0, + 0, 245, 0, 0, 0, 0, 0, 0, 337, 246, + 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 492, 521, - 0, 534, 0, 406, 407, 0, 0, 0, 0, 0, - 0, 0, 324, 499, 518, 338, 486, 532, 343, 494, - 511, 333, 452, 483, 0, 0, 326, 516, 493, 434, - 325, 0, 477, 366, 383, 363, 450, 0, 0, 515, - 545, 362, 535, 0, 526, 328, 0, 525, 449, 512, - 517, 435, 428, 0, 327, 514, 433, 427, 412, 373, - 561, 413, 414, 387, 464, 425, 465, 388, 439, 438, - 440, 389, 390, 391, 392, 393, 394, 395, 396, 397, - 398, 0, 0, 0, 0, 0, 556, 557, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 689, 0, 0, 693, 0, 528, 0, - 1068, 0, 0, 0, 0, 497, 0, 0, 415, 0, - 0, 0, 546, 0, 480, 455, 731, 0, 0, 478, - 423, 513, 466, 519, 500, 527, 472, 467, 318, 501, - 365, 436, 334, 336, 721, 367, 370, 374, 375, 445, - 446, 460, 485, 504, 505, 506, 364, 348, 479, 349, - 384, 350, 319, 356, 354, 357, 487, 358, 321, 461, - 510, 0, 380, 475, 431, 322, 430, 462, 509, 508, - 335, 536, 543, 544, 634, 0, 549, 732, 733, 734, - 558, 0, 468, 331, 330, 0, 0, 0, 360, 463, - 344, 346, 347, 345, 458, 459, 563, 564, 565, 567, - 0, 568, 569, 0, 0, 0, 0, 570, 635, 651, - 619, 588, 551, 643, 585, 589, 590, 401, 402, 403, - 654, 0, 0, 0, 542, 416, 417, 0, 372, 371, - 432, 323, 0, 0, 409, 400, 469, 329, 368, 411, - 405, 418, 419, 420, 378, 313, 314, 727, 361, 451, - 656, 691, 692, 581, 0, 644, 582, 591, 353, 616, - 628, 627, 447, 541, 0, 639, 642, 571, 726, 0, - 636, 650, 730, 649, 723, 457, 0, 484, 647, 594, - 0, 640, 613, 614, 0, 641, 609, 645, 0, 583, - 0, 552, 555, 584, 669, 670, 671, 320, 554, 673, - 674, 675, 676, 677, 678, 679, 672, 524, 617, 593, - 620, 533, 596, 595, 0, 0, 631, 550, 632, 633, - 441, 442, 443, 444, 382, 657, 342, 553, 471, 0, - 618, 0, 0, 0, 0, 0, 0, 0, 0, 623, - 624, 621, 735, 0, 680, 681, 0, 0, 547, 548, - 377, 0, 566, 385, 341, 456, 379, 531, 408, 0, - 559, 625, 560, 473, 474, 683, 688, 684, 685, 687, - 707, 448, 399, 404, 488, 410, 424, 476, 530, 454, - 481, 339, 520, 490, 429, 610, 638, 0, 0, 0, + 0, 0, 0, 497, 526, 0, 539, 0, 407, 408, + 0, 0, 0, 0, 0, 0, 0, 324, 504, 523, + 338, 491, 537, 343, 499, 516, 333, 457, 488, 0, + 0, 326, 521, 498, 439, 325, 0, 482, 366, 383, + 363, 455, 0, 0, 520, 550, 362, 540, 0, 531, + 328, 0, 530, 454, 517, 522, 440, 433, 0, 327, + 519, 438, 432, 413, 373, 566, 414, 415, 416, 417, + 418, 419, 387, 469, 430, 470, 388, 444, 443, 445, + 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, + 0, 0, 0, 0, 0, 561, 562, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 694, 0, 0, 698, 0, 533, 0, 0, + 0, 0, 0, 0, 502, 0, 0, 420, 0, 0, + 0, 551, 0, 485, 460, 736, 0, 0, 483, 428, + 518, 471, 524, 505, 532, 477, 472, 318, 506, 365, + 441, 334, 336, 726, 367, 370, 374, 375, 450, 451, + 465, 490, 509, 510, 511, 364, 348, 484, 349, 384, + 350, 319, 356, 354, 357, 492, 358, 321, 466, 515, + 0, 380, 480, 436, 322, 435, 467, 514, 513, 335, + 541, 548, 549, 639, 0, 554, 737, 738, 739, 563, + 0, 473, 331, 330, 0, 0, 0, 360, 468, 344, + 346, 347, 345, 463, 464, 568, 569, 570, 572, 0, + 573, 574, 0, 0, 0, 0, 575, 640, 656, 624, + 593, 556, 648, 590, 594, 595, 401, 402, 403, 404, + 659, 0, 0, 0, 547, 421, 422, 0, 372, 371, + 437, 323, 0, 0, 410, 400, 474, 329, 368, 412, + 406, 423, 424, 425, 378, 313, 314, 732, 361, 456, + 661, 696, 697, 586, 0, 649, 587, 596, 353, 621, + 633, 632, 452, 546, 0, 644, 647, 576, 731, 0, + 641, 655, 735, 654, 728, 462, 0, 489, 652, 599, + 0, 645, 618, 619, 0, 646, 614, 650, 0, 588, + 0, 557, 560, 589, 674, 675, 676, 320, 559, 678, + 679, 680, 681, 682, 683, 684, 677, 529, 622, 598, + 625, 538, 601, 600, 0, 0, 636, 555, 637, 638, + 446, 447, 448, 449, 382, 662, 342, 558, 476, 0, + 623, 0, 0, 0, 0, 0, 0, 0, 0, 628, + 629, 626, 740, 0, 685, 686, 0, 0, 552, 553, + 377, 0, 571, 385, 341, 461, 379, 536, 409, 0, + 564, 630, 565, 478, 479, 688, 693, 689, 690, 692, + 712, 453, 399, 405, 493, 411, 429, 481, 535, 459, + 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 665, 664, 663, 662, 661, 660, - 659, 658, 0, 0, 607, 507, 355, 307, 351, 352, - 359, 724, 720, 725, 708, 711, 710, 686, 0, 315, - 587, 422, 470, 376, 652, 653, 0, 706, 259, 260, + 0, 0, 0, 0, 670, 669, 668, 667, 666, 665, + 664, 663, 0, 0, 612, 512, 355, 307, 351, 352, + 359, 729, 725, 730, 713, 716, 715, 691, 0, 315, + 592, 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, 283, - 284, 285, 655, 276, 277, 286, 287, 288, 289, 290, + 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, - 0, 0, 0, 309, 712, 713, 714, 715, 716, 0, - 0, 310, 311, 312, 0, 0, 274, 275, 302, 498, - 303, 304, 305, 306, 0, 0, 537, 538, 539, 562, - 0, 540, 522, 586, 386, 316, 502, 529, 722, 0, - 0, 0, 0, 0, 0, 0, 637, 648, 682, 0, - 694, 695, 697, 699, 698, 701, 495, 496, 709, 0, - 0, 703, 704, 705, 702, 426, 482, 503, 489, 0, - 728, 577, 578, 729, 690, 317, 453, 0, 0, 592, - 626, 615, 700, 580, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 369, 0, 0, 421, 630, 611, - 622, 612, 597, 598, 599, 606, 381, 600, 601, 602, - 572, 603, 573, 604, 605, 0, 629, 579, 491, 437, - 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 245, 944, 0, 0, 0, 0, 0, 337, - 246, 574, 696, 576, 575, 0, 0, 0, 0, 0, + 0, 0, 0, 309, 717, 718, 719, 720, 721, 0, + 0, 310, 311, 312, 0, 0, 274, 275, 302, 503, + 303, 304, 305, 306, 0, 0, 542, 543, 544, 567, + 0, 545, 527, 591, 386, 316, 507, 534, 727, 0, + 0, 0, 0, 0, 0, 0, 642, 653, 687, 0, + 699, 700, 702, 704, 703, 706, 500, 501, 714, 0, + 0, 708, 709, 710, 707, 431, 487, 508, 494, 0, + 733, 582, 583, 734, 695, 317, 458, 0, 0, 597, + 631, 620, 705, 585, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 369, 0, 0, 426, 635, 616, + 627, 617, 602, 603, 604, 611, 381, 605, 606, 607, + 577, 608, 578, 609, 610, 0, 634, 584, 496, 442, + 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 245, 0, 0, 4381, 0, 0, 0, 337, + 246, 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 492, 521, 0, 534, 0, 406, - 407, 0, 0, 0, 0, 0, 0, 0, 324, 499, - 518, 338, 486, 532, 343, 494, 511, 333, 452, 483, - 0, 0, 326, 516, 493, 434, 325, 0, 477, 366, - 383, 363, 450, 0, 0, 515, 545, 362, 535, 0, - 526, 328, 0, 525, 449, 512, 517, 435, 428, 0, - 327, 514, 433, 427, 412, 373, 561, 413, 414, 387, - 464, 425, 465, 388, 439, 438, 440, 389, 390, 391, - 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, - 0, 0, 556, 557, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, - 0, 0, 693, 0, 528, 0, 0, 0, 0, 0, - 0, 497, 0, 0, 415, 0, 0, 0, 546, 0, - 480, 455, 731, 0, 0, 478, 423, 513, 466, 519, - 500, 527, 472, 467, 318, 501, 365, 436, 334, 336, - 721, 367, 370, 374, 375, 445, 446, 460, 485, 504, - 505, 506, 364, 348, 479, 349, 384, 350, 319, 356, - 354, 357, 487, 358, 321, 461, 510, 0, 380, 475, - 431, 322, 430, 462, 509, 508, 335, 536, 543, 544, - 634, 0, 549, 732, 733, 734, 558, 0, 468, 331, - 330, 0, 0, 0, 360, 463, 344, 346, 347, 345, - 458, 459, 563, 564, 565, 567, 0, 568, 569, 0, - 0, 0, 0, 570, 635, 651, 619, 588, 551, 643, - 585, 589, 590, 401, 402, 403, 654, 0, 0, 0, - 542, 416, 417, 0, 372, 371, 432, 323, 0, 0, - 409, 400, 469, 329, 368, 411, 405, 418, 419, 420, - 378, 313, 314, 727, 361, 451, 656, 691, 692, 581, - 0, 644, 582, 591, 353, 616, 628, 627, 447, 541, - 0, 639, 642, 571, 726, 0, 636, 650, 730, 649, - 723, 457, 0, 484, 647, 594, 0, 640, 613, 614, - 0, 641, 609, 645, 0, 583, 0, 552, 555, 584, - 669, 670, 671, 320, 554, 673, 674, 675, 676, 677, - 678, 679, 672, 524, 617, 593, 620, 533, 596, 595, - 0, 0, 631, 550, 632, 633, 441, 442, 443, 444, - 382, 657, 342, 553, 471, 0, 618, 0, 0, 0, - 0, 0, 0, 0, 0, 623, 624, 621, 735, 0, - 680, 681, 0, 0, 547, 548, 377, 0, 566, 385, - 341, 456, 379, 531, 408, 0, 559, 625, 560, 473, - 474, 683, 688, 684, 685, 687, 707, 448, 399, 404, - 488, 410, 424, 476, 530, 454, 481, 339, 520, 490, - 429, 610, 638, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, + 0, 0, 0, 0, 497, 526, 0, 539, 0, 407, + 408, 0, 0, 0, 0, 0, 0, 0, 324, 504, + 523, 338, 491, 537, 343, 499, 516, 333, 457, 488, + 0, 0, 326, 521, 498, 439, 325, 0, 482, 366, + 383, 363, 455, 0, 0, 520, 550, 362, 540, 0, + 531, 328, 0, 530, 454, 517, 522, 440, 433, 0, + 327, 519, 438, 432, 413, 373, 566, 414, 415, 416, + 417, 418, 419, 387, 469, 430, 470, 388, 444, 443, + 445, 389, 390, 391, 392, 393, 394, 395, 396, 397, + 398, 0, 0, 0, 0, 0, 561, 562, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 694, 0, 0, 698, 0, 533, 0, + 0, 0, 0, 0, 0, 502, 0, 0, 420, 0, + 0, 0, 551, 0, 485, 460, 736, 0, 0, 483, + 428, 518, 471, 524, 505, 532, 477, 472, 318, 506, + 365, 441, 334, 336, 726, 367, 370, 374, 375, 450, + 451, 465, 490, 509, 510, 511, 364, 348, 484, 349, + 384, 350, 319, 356, 354, 357, 492, 358, 321, 466, + 515, 0, 380, 480, 436, 322, 435, 467, 514, 513, + 335, 541, 548, 549, 639, 0, 554, 737, 738, 739, + 563, 0, 473, 331, 330, 0, 0, 0, 360, 468, + 344, 346, 347, 345, 463, 464, 568, 569, 570, 572, + 0, 573, 574, 0, 0, 0, 0, 575, 640, 656, + 624, 593, 556, 648, 590, 594, 595, 401, 402, 403, + 404, 659, 0, 0, 0, 547, 421, 422, 0, 372, + 371, 437, 323, 0, 0, 410, 400, 474, 329, 368, + 412, 406, 423, 424, 425, 378, 313, 314, 732, 361, + 456, 661, 696, 697, 586, 0, 649, 587, 596, 353, + 621, 633, 632, 452, 546, 0, 644, 647, 576, 731, + 0, 641, 655, 735, 654, 728, 462, 0, 489, 652, + 599, 0, 645, 618, 619, 0, 646, 614, 650, 0, + 588, 0, 557, 560, 589, 674, 675, 676, 320, 559, + 678, 679, 680, 681, 682, 683, 684, 677, 529, 622, + 598, 625, 538, 601, 600, 0, 0, 636, 555, 637, + 638, 446, 447, 448, 449, 382, 662, 342, 558, 476, + 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, + 628, 629, 626, 740, 0, 685, 686, 0, 0, 552, + 553, 377, 0, 571, 385, 341, 461, 379, 536, 409, + 0, 564, 630, 565, 478, 479, 688, 693, 689, 690, + 692, 712, 453, 399, 405, 493, 411, 429, 481, 535, + 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, + 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 670, 669, 668, 667, 666, + 665, 664, 663, 0, 0, 612, 512, 355, 307, 351, + 352, 359, 729, 725, 730, 713, 716, 715, 691, 0, + 315, 592, 427, 475, 376, 657, 658, 0, 711, 259, + 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, + 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, + 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, + 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, + 0, 0, 0, 0, 309, 717, 718, 719, 720, 721, + 0, 0, 310, 311, 312, 0, 0, 274, 275, 302, + 503, 303, 304, 305, 306, 0, 0, 542, 543, 544, + 567, 0, 545, 527, 591, 386, 316, 507, 534, 727, + 0, 0, 0, 0, 0, 0, 0, 642, 653, 687, + 0, 699, 700, 702, 704, 703, 706, 500, 501, 714, + 0, 0, 708, 709, 710, 707, 431, 487, 508, 494, + 0, 733, 582, 583, 734, 695, 317, 458, 0, 0, + 597, 631, 620, 705, 585, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 369, 0, 0, 426, 635, + 616, 627, 617, 602, 603, 604, 611, 381, 605, 606, + 607, 577, 608, 578, 609, 610, 0, 634, 584, 496, + 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 665, 664, 663, 662, 661, 660, 659, 658, 0, 0, - 607, 507, 355, 307, 351, 352, 359, 724, 720, 725, - 708, 711, 710, 686, 0, 315, 587, 422, 470, 376, - 652, 653, 0, 706, 259, 260, 261, 262, 263, 264, - 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, - 278, 279, 280, 281, 282, 283, 284, 285, 655, 276, - 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 298, 299, 0, 0, 0, 0, 309, - 712, 713, 714, 715, 716, 0, 0, 310, 311, 312, - 0, 0, 274, 275, 302, 498, 303, 304, 305, 306, - 0, 0, 537, 538, 539, 562, 0, 540, 522, 586, - 386, 316, 502, 529, 722, 0, 0, 0, 0, 0, - 0, 0, 637, 648, 682, 0, 694, 695, 697, 699, - 698, 701, 495, 496, 709, 0, 0, 703, 704, 705, - 702, 426, 482, 503, 489, 0, 728, 577, 578, 729, - 690, 317, 453, 0, 0, 592, 626, 615, 700, 580, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 369, 0, 0, 421, 630, 611, 622, 612, 597, 598, - 599, 606, 381, 600, 601, 602, 572, 603, 573, 604, - 605, 0, 629, 579, 491, 437, 0, 646, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 4642, 0, 0, 245, 0, - 0, 0, 0, 0, 0, 337, 246, 574, 696, 576, - 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, + 337, 246, 579, 701, 581, 580, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 492, 521, 0, 534, 0, 406, 407, 0, 0, 0, - 0, 0, 0, 0, 324, 499, 518, 338, 486, 532, - 343, 494, 511, 333, 452, 483, 0, 0, 326, 516, - 493, 434, 325, 0, 477, 366, 383, 363, 450, 0, - 0, 515, 545, 362, 535, 0, 526, 328, 0, 525, - 449, 512, 517, 435, 428, 0, 327, 514, 433, 427, - 412, 373, 561, 413, 414, 387, 464, 425, 465, 388, - 439, 438, 440, 389, 390, 391, 392, 393, 394, 395, - 396, 397, 398, 0, 0, 0, 0, 0, 556, 557, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 689, 0, 0, 693, 0, - 528, 0, 0, 0, 0, 0, 0, 497, 0, 0, - 415, 0, 0, 0, 546, 0, 480, 455, 731, 0, - 0, 478, 423, 513, 466, 519, 500, 527, 472, 467, - 318, 501, 365, 436, 334, 336, 721, 367, 370, 374, - 375, 445, 446, 460, 485, 504, 505, 506, 364, 348, - 479, 349, 384, 350, 319, 356, 354, 357, 487, 358, - 321, 461, 510, 0, 380, 475, 431, 322, 430, 462, - 509, 508, 335, 536, 543, 544, 634, 0, 549, 732, - 733, 734, 558, 0, 468, 331, 330, 0, 0, 0, - 360, 463, 344, 346, 347, 345, 458, 459, 563, 564, - 565, 567, 0, 568, 569, 0, 0, 0, 0, 570, - 635, 651, 619, 588, 551, 643, 585, 589, 590, 401, - 402, 403, 654, 0, 0, 0, 542, 416, 417, 0, - 372, 371, 432, 323, 0, 0, 409, 400, 469, 329, - 368, 411, 405, 418, 419, 420, 378, 313, 314, 727, - 361, 451, 656, 691, 692, 581, 0, 644, 582, 591, - 353, 616, 628, 627, 447, 541, 0, 639, 642, 571, - 726, 0, 636, 650, 730, 649, 723, 457, 0, 484, - 647, 594, 0, 640, 613, 614, 0, 641, 609, 645, - 0, 583, 0, 552, 555, 584, 669, 670, 671, 320, - 554, 673, 674, 675, 676, 677, 678, 679, 672, 524, - 617, 593, 620, 533, 596, 595, 0, 0, 631, 550, - 632, 633, 441, 442, 443, 444, 382, 657, 342, 553, - 471, 0, 618, 0, 0, 0, 0, 0, 0, 0, - 0, 623, 624, 621, 735, 0, 680, 681, 0, 0, - 547, 548, 377, 0, 566, 385, 341, 456, 379, 531, - 408, 0, 559, 625, 560, 473, 474, 683, 688, 684, - 685, 687, 707, 448, 399, 404, 488, 410, 424, 476, - 530, 454, 481, 339, 520, 490, 429, 610, 638, 0, + 0, 0, 0, 0, 0, 497, 526, 0, 539, 0, + 407, 408, 0, 0, 0, 0, 0, 0, 0, 324, + 504, 523, 338, 491, 537, 343, 499, 516, 333, 457, + 488, 0, 0, 326, 521, 498, 439, 325, 0, 482, + 366, 383, 363, 455, 0, 0, 520, 550, 362, 540, + 0, 531, 328, 0, 530, 454, 517, 522, 440, 433, + 0, 327, 519, 438, 432, 413, 373, 566, 414, 415, + 416, 417, 418, 419, 387, 469, 430, 470, 388, 444, + 443, 445, 389, 390, 391, 392, 393, 394, 395, 396, + 397, 398, 0, 0, 0, 0, 0, 561, 562, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 694, 0, 0, 698, 0, 533, + 0, 0, 0, 4572, 0, 0, 502, 0, 0, 420, + 0, 0, 0, 551, 0, 485, 460, 736, 0, 0, + 483, 428, 518, 471, 524, 505, 532, 477, 472, 318, + 506, 365, 441, 334, 336, 726, 367, 370, 374, 375, + 450, 451, 465, 490, 509, 510, 511, 364, 348, 484, + 349, 384, 350, 319, 356, 354, 357, 492, 358, 321, + 466, 515, 0, 380, 480, 436, 322, 435, 467, 514, + 513, 335, 541, 548, 549, 639, 0, 554, 737, 738, + 739, 563, 0, 473, 331, 330, 0, 0, 0, 360, + 468, 344, 346, 347, 345, 463, 464, 568, 569, 570, + 572, 0, 573, 574, 0, 0, 0, 0, 575, 640, + 656, 624, 593, 556, 648, 590, 594, 595, 401, 402, + 403, 404, 659, 0, 0, 0, 547, 421, 422, 0, + 372, 371, 437, 323, 0, 0, 410, 400, 474, 329, + 368, 412, 406, 423, 424, 425, 378, 313, 314, 732, + 361, 456, 661, 696, 697, 586, 0, 649, 587, 596, + 353, 621, 633, 632, 452, 546, 0, 644, 647, 576, + 731, 0, 641, 655, 735, 654, 728, 462, 0, 489, + 652, 599, 0, 645, 618, 619, 0, 646, 614, 650, + 0, 588, 0, 557, 560, 589, 674, 675, 676, 320, + 559, 678, 679, 680, 681, 682, 683, 684, 677, 529, + 622, 598, 625, 538, 601, 600, 0, 0, 636, 555, + 637, 638, 446, 447, 448, 449, 382, 662, 342, 558, + 476, 0, 623, 0, 0, 0, 0, 0, 0, 0, + 0, 628, 629, 626, 740, 0, 685, 686, 0, 0, + 552, 553, 377, 0, 571, 385, 341, 461, 379, 536, + 409, 0, 564, 630, 565, 478, 479, 688, 693, 689, + 690, 692, 712, 453, 399, 405, 493, 411, 429, 481, + 535, 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 665, 664, 663, 662, - 661, 660, 659, 658, 0, 0, 607, 507, 355, 307, - 351, 352, 359, 724, 720, 725, 708, 711, 710, 686, - 0, 315, 587, 422, 470, 376, 652, 653, 0, 706, + 0, 0, 0, 0, 0, 0, 670, 669, 668, 667, + 666, 665, 664, 663, 0, 0, 612, 512, 355, 307, + 351, 352, 359, 729, 725, 730, 713, 716, 715, 691, + 0, 315, 592, 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, - 282, 283, 284, 285, 655, 276, 277, 286, 287, 288, + 282, 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, - 299, 0, 0, 0, 0, 309, 712, 713, 714, 715, - 716, 0, 0, 310, 311, 312, 0, 0, 274, 275, - 302, 498, 303, 304, 305, 306, 0, 0, 537, 538, - 539, 562, 0, 540, 522, 586, 386, 316, 502, 529, - 722, 0, 0, 0, 0, 0, 0, 0, 637, 648, - 682, 0, 694, 695, 697, 699, 698, 701, 495, 496, - 709, 0, 0, 703, 704, 705, 702, 426, 482, 503, - 489, 0, 728, 577, 578, 729, 690, 317, 453, 0, - 0, 592, 626, 615, 700, 580, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 369, 0, 0, 421, - 630, 611, 622, 612, 597, 598, 599, 606, 381, 600, - 601, 602, 572, 603, 573, 604, 605, 0, 629, 579, - 491, 437, 0, 646, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 245, 0, 0, 4350, 0, 0, - 0, 337, 246, 574, 696, 576, 575, 0, 0, 0, + 299, 0, 0, 0, 0, 309, 717, 718, 719, 720, + 721, 0, 0, 310, 311, 312, 0, 0, 274, 275, + 302, 503, 303, 304, 305, 306, 0, 0, 542, 543, + 544, 567, 0, 545, 527, 591, 386, 316, 507, 534, + 727, 0, 0, 0, 0, 0, 0, 0, 642, 653, + 687, 0, 699, 700, 702, 704, 703, 706, 500, 501, + 714, 0, 0, 708, 709, 710, 707, 431, 487, 508, + 494, 0, 733, 582, 583, 734, 695, 317, 458, 0, + 0, 597, 631, 620, 705, 585, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 369, 0, 0, 426, + 635, 616, 627, 617, 602, 603, 604, 611, 381, 605, + 606, 607, 577, 608, 578, 609, 610, 0, 634, 584, + 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1965, 0, 0, 245, 0, 0, 0, 0, 0, + 0, 337, 246, 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 492, 521, 0, 534, - 0, 406, 407, 0, 0, 0, 0, 0, 0, 0, - 324, 499, 518, 338, 486, 532, 343, 494, 511, 333, - 452, 483, 0, 0, 326, 516, 493, 434, 325, 0, - 477, 366, 383, 363, 450, 0, 0, 515, 545, 362, - 535, 0, 526, 328, 0, 525, 449, 512, 517, 435, - 428, 0, 327, 514, 433, 427, 412, 373, 561, 413, - 414, 387, 464, 425, 465, 388, 439, 438, 440, 389, - 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, - 0, 0, 0, 0, 556, 557, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 689, 0, 0, 693, 0, 528, 0, 0, 0, - 0, 0, 0, 497, 0, 0, 415, 0, 0, 0, - 546, 0, 480, 455, 731, 0, 0, 478, 423, 513, - 466, 519, 500, 527, 472, 467, 318, 501, 365, 436, - 334, 336, 721, 367, 370, 374, 375, 445, 446, 460, - 485, 504, 505, 506, 364, 348, 479, 349, 384, 350, - 319, 356, 354, 357, 487, 358, 321, 461, 510, 0, - 380, 475, 431, 322, 430, 462, 509, 508, 335, 536, - 543, 544, 634, 0, 549, 732, 733, 734, 558, 0, - 468, 331, 330, 0, 0, 0, 360, 463, 344, 346, - 347, 345, 458, 459, 563, 564, 565, 567, 0, 568, - 569, 0, 0, 0, 0, 570, 635, 651, 619, 588, - 551, 643, 585, 589, 590, 401, 402, 403, 654, 0, - 0, 0, 542, 416, 417, 0, 372, 371, 432, 323, - 0, 0, 409, 400, 469, 329, 368, 411, 405, 418, - 419, 420, 378, 313, 314, 727, 361, 451, 656, 691, - 692, 581, 0, 644, 582, 591, 353, 616, 628, 627, - 447, 541, 0, 639, 642, 571, 726, 0, 636, 650, - 730, 649, 723, 457, 0, 484, 647, 594, 0, 640, - 613, 614, 0, 641, 609, 645, 0, 583, 0, 552, - 555, 584, 669, 670, 671, 320, 554, 673, 674, 675, - 676, 677, 678, 679, 672, 524, 617, 593, 620, 533, - 596, 595, 0, 0, 631, 550, 632, 633, 441, 442, - 443, 444, 382, 657, 342, 553, 471, 0, 618, 0, - 0, 0, 0, 0, 0, 0, 0, 623, 624, 621, - 735, 0, 680, 681, 0, 0, 547, 548, 377, 0, - 566, 385, 341, 456, 379, 531, 408, 0, 559, 625, - 560, 473, 474, 683, 688, 684, 685, 687, 707, 448, - 399, 404, 488, 410, 424, 476, 530, 454, 481, 339, - 520, 490, 429, 610, 638, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 665, 664, 663, 662, 661, 660, 659, 658, - 0, 0, 607, 507, 355, 307, 351, 352, 359, 724, - 720, 725, 708, 711, 710, 686, 0, 315, 587, 422, - 470, 376, 652, 653, 0, 706, 259, 260, 261, 262, - 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, - 272, 273, 278, 279, 280, 281, 282, 283, 284, 285, - 655, 276, 277, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, - 0, 309, 712, 713, 714, 715, 716, 0, 0, 310, - 311, 312, 0, 0, 274, 275, 302, 498, 303, 304, - 305, 306, 0, 0, 537, 538, 539, 562, 0, 540, - 522, 586, 386, 316, 502, 529, 722, 0, 0, 0, - 0, 0, 0, 0, 637, 648, 682, 0, 694, 695, - 697, 699, 698, 701, 495, 496, 709, 0, 0, 703, - 704, 705, 702, 426, 482, 503, 489, 0, 728, 577, - 578, 729, 690, 317, 453, 0, 0, 592, 626, 615, - 700, 580, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 369, 0, 0, 421, 630, 611, 622, 612, - 597, 598, 599, 606, 381, 600, 601, 602, 572, 603, - 573, 604, 605, 0, 629, 579, 491, 437, 0, 646, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 0, 0, 0, 0, 337, 246, 574, - 696, 576, 575, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 497, 526, 0, 539, + 0, 407, 408, 0, 0, 0, 0, 0, 0, 0, + 324, 504, 523, 338, 491, 537, 343, 499, 516, 333, + 457, 488, 0, 0, 326, 521, 498, 439, 325, 0, + 482, 366, 383, 363, 455, 0, 0, 520, 550, 362, + 540, 0, 531, 328, 0, 530, 454, 517, 522, 440, + 433, 0, 327, 519, 438, 432, 413, 373, 566, 414, + 415, 416, 417, 418, 419, 387, 469, 430, 470, 388, + 444, 443, 445, 389, 390, 391, 392, 393, 394, 395, + 396, 397, 398, 0, 0, 0, 0, 0, 561, 562, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 694, 0, 0, 698, 0, + 533, 0, 0, 0, 0, 0, 0, 502, 0, 0, + 420, 0, 0, 0, 551, 0, 485, 460, 736, 0, + 0, 483, 428, 518, 471, 524, 505, 532, 477, 472, + 318, 506, 365, 441, 334, 336, 726, 367, 370, 374, + 375, 450, 451, 465, 490, 509, 510, 511, 364, 348, + 484, 349, 384, 350, 319, 356, 354, 357, 492, 358, + 321, 466, 515, 0, 380, 480, 436, 322, 435, 467, + 514, 513, 335, 541, 548, 549, 639, 0, 554, 737, + 738, 739, 563, 0, 473, 331, 330, 0, 0, 0, + 360, 468, 344, 346, 347, 345, 463, 464, 568, 569, + 570, 572, 0, 573, 574, 0, 0, 0, 0, 575, + 640, 656, 624, 593, 556, 648, 590, 594, 595, 401, + 402, 403, 404, 659, 0, 0, 0, 547, 421, 422, + 0, 372, 371, 437, 323, 0, 0, 410, 400, 474, + 329, 368, 412, 406, 423, 424, 425, 378, 313, 314, + 732, 361, 456, 661, 696, 697, 586, 0, 649, 587, + 596, 353, 621, 633, 632, 452, 546, 0, 644, 647, + 576, 731, 0, 641, 655, 735, 654, 728, 462, 0, + 489, 652, 599, 0, 645, 618, 619, 0, 646, 614, + 650, 0, 588, 0, 557, 560, 589, 674, 675, 676, + 320, 559, 678, 679, 680, 681, 682, 683, 684, 677, + 529, 622, 598, 625, 538, 601, 600, 0, 0, 636, + 555, 637, 638, 446, 447, 448, 449, 382, 662, 342, + 558, 476, 0, 623, 0, 0, 0, 0, 0, 0, + 0, 0, 628, 629, 626, 740, 0, 685, 686, 0, + 0, 552, 553, 377, 0, 571, 385, 341, 461, 379, + 536, 409, 0, 564, 630, 565, 478, 479, 688, 693, + 689, 690, 692, 712, 453, 399, 405, 493, 411, 429, + 481, 535, 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 492, 521, 0, 534, 0, 406, 407, 0, - 0, 0, 0, 0, 0, 0, 324, 499, 518, 338, - 486, 532, 343, 494, 511, 333, 452, 483, 0, 0, - 326, 516, 493, 434, 325, 0, 477, 366, 383, 363, - 450, 0, 0, 515, 545, 362, 535, 0, 526, 328, - 0, 525, 449, 512, 517, 435, 428, 0, 327, 514, - 433, 427, 412, 373, 561, 413, 414, 387, 464, 425, - 465, 388, 439, 438, 440, 389, 390, 391, 392, 393, - 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, - 556, 557, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 689, 0, 0, - 693, 0, 528, 0, 0, 0, 4539, 0, 0, 497, - 0, 0, 415, 0, 0, 0, 546, 0, 480, 455, - 731, 0, 0, 478, 423, 513, 466, 519, 500, 527, - 472, 467, 318, 501, 365, 436, 334, 336, 721, 367, - 370, 374, 375, 445, 446, 460, 485, 504, 505, 506, - 364, 348, 479, 349, 384, 350, 319, 356, 354, 357, - 487, 358, 321, 461, 510, 0, 380, 475, 431, 322, - 430, 462, 509, 508, 335, 536, 543, 544, 634, 0, - 549, 732, 733, 734, 558, 0, 468, 331, 330, 0, - 0, 0, 360, 463, 344, 346, 347, 345, 458, 459, - 563, 564, 565, 567, 0, 568, 569, 0, 0, 0, - 0, 570, 635, 651, 619, 588, 551, 643, 585, 589, - 590, 401, 402, 403, 654, 0, 0, 0, 542, 416, - 417, 0, 372, 371, 432, 323, 0, 0, 409, 400, - 469, 329, 368, 411, 405, 418, 419, 420, 378, 313, - 314, 727, 361, 451, 656, 691, 692, 581, 0, 644, - 582, 591, 353, 616, 628, 627, 447, 541, 0, 639, - 642, 571, 726, 0, 636, 650, 730, 649, 723, 457, - 0, 484, 647, 594, 0, 640, 613, 614, 0, 641, - 609, 645, 0, 583, 0, 552, 555, 584, 669, 670, - 671, 320, 554, 673, 674, 675, 676, 677, 678, 679, - 672, 524, 617, 593, 620, 533, 596, 595, 0, 0, - 631, 550, 632, 633, 441, 442, 443, 444, 382, 657, - 342, 553, 471, 0, 618, 0, 0, 0, 0, 0, - 0, 0, 0, 623, 624, 621, 735, 0, 680, 681, - 0, 0, 547, 548, 377, 0, 566, 385, 341, 456, - 379, 531, 408, 0, 559, 625, 560, 473, 474, 683, - 688, 684, 685, 687, 707, 448, 399, 404, 488, 410, - 424, 476, 530, 454, 481, 339, 520, 490, 429, 610, - 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 670, 669, 668, + 667, 666, 665, 664, 663, 0, 0, 612, 512, 355, + 307, 351, 352, 359, 729, 725, 730, 713, 716, 715, + 691, 0, 315, 592, 427, 475, 376, 657, 658, 0, + 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, + 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, + 281, 282, 283, 284, 285, 660, 276, 277, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 298, 299, 0, 0, 0, 0, 309, 717, 718, 719, + 720, 721, 0, 0, 310, 311, 312, 0, 0, 274, + 275, 302, 503, 303, 304, 305, 306, 0, 0, 542, + 543, 544, 567, 0, 545, 527, 591, 386, 316, 507, + 534, 727, 0, 0, 0, 0, 0, 0, 0, 642, + 653, 687, 0, 699, 700, 702, 704, 703, 706, 500, + 501, 714, 0, 0, 708, 709, 710, 707, 431, 487, + 508, 494, 0, 733, 582, 583, 734, 695, 317, 458, + 0, 0, 597, 631, 620, 705, 585, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, + 426, 635, 616, 627, 617, 602, 603, 604, 611, 381, + 605, 606, 607, 577, 608, 578, 609, 610, 0, 634, + 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 4396, 0, 245, 0, 0, 0, 0, + 0, 0, 337, 246, 579, 701, 581, 580, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 497, 526, 0, + 539, 0, 407, 408, 0, 0, 0, 0, 0, 0, + 0, 324, 504, 523, 338, 491, 537, 343, 499, 516, + 333, 457, 488, 0, 0, 326, 521, 498, 439, 325, + 0, 482, 366, 383, 363, 455, 0, 0, 520, 550, + 362, 540, 0, 531, 328, 0, 530, 454, 517, 522, + 440, 433, 0, 327, 519, 438, 432, 413, 373, 566, + 414, 415, 416, 417, 418, 419, 387, 469, 430, 470, + 388, 444, 443, 445, 389, 390, 391, 392, 393, 394, + 395, 396, 397, 398, 0, 0, 0, 0, 0, 561, + 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 694, 0, 0, 698, + 0, 533, 0, 0, 0, 0, 0, 0, 502, 0, + 0, 420, 0, 0, 0, 551, 0, 485, 460, 736, + 0, 0, 483, 428, 518, 471, 524, 505, 532, 477, + 472, 318, 506, 365, 441, 334, 336, 726, 367, 370, + 374, 375, 450, 451, 465, 490, 509, 510, 511, 364, + 348, 484, 349, 384, 350, 319, 356, 354, 357, 492, + 358, 321, 466, 515, 0, 380, 480, 436, 322, 435, + 467, 514, 513, 335, 541, 548, 549, 639, 0, 554, + 737, 738, 739, 563, 0, 473, 331, 330, 0, 0, + 0, 360, 468, 344, 346, 347, 345, 463, 464, 568, + 569, 570, 572, 0, 573, 574, 0, 0, 0, 0, + 575, 640, 656, 624, 593, 556, 648, 590, 594, 595, + 401, 402, 403, 404, 659, 0, 0, 0, 547, 421, + 422, 0, 372, 371, 437, 323, 0, 0, 410, 400, + 474, 329, 368, 412, 406, 423, 424, 425, 378, 313, + 314, 732, 361, 456, 661, 696, 697, 586, 0, 649, + 587, 596, 353, 621, 633, 632, 452, 546, 0, 644, + 647, 576, 731, 0, 641, 655, 735, 654, 728, 462, + 0, 489, 652, 599, 0, 645, 618, 619, 0, 646, + 614, 650, 0, 588, 0, 557, 560, 589, 674, 675, + 676, 320, 559, 678, 679, 680, 681, 682, 683, 684, + 677, 529, 622, 598, 625, 538, 601, 600, 0, 0, + 636, 555, 637, 638, 446, 447, 448, 449, 382, 662, + 342, 558, 476, 0, 623, 0, 0, 0, 0, 0, + 0, 0, 0, 628, 629, 626, 740, 0, 685, 686, + 0, 0, 552, 553, 377, 0, 571, 385, 341, 461, + 379, 536, 409, 0, 564, 630, 565, 478, 479, 688, + 693, 689, 690, 692, 712, 453, 399, 405, 493, 411, + 429, 481, 535, 459, 486, 339, 525, 495, 434, 615, + 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 665, 664, - 663, 662, 661, 660, 659, 658, 0, 0, 607, 507, - 355, 307, 351, 352, 359, 724, 720, 725, 708, 711, - 710, 686, 0, 315, 587, 422, 470, 376, 652, 653, - 0, 706, 259, 260, 261, 262, 263, 264, 265, 266, + 0, 0, 0, 0, 0, 0, 0, 0, 670, 669, + 668, 667, 666, 665, 664, 663, 0, 0, 612, 512, + 355, 307, 351, 352, 359, 729, 725, 730, 713, 716, + 715, 691, 0, 315, 592, 427, 475, 376, 657, 658, + 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, - 280, 281, 282, 283, 284, 285, 655, 276, 277, 286, + 280, 281, 282, 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 298, 299, 0, 0, 0, 0, 309, 712, 713, - 714, 715, 716, 0, 0, 310, 311, 312, 0, 0, - 274, 275, 302, 498, 303, 304, 305, 306, 0, 0, - 537, 538, 539, 562, 0, 540, 522, 586, 386, 316, - 502, 529, 722, 0, 0, 0, 0, 0, 0, 0, - 637, 648, 682, 0, 694, 695, 697, 699, 698, 701, - 495, 496, 709, 0, 0, 703, 704, 705, 702, 426, - 482, 503, 489, 0, 728, 577, 578, 729, 690, 317, - 453, 0, 0, 592, 626, 615, 700, 580, 0, 0, + 297, 298, 299, 0, 0, 0, 0, 309, 717, 718, + 719, 720, 721, 0, 0, 310, 311, 312, 0, 0, + 274, 275, 302, 503, 303, 304, 305, 306, 0, 0, + 542, 543, 544, 567, 0, 545, 527, 591, 386, 316, + 507, 534, 727, 0, 0, 0, 0, 0, 0, 0, + 642, 653, 687, 0, 699, 700, 702, 704, 703, 706, + 500, 501, 714, 0, 0, 708, 709, 710, 707, 431, + 487, 508, 494, 0, 733, 582, 583, 734, 695, 317, + 458, 0, 0, 597, 631, 620, 705, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, - 0, 421, 630, 611, 622, 612, 597, 598, 599, 606, - 381, 600, 601, 602, 572, 603, 573, 604, 605, 0, - 629, 579, 491, 437, 0, 646, 0, 0, 0, 0, + 0, 426, 635, 616, 627, 617, 602, 603, 604, 611, + 381, 605, 606, 607, 577, 608, 578, 609, 610, 0, + 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1950, 0, 0, 245, 0, 0, 0, - 0, 0, 0, 337, 246, 574, 696, 576, 575, 0, + 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, + 0, 0, 0, 337, 246, 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 492, 521, - 0, 534, 0, 406, 407, 0, 0, 0, 0, 0, - 0, 0, 324, 499, 518, 338, 486, 532, 343, 494, - 511, 333, 452, 483, 0, 0, 326, 516, 493, 434, - 325, 0, 477, 366, 383, 363, 450, 0, 0, 515, - 545, 362, 535, 0, 526, 328, 0, 525, 449, 512, - 517, 435, 428, 0, 327, 514, 433, 427, 412, 373, - 561, 413, 414, 387, 464, 425, 465, 388, 439, 438, - 440, 389, 390, 391, 392, 393, 394, 395, 396, 397, - 398, 0, 0, 0, 0, 0, 556, 557, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 689, 0, 0, 693, 0, 528, 0, - 0, 0, 0, 0, 0, 497, 0, 0, 415, 0, - 0, 0, 546, 0, 480, 455, 731, 0, 0, 478, - 423, 513, 466, 519, 500, 527, 472, 467, 318, 501, - 365, 436, 334, 336, 721, 367, 370, 374, 375, 445, - 446, 460, 485, 504, 505, 506, 364, 348, 479, 349, - 384, 350, 319, 356, 354, 357, 487, 358, 321, 461, - 510, 0, 380, 475, 431, 322, 430, 462, 509, 508, - 335, 536, 543, 544, 634, 0, 549, 732, 733, 734, - 558, 0, 468, 331, 330, 0, 0, 0, 360, 463, - 344, 346, 347, 345, 458, 459, 563, 564, 565, 567, - 0, 568, 569, 0, 0, 0, 0, 570, 635, 651, - 619, 588, 551, 643, 585, 589, 590, 401, 402, 403, - 654, 0, 0, 0, 542, 416, 417, 0, 372, 371, - 432, 323, 0, 0, 409, 400, 469, 329, 368, 411, - 405, 418, 419, 420, 378, 313, 314, 727, 361, 451, - 656, 691, 692, 581, 0, 644, 582, 591, 353, 616, - 628, 627, 447, 541, 0, 639, 642, 571, 726, 0, - 636, 650, 730, 649, 723, 457, 0, 484, 647, 594, - 0, 640, 613, 614, 0, 641, 609, 645, 0, 583, - 0, 552, 555, 584, 669, 670, 671, 320, 554, 673, - 674, 675, 676, 677, 678, 679, 672, 524, 617, 593, - 620, 533, 596, 595, 0, 0, 631, 550, 632, 633, - 441, 442, 443, 444, 382, 657, 342, 553, 471, 0, - 618, 0, 0, 0, 0, 0, 0, 0, 0, 623, - 624, 621, 735, 0, 680, 681, 0, 0, 547, 548, - 377, 0, 566, 385, 341, 456, 379, 531, 408, 0, - 559, 625, 560, 473, 474, 683, 688, 684, 685, 687, - 707, 448, 399, 404, 488, 410, 424, 476, 530, 454, - 481, 339, 520, 490, 429, 610, 638, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, + 0, 0, 0, 0, 0, 0, 0, 0, 497, 526, + 0, 539, 0, 407, 408, 0, 0, 0, 0, 0, + 0, 0, 324, 504, 523, 338, 491, 537, 343, 499, + 516, 333, 457, 488, 0, 0, 326, 521, 498, 439, + 325, 0, 482, 366, 383, 363, 455, 0, 0, 520, + 550, 362, 540, 0, 531, 328, 0, 530, 454, 517, + 522, 440, 433, 0, 327, 519, 438, 432, 413, 373, + 566, 414, 415, 416, 417, 418, 419, 387, 469, 430, + 470, 388, 444, 443, 445, 389, 390, 391, 392, 393, + 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, + 561, 562, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 694, 0, 0, + 698, 0, 533, 0, 0, 0, 4287, 0, 0, 502, + 0, 0, 420, 0, 0, 0, 551, 0, 485, 460, + 736, 0, 0, 483, 428, 518, 471, 524, 505, 532, + 477, 472, 318, 506, 365, 441, 334, 336, 726, 367, + 370, 374, 375, 450, 451, 465, 490, 509, 510, 511, + 364, 348, 484, 349, 384, 350, 319, 356, 354, 357, + 492, 358, 321, 466, 515, 0, 380, 480, 436, 322, + 435, 467, 514, 513, 335, 541, 548, 549, 639, 0, + 554, 737, 738, 739, 563, 0, 473, 331, 330, 0, + 0, 0, 360, 468, 344, 346, 347, 345, 463, 464, + 568, 569, 570, 572, 0, 573, 574, 0, 0, 0, + 0, 575, 640, 656, 624, 593, 556, 648, 590, 594, + 595, 401, 402, 403, 404, 659, 0, 0, 0, 547, + 421, 422, 0, 372, 371, 437, 323, 0, 0, 410, + 400, 474, 329, 368, 412, 406, 423, 424, 425, 378, + 313, 314, 732, 361, 456, 661, 696, 697, 586, 0, + 649, 587, 596, 353, 621, 633, 632, 452, 546, 0, + 644, 647, 576, 731, 0, 641, 655, 735, 654, 728, + 462, 0, 489, 652, 599, 0, 645, 618, 619, 0, + 646, 614, 650, 0, 588, 0, 557, 560, 589, 674, + 675, 676, 320, 559, 678, 679, 680, 681, 682, 683, + 684, 677, 529, 622, 598, 625, 538, 601, 600, 0, + 0, 636, 555, 637, 638, 446, 447, 448, 449, 382, + 662, 342, 558, 476, 0, 623, 0, 0, 0, 0, + 0, 0, 0, 0, 628, 629, 626, 740, 0, 685, + 686, 0, 0, 552, 553, 377, 0, 571, 385, 341, + 461, 379, 536, 409, 0, 564, 630, 565, 478, 479, + 688, 693, 689, 690, 692, 712, 453, 399, 405, 493, + 411, 429, 481, 535, 459, 486, 339, 525, 495, 434, + 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 670, + 669, 668, 667, 666, 665, 664, 663, 0, 0, 612, + 512, 355, 307, 351, 352, 359, 729, 725, 730, 713, + 716, 715, 691, 0, 315, 592, 427, 475, 376, 657, + 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, + 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, + 279, 280, 281, 282, 283, 284, 285, 660, 276, 277, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 298, 299, 0, 0, 0, 0, 309, 717, + 718, 719, 720, 721, 0, 0, 310, 311, 312, 0, + 0, 274, 275, 302, 503, 303, 304, 305, 306, 0, + 0, 542, 543, 544, 567, 0, 545, 527, 591, 386, + 316, 507, 534, 727, 0, 0, 0, 0, 0, 0, + 0, 642, 653, 687, 0, 699, 700, 702, 704, 703, + 706, 500, 501, 714, 0, 0, 708, 709, 710, 707, + 431, 487, 508, 494, 0, 733, 582, 583, 734, 695, + 317, 458, 0, 0, 597, 631, 620, 705, 585, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, + 0, 0, 426, 635, 616, 627, 617, 602, 603, 604, + 611, 381, 605, 606, 607, 577, 608, 578, 609, 610, + 0, 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 665, 664, 663, 662, 661, 660, - 659, 658, 0, 0, 607, 507, 355, 307, 351, 352, - 359, 724, 720, 725, 708, 711, 710, 686, 0, 315, - 587, 422, 470, 376, 652, 653, 0, 706, 259, 260, - 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, - 270, 271, 272, 273, 278, 279, 280, 281, 282, 283, - 284, 285, 655, 276, 277, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, - 0, 0, 0, 309, 712, 713, 714, 715, 716, 0, - 0, 310, 311, 312, 0, 0, 274, 275, 302, 498, - 303, 304, 305, 306, 0, 0, 537, 538, 539, 562, - 0, 540, 522, 586, 386, 316, 502, 529, 722, 0, - 0, 0, 0, 0, 0, 0, 637, 648, 682, 0, - 694, 695, 697, 699, 698, 701, 495, 496, 709, 0, - 0, 703, 704, 705, 702, 426, 482, 503, 489, 0, - 728, 577, 578, 729, 690, 317, 453, 0, 0, 592, - 626, 615, 700, 580, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 369, 0, 0, 421, 630, 611, - 622, 612, 597, 598, 599, 606, 381, 600, 601, 602, - 572, 603, 573, 604, 605, 0, 629, 579, 491, 437, - 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 4365, 0, 245, 0, 0, 0, 0, 0, 0, 337, - 246, 574, 696, 576, 575, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, + 3627, 0, 0, 0, 337, 246, 579, 701, 581, 580, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 492, 521, 0, 534, 0, 406, - 407, 0, 0, 0, 0, 0, 0, 0, 324, 499, - 518, 338, 486, 532, 343, 494, 511, 333, 452, 483, - 0, 0, 326, 516, 493, 434, 325, 0, 477, 366, - 383, 363, 450, 0, 0, 515, 545, 362, 535, 0, - 526, 328, 0, 525, 449, 512, 517, 435, 428, 0, - 327, 514, 433, 427, 412, 373, 561, 413, 414, 387, - 464, 425, 465, 388, 439, 438, 440, 389, 390, 391, - 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, - 0, 0, 556, 557, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, - 0, 0, 693, 0, 528, 0, 0, 0, 0, 0, - 0, 497, 0, 0, 415, 0, 0, 0, 546, 0, - 480, 455, 731, 0, 0, 478, 423, 513, 466, 519, - 500, 527, 472, 467, 318, 501, 365, 436, 334, 336, - 721, 367, 370, 374, 375, 445, 446, 460, 485, 504, - 505, 506, 364, 348, 479, 349, 384, 350, 319, 356, - 354, 357, 487, 358, 321, 461, 510, 0, 380, 475, - 431, 322, 430, 462, 509, 508, 335, 536, 543, 544, - 634, 0, 549, 732, 733, 734, 558, 0, 468, 331, - 330, 0, 0, 0, 360, 463, 344, 346, 347, 345, - 458, 459, 563, 564, 565, 567, 0, 568, 569, 0, - 0, 0, 0, 570, 635, 651, 619, 588, 551, 643, - 585, 589, 590, 401, 402, 403, 654, 0, 0, 0, - 542, 416, 417, 0, 372, 371, 432, 323, 0, 0, - 409, 400, 469, 329, 368, 411, 405, 418, 419, 420, - 378, 313, 314, 727, 361, 451, 656, 691, 692, 581, - 0, 644, 582, 591, 353, 616, 628, 627, 447, 541, - 0, 639, 642, 571, 726, 0, 636, 650, 730, 649, - 723, 457, 0, 484, 647, 594, 0, 640, 613, 614, - 0, 641, 609, 645, 0, 583, 0, 552, 555, 584, - 669, 670, 671, 320, 554, 673, 674, 675, 676, 677, - 678, 679, 672, 524, 617, 593, 620, 533, 596, 595, - 0, 0, 631, 550, 632, 633, 441, 442, 443, 444, - 382, 657, 342, 553, 471, 0, 618, 0, 0, 0, - 0, 0, 0, 0, 0, 623, 624, 621, 735, 0, - 680, 681, 0, 0, 547, 548, 377, 0, 566, 385, - 341, 456, 379, 531, 408, 0, 559, 625, 560, 473, - 474, 683, 688, 684, 685, 687, 707, 448, 399, 404, - 488, 410, 424, 476, 530, 454, 481, 339, 520, 490, - 429, 610, 638, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, + 526, 0, 539, 0, 407, 408, 0, 0, 0, 0, + 0, 0, 0, 324, 504, 523, 338, 491, 537, 343, + 499, 516, 333, 457, 488, 0, 0, 326, 521, 498, + 439, 325, 0, 482, 366, 383, 363, 455, 0, 0, + 520, 550, 362, 540, 0, 531, 328, 0, 530, 454, + 517, 522, 440, 433, 0, 327, 519, 438, 432, 413, + 373, 566, 414, 415, 416, 417, 418, 419, 387, 469, + 430, 470, 388, 444, 443, 445, 389, 390, 391, 392, + 393, 394, 395, 396, 397, 398, 0, 0, 0, 0, + 0, 561, 562, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 694, 0, + 0, 698, 0, 533, 0, 0, 0, 0, 0, 0, + 502, 0, 0, 420, 0, 0, 0, 551, 0, 485, + 460, 736, 0, 0, 483, 428, 518, 471, 524, 505, + 532, 477, 472, 318, 506, 365, 441, 334, 336, 726, + 367, 370, 374, 375, 450, 451, 465, 490, 509, 510, + 511, 364, 348, 484, 349, 384, 350, 319, 356, 354, + 357, 492, 358, 321, 466, 515, 0, 380, 480, 436, + 322, 435, 467, 514, 513, 335, 541, 548, 549, 639, + 0, 554, 737, 738, 739, 563, 0, 473, 331, 330, + 0, 0, 0, 360, 468, 344, 346, 347, 345, 463, + 464, 568, 569, 570, 572, 0, 573, 574, 0, 0, + 0, 0, 575, 640, 656, 624, 593, 556, 648, 590, + 594, 595, 401, 402, 403, 404, 659, 0, 0, 0, + 547, 421, 422, 0, 372, 371, 437, 323, 0, 0, + 410, 400, 474, 329, 368, 412, 406, 423, 424, 425, + 378, 313, 314, 732, 361, 456, 661, 696, 697, 586, + 0, 649, 587, 596, 353, 621, 633, 632, 452, 546, + 0, 644, 647, 576, 731, 0, 641, 655, 735, 654, + 728, 462, 0, 489, 652, 599, 0, 645, 618, 619, + 0, 646, 614, 650, 0, 588, 0, 557, 560, 589, + 674, 675, 676, 320, 559, 678, 679, 680, 681, 682, + 683, 684, 677, 529, 622, 598, 625, 538, 601, 600, + 0, 0, 636, 555, 637, 638, 446, 447, 448, 449, + 382, 662, 342, 558, 476, 0, 623, 0, 0, 0, + 0, 0, 0, 0, 0, 628, 629, 626, 740, 0, + 685, 686, 0, 0, 552, 553, 377, 0, 571, 385, + 341, 461, 379, 536, 409, 0, 564, 630, 565, 478, + 479, 688, 693, 689, 690, 692, 712, 453, 399, 405, + 493, 411, 429, 481, 535, 459, 486, 339, 525, 495, + 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 665, 664, 663, 662, 661, 660, 659, 658, 0, 0, - 607, 507, 355, 307, 351, 352, 359, 724, 720, 725, - 708, 711, 710, 686, 0, 315, 587, 422, 470, 376, - 652, 653, 0, 706, 259, 260, 261, 262, 263, 264, + 670, 669, 668, 667, 666, 665, 664, 663, 0, 0, + 612, 512, 355, 307, 351, 352, 359, 729, 725, 730, + 713, 716, 715, 691, 0, 315, 592, 427, 475, 376, + 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, - 278, 279, 280, 281, 282, 283, 284, 285, 655, 276, + 278, 279, 280, 281, 282, 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, 0, 309, - 712, 713, 714, 715, 716, 0, 0, 310, 311, 312, - 0, 0, 274, 275, 302, 498, 303, 304, 305, 306, - 0, 0, 537, 538, 539, 562, 0, 540, 522, 586, - 386, 316, 502, 529, 722, 0, 0, 0, 0, 0, - 0, 0, 637, 648, 682, 0, 694, 695, 697, 699, - 698, 701, 495, 496, 709, 0, 0, 703, 704, 705, - 702, 426, 482, 503, 489, 0, 728, 577, 578, 729, - 690, 317, 453, 0, 0, 592, 626, 615, 700, 580, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 369, 0, 0, 421, 630, 611, 622, 612, 597, 598, - 599, 606, 381, 600, 601, 602, 572, 603, 573, 604, - 605, 0, 629, 579, 491, 437, 0, 646, 0, 0, + 717, 718, 719, 720, 721, 0, 0, 310, 311, 312, + 0, 0, 274, 275, 302, 503, 303, 304, 305, 306, + 0, 0, 542, 543, 544, 567, 0, 545, 527, 591, + 386, 316, 507, 534, 727, 0, 0, 0, 0, 0, + 0, 0, 642, 653, 687, 0, 699, 700, 702, 704, + 703, 706, 500, 501, 714, 0, 0, 708, 709, 710, + 707, 431, 487, 508, 494, 0, 733, 582, 583, 734, + 695, 317, 458, 0, 0, 597, 631, 620, 705, 585, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 369, 0, 0, 426, 635, 616, 627, 617, 602, 603, + 604, 611, 381, 605, 606, 607, 577, 608, 578, 609, + 610, 0, 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 0, 0, 0, 0, 337, 246, 574, 696, 576, - 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 4115, 0, 0, 0, 337, 246, 579, 701, 581, + 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 492, 521, 0, 534, 0, 406, 407, 0, 0, 0, - 0, 0, 0, 0, 324, 499, 518, 338, 486, 532, - 343, 494, 511, 333, 452, 483, 0, 0, 326, 516, - 493, 434, 325, 0, 477, 366, 383, 363, 450, 0, - 0, 515, 545, 362, 535, 0, 526, 328, 0, 525, - 449, 512, 517, 435, 428, 0, 327, 514, 433, 427, - 412, 373, 561, 413, 414, 387, 464, 425, 465, 388, - 439, 438, 440, 389, 390, 391, 392, 393, 394, 395, - 396, 397, 398, 0, 0, 0, 0, 0, 556, 557, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 689, 0, 0, 693, 0, - 528, 0, 0, 0, 4256, 0, 0, 497, 0, 0, - 415, 0, 0, 0, 546, 0, 480, 455, 731, 0, - 0, 478, 423, 513, 466, 519, 500, 527, 472, 467, - 318, 501, 365, 436, 334, 336, 721, 367, 370, 374, - 375, 445, 446, 460, 485, 504, 505, 506, 364, 348, - 479, 349, 384, 350, 319, 356, 354, 357, 487, 358, - 321, 461, 510, 0, 380, 475, 431, 322, 430, 462, - 509, 508, 335, 536, 543, 544, 634, 0, 549, 732, - 733, 734, 558, 0, 468, 331, 330, 0, 0, 0, - 360, 463, 344, 346, 347, 345, 458, 459, 563, 564, - 565, 567, 0, 568, 569, 0, 0, 0, 0, 570, - 635, 651, 619, 588, 551, 643, 585, 589, 590, 401, - 402, 403, 654, 0, 0, 0, 542, 416, 417, 0, - 372, 371, 432, 323, 0, 0, 409, 400, 469, 329, - 368, 411, 405, 418, 419, 420, 378, 313, 314, 727, - 361, 451, 656, 691, 692, 581, 0, 644, 582, 591, - 353, 616, 628, 627, 447, 541, 0, 639, 642, 571, - 726, 0, 636, 650, 730, 649, 723, 457, 0, 484, - 647, 594, 0, 640, 613, 614, 0, 641, 609, 645, - 0, 583, 0, 552, 555, 584, 669, 670, 671, 320, - 554, 673, 674, 675, 676, 677, 678, 679, 672, 524, - 617, 593, 620, 533, 596, 595, 0, 0, 631, 550, - 632, 633, 441, 442, 443, 444, 382, 657, 342, 553, - 471, 0, 618, 0, 0, 0, 0, 0, 0, 0, - 0, 623, 624, 621, 735, 0, 680, 681, 0, 0, - 547, 548, 377, 0, 566, 385, 341, 456, 379, 531, - 408, 0, 559, 625, 560, 473, 474, 683, 688, 684, - 685, 687, 707, 448, 399, 404, 488, 410, 424, 476, - 530, 454, 481, 339, 520, 490, 429, 610, 638, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 665, 664, 663, 662, - 661, 660, 659, 658, 0, 0, 607, 507, 355, 307, - 351, 352, 359, 724, 720, 725, 708, 711, 710, 686, - 0, 315, 587, 422, 470, 376, 652, 653, 0, 706, - 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, - 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, - 282, 283, 284, 285, 655, 276, 277, 286, 287, 288, - 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, - 299, 0, 0, 0, 0, 309, 712, 713, 714, 715, - 716, 0, 0, 310, 311, 312, 0, 0, 274, 275, - 302, 498, 303, 304, 305, 306, 0, 0, 537, 538, - 539, 562, 0, 540, 522, 586, 386, 316, 502, 529, - 722, 0, 0, 0, 0, 0, 0, 0, 637, 648, - 682, 0, 694, 695, 697, 699, 698, 701, 495, 496, - 709, 0, 0, 703, 704, 705, 702, 426, 482, 503, - 489, 0, 728, 577, 578, 729, 690, 317, 453, 0, - 0, 592, 626, 615, 700, 580, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 369, 0, 0, 421, - 630, 611, 622, 612, 597, 598, 599, 606, 381, 600, - 601, 602, 572, 603, 573, 604, 605, 0, 629, 579, - 491, 437, 0, 646, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 245, 0, 0, 3603, 0, 0, - 0, 337, 246, 574, 696, 576, 575, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 497, 526, 0, 539, 0, 407, 408, 0, 0, 0, + 0, 0, 0, 0, 324, 504, 523, 338, 491, 537, + 343, 499, 516, 333, 457, 488, 0, 0, 326, 521, + 498, 439, 325, 0, 482, 366, 383, 363, 455, 0, + 0, 520, 550, 362, 540, 0, 531, 328, 0, 530, + 454, 517, 522, 440, 433, 0, 327, 519, 438, 432, + 413, 373, 566, 414, 415, 416, 417, 418, 419, 387, + 469, 430, 470, 388, 444, 443, 445, 389, 390, 391, + 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, + 0, 0, 561, 562, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, + 0, 0, 698, 0, 533, 0, 0, 0, 0, 0, + 0, 502, 0, 0, 420, 0, 0, 0, 551, 0, + 485, 460, 736, 0, 0, 483, 428, 518, 471, 524, + 505, 532, 477, 472, 318, 506, 365, 441, 334, 336, + 726, 367, 370, 374, 375, 450, 451, 465, 490, 509, + 510, 511, 364, 348, 484, 349, 384, 350, 319, 356, + 354, 357, 492, 358, 321, 466, 515, 0, 380, 480, + 436, 322, 435, 467, 514, 513, 335, 541, 548, 549, + 639, 0, 554, 737, 738, 739, 563, 0, 473, 331, + 330, 0, 0, 0, 360, 468, 344, 346, 347, 345, + 463, 464, 568, 569, 570, 572, 0, 573, 574, 0, + 0, 0, 0, 575, 640, 656, 624, 593, 556, 648, + 590, 594, 595, 401, 402, 403, 404, 659, 0, 0, + 0, 547, 421, 422, 0, 372, 371, 437, 323, 0, + 0, 410, 400, 474, 329, 368, 412, 406, 423, 424, + 425, 378, 313, 314, 732, 361, 456, 661, 696, 697, + 586, 0, 649, 587, 596, 353, 621, 633, 632, 452, + 546, 0, 644, 647, 576, 731, 0, 641, 655, 735, + 654, 728, 462, 0, 489, 652, 599, 0, 645, 618, + 619, 0, 646, 614, 650, 0, 588, 0, 557, 560, + 589, 674, 675, 676, 320, 559, 678, 679, 680, 681, + 682, 683, 684, 677, 529, 622, 598, 625, 538, 601, + 600, 0, 0, 636, 555, 637, 638, 446, 447, 448, + 449, 382, 662, 342, 558, 476, 0, 623, 0, 0, + 0, 0, 0, 0, 0, 0, 628, 629, 626, 740, + 0, 685, 686, 0, 0, 552, 553, 377, 0, 571, + 385, 341, 461, 379, 536, 409, 0, 564, 630, 565, + 478, 479, 688, 693, 689, 690, 692, 712, 453, 399, + 405, 493, 411, 429, 481, 535, 459, 486, 339, 525, + 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 492, 521, 0, 534, - 0, 406, 407, 0, 0, 0, 0, 0, 0, 0, - 324, 499, 518, 338, 486, 532, 343, 494, 511, 333, - 452, 483, 0, 0, 326, 516, 493, 434, 325, 0, - 477, 366, 383, 363, 450, 0, 0, 515, 545, 362, - 535, 0, 526, 328, 0, 525, 449, 512, 517, 435, - 428, 0, 327, 514, 433, 427, 412, 373, 561, 413, - 414, 387, 464, 425, 465, 388, 439, 438, 440, 389, - 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, - 0, 0, 0, 0, 556, 557, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 689, 0, 0, 693, 0, 528, 0, 0, 0, - 0, 0, 0, 497, 0, 0, 415, 0, 0, 0, - 546, 0, 480, 455, 731, 0, 0, 478, 423, 513, - 466, 519, 500, 527, 472, 467, 318, 501, 365, 436, - 334, 336, 721, 367, 370, 374, 375, 445, 446, 460, - 485, 504, 505, 506, 364, 348, 479, 349, 384, 350, - 319, 356, 354, 357, 487, 358, 321, 461, 510, 0, - 380, 475, 431, 322, 430, 462, 509, 508, 335, 536, - 543, 544, 634, 0, 549, 732, 733, 734, 558, 0, - 468, 331, 330, 0, 0, 0, 360, 463, 344, 346, - 347, 345, 458, 459, 563, 564, 565, 567, 0, 568, - 569, 0, 0, 0, 0, 570, 635, 651, 619, 588, - 551, 643, 585, 589, 590, 401, 402, 403, 654, 0, - 0, 0, 542, 416, 417, 0, 372, 371, 432, 323, - 0, 0, 409, 400, 469, 329, 368, 411, 405, 418, - 419, 420, 378, 313, 314, 727, 361, 451, 656, 691, - 692, 581, 0, 644, 582, 591, 353, 616, 628, 627, - 447, 541, 0, 639, 642, 571, 726, 0, 636, 650, - 730, 649, 723, 457, 0, 484, 647, 594, 0, 640, - 613, 614, 0, 641, 609, 645, 0, 583, 0, 552, - 555, 584, 669, 670, 671, 320, 554, 673, 674, 675, - 676, 677, 678, 679, 672, 524, 617, 593, 620, 533, - 596, 595, 0, 0, 631, 550, 632, 633, 441, 442, - 443, 444, 382, 657, 342, 553, 471, 0, 618, 0, - 0, 0, 0, 0, 0, 0, 0, 623, 624, 621, - 735, 0, 680, 681, 0, 0, 547, 548, 377, 0, - 566, 385, 341, 456, 379, 531, 408, 0, 559, 625, - 560, 473, 474, 683, 688, 684, 685, 687, 707, 448, - 399, 404, 488, 410, 424, 476, 530, 454, 481, 339, - 520, 490, 429, 610, 638, 0, 0, 0, 0, 0, + 0, 670, 669, 668, 667, 666, 665, 664, 663, 0, + 0, 612, 512, 355, 307, 351, 352, 359, 729, 725, + 730, 713, 716, 715, 691, 0, 315, 592, 427, 475, + 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, + 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, + 273, 278, 279, 280, 281, 282, 283, 284, 285, 660, + 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 298, 299, 0, 0, 0, 0, + 309, 717, 718, 719, 720, 721, 0, 0, 310, 311, + 312, 0, 0, 274, 275, 302, 503, 303, 304, 305, + 306, 0, 0, 542, 543, 544, 567, 0, 545, 527, + 591, 386, 316, 507, 534, 727, 0, 0, 0, 0, + 0, 0, 0, 642, 653, 687, 0, 699, 700, 702, + 704, 703, 706, 500, 501, 714, 0, 0, 708, 709, + 710, 707, 431, 487, 508, 494, 0, 733, 582, 583, + 734, 695, 317, 458, 0, 0, 597, 631, 620, 705, + 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 369, 0, 0, 426, 635, 616, 627, 617, 602, + 603, 604, 611, 381, 605, 606, 607, 577, 608, 578, + 609, 610, 0, 634, 584, 496, 442, 0, 651, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 2308, 0, 0, 245, + 0, 0, 0, 0, 0, 0, 337, 246, 579, 701, + 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 497, 526, 0, 539, 0, 407, 408, 0, 0, + 0, 0, 0, 0, 0, 324, 504, 523, 338, 491, + 537, 343, 499, 516, 333, 457, 488, 0, 0, 326, + 521, 498, 439, 325, 0, 482, 366, 383, 363, 455, + 0, 0, 520, 550, 362, 540, 0, 531, 328, 0, + 530, 454, 517, 522, 440, 433, 0, 327, 519, 438, + 432, 413, 373, 566, 414, 415, 416, 417, 418, 419, + 387, 469, 430, 470, 388, 444, 443, 445, 389, 390, + 391, 392, 393, 394, 395, 396, 397, 398, 0, 0, + 0, 0, 0, 561, 562, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 694, 0, 0, 698, 0, 533, 0, 0, 0, 0, + 0, 0, 502, 0, 0, 420, 0, 0, 0, 551, + 0, 485, 460, 736, 0, 0, 483, 428, 518, 471, + 524, 505, 532, 477, 472, 318, 506, 365, 441, 334, + 336, 726, 367, 370, 374, 375, 450, 451, 465, 490, + 509, 510, 511, 364, 348, 484, 349, 384, 350, 319, + 356, 354, 357, 492, 358, 321, 466, 515, 0, 380, + 480, 436, 322, 435, 467, 514, 513, 335, 541, 548, + 549, 639, 0, 554, 737, 738, 739, 563, 0, 473, + 331, 330, 0, 0, 0, 360, 468, 344, 346, 347, + 345, 463, 464, 568, 569, 570, 572, 0, 573, 574, + 0, 0, 0, 0, 575, 640, 656, 624, 593, 556, + 648, 590, 594, 595, 401, 402, 403, 404, 659, 0, + 0, 0, 547, 421, 422, 0, 372, 371, 437, 323, + 0, 0, 410, 400, 474, 329, 368, 412, 406, 423, + 424, 425, 378, 313, 314, 732, 361, 456, 661, 696, + 697, 586, 0, 649, 587, 596, 353, 621, 633, 632, + 452, 546, 0, 644, 647, 576, 731, 0, 641, 655, + 735, 654, 728, 462, 0, 489, 652, 599, 0, 645, + 618, 619, 0, 646, 614, 650, 0, 588, 0, 557, + 560, 589, 674, 675, 676, 320, 559, 678, 679, 680, + 681, 682, 683, 684, 677, 529, 622, 598, 625, 538, + 601, 600, 0, 0, 636, 555, 637, 638, 446, 447, + 448, 449, 382, 662, 342, 558, 476, 0, 623, 0, + 0, 0, 0, 0, 0, 0, 0, 628, 629, 626, + 740, 0, 685, 686, 0, 0, 552, 553, 377, 0, + 571, 385, 341, 461, 379, 536, 409, 0, 564, 630, + 565, 478, 479, 688, 693, 689, 690, 692, 712, 453, + 399, 405, 493, 411, 429, 481, 535, 459, 486, 339, + 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 665, 664, 663, 662, 661, 660, 659, 658, - 0, 0, 607, 507, 355, 307, 351, 352, 359, 724, - 720, 725, 708, 711, 710, 686, 0, 315, 587, 422, - 470, 376, 652, 653, 0, 706, 259, 260, 261, 262, + 0, 0, 670, 669, 668, 667, 666, 665, 664, 663, + 0, 0, 612, 512, 355, 307, 351, 352, 359, 729, + 725, 730, 713, 716, 715, 691, 0, 315, 592, 427, + 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, 283, 284, 285, - 655, 276, 277, 286, 287, 288, 289, 290, 291, 292, + 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, - 0, 309, 712, 713, 714, 715, 716, 0, 0, 310, - 311, 312, 0, 0, 274, 275, 302, 498, 303, 304, - 305, 306, 0, 0, 537, 538, 539, 562, 0, 540, - 522, 586, 386, 316, 502, 529, 722, 0, 0, 0, - 0, 0, 0, 0, 637, 648, 682, 0, 694, 695, - 697, 699, 698, 701, 495, 496, 709, 0, 0, 703, - 704, 705, 702, 426, 482, 503, 489, 0, 728, 577, - 578, 729, 690, 317, 453, 0, 0, 592, 626, 615, - 700, 580, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 369, 0, 0, 421, 630, 611, 622, 612, - 597, 598, 599, 606, 381, 600, 601, 602, 572, 603, - 573, 604, 605, 0, 629, 579, 491, 437, 0, 646, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 4087, 0, 0, 0, 337, 246, 574, - 696, 576, 575, 0, 0, 0, 0, 0, 0, 0, + 0, 309, 717, 718, 719, 720, 721, 0, 0, 310, + 311, 312, 0, 0, 274, 275, 302, 503, 303, 304, + 305, 306, 0, 0, 542, 543, 544, 567, 0, 545, + 527, 591, 386, 316, 507, 534, 727, 0, 0, 0, + 0, 0, 0, 0, 642, 653, 687, 0, 699, 700, + 702, 704, 703, 706, 500, 501, 714, 0, 0, 708, + 709, 710, 707, 431, 487, 508, 494, 0, 733, 582, + 583, 734, 695, 317, 458, 0, 0, 597, 631, 620, + 705, 585, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 369, 0, 0, 426, 635, 616, 627, 617, + 602, 603, 604, 611, 381, 605, 606, 607, 577, 608, + 578, 609, 610, 0, 634, 584, 496, 442, 0, 651, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 245, 0, 0, 0, 0, 0, 0, 337, 246, 579, + 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 3659, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 497, 526, 0, 539, 0, 407, 408, 0, + 0, 0, 0, 0, 0, 0, 324, 504, 523, 338, + 491, 537, 343, 499, 516, 333, 457, 488, 0, 0, + 326, 521, 498, 439, 325, 0, 482, 366, 383, 363, + 455, 0, 0, 520, 550, 362, 540, 0, 531, 328, + 0, 530, 454, 517, 522, 440, 433, 0, 327, 519, + 438, 432, 413, 373, 566, 414, 415, 416, 417, 418, + 419, 387, 469, 430, 470, 388, 444, 443, 445, 389, + 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, + 0, 0, 0, 0, 561, 562, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 694, 0, 0, 698, 0, 533, 0, 0, 0, + 0, 0, 0, 502, 0, 0, 420, 0, 0, 0, + 551, 0, 485, 460, 736, 0, 0, 483, 428, 518, + 471, 524, 505, 532, 477, 472, 318, 506, 365, 441, + 334, 336, 726, 367, 370, 374, 375, 450, 451, 465, + 490, 509, 510, 511, 364, 348, 484, 349, 384, 350, + 319, 356, 354, 357, 492, 358, 321, 466, 515, 0, + 380, 480, 436, 322, 435, 467, 514, 513, 335, 541, + 548, 549, 639, 0, 554, 737, 738, 739, 563, 0, + 473, 331, 330, 0, 0, 0, 360, 468, 344, 346, + 347, 345, 463, 464, 568, 569, 570, 572, 0, 573, + 574, 0, 0, 0, 0, 575, 640, 656, 624, 593, + 556, 648, 590, 594, 595, 401, 402, 403, 404, 659, + 0, 0, 0, 547, 421, 422, 0, 372, 371, 437, + 323, 0, 0, 410, 400, 474, 329, 368, 412, 406, + 423, 424, 425, 378, 313, 314, 732, 361, 456, 661, + 696, 697, 586, 0, 649, 587, 596, 353, 621, 633, + 632, 452, 546, 0, 644, 647, 576, 731, 0, 641, + 655, 735, 654, 728, 462, 0, 489, 652, 599, 0, + 645, 618, 619, 0, 646, 614, 650, 0, 588, 0, + 557, 560, 589, 674, 675, 676, 320, 559, 678, 679, + 680, 681, 682, 683, 684, 677, 529, 622, 598, 625, + 538, 601, 600, 0, 0, 636, 555, 637, 638, 446, + 447, 448, 449, 382, 662, 342, 558, 476, 0, 623, + 0, 0, 0, 0, 0, 0, 0, 0, 628, 629, + 626, 740, 0, 685, 686, 0, 0, 552, 553, 377, + 0, 571, 385, 341, 461, 379, 536, 409, 0, 564, + 630, 565, 478, 479, 688, 693, 689, 690, 692, 712, + 453, 399, 405, 493, 411, 429, 481, 535, 459, 486, + 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 492, 521, 0, 534, 0, 406, 407, 0, - 0, 0, 0, 0, 0, 0, 324, 499, 518, 338, - 486, 532, 343, 494, 511, 333, 452, 483, 0, 0, - 326, 516, 493, 434, 325, 0, 477, 366, 383, 363, - 450, 0, 0, 515, 545, 362, 535, 0, 526, 328, - 0, 525, 449, 512, 517, 435, 428, 0, 327, 514, - 433, 427, 412, 373, 561, 413, 414, 387, 464, 425, - 465, 388, 439, 438, 440, 389, 390, 391, 392, 393, - 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, - 556, 557, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 689, 0, 0, - 693, 0, 528, 0, 0, 0, 0, 0, 0, 497, - 0, 0, 415, 0, 0, 0, 546, 0, 480, 455, - 731, 0, 0, 478, 423, 513, 466, 519, 500, 527, - 472, 467, 318, 501, 365, 436, 334, 336, 721, 367, - 370, 374, 375, 445, 446, 460, 485, 504, 505, 506, - 364, 348, 479, 349, 384, 350, 319, 356, 354, 357, - 487, 358, 321, 461, 510, 0, 380, 475, 431, 322, - 430, 462, 509, 508, 335, 536, 543, 544, 634, 0, - 549, 732, 733, 734, 558, 0, 468, 331, 330, 0, - 0, 0, 360, 463, 344, 346, 347, 345, 458, 459, - 563, 564, 565, 567, 0, 568, 569, 0, 0, 0, - 0, 570, 635, 651, 619, 588, 551, 643, 585, 589, - 590, 401, 402, 403, 654, 0, 0, 0, 542, 416, - 417, 0, 372, 371, 432, 323, 0, 0, 409, 400, - 469, 329, 368, 411, 405, 418, 419, 420, 378, 313, - 314, 727, 361, 451, 656, 691, 692, 581, 0, 644, - 582, 591, 353, 616, 628, 627, 447, 541, 0, 639, - 642, 571, 726, 0, 636, 650, 730, 649, 723, 457, - 0, 484, 647, 594, 0, 640, 613, 614, 0, 641, - 609, 645, 0, 583, 0, 552, 555, 584, 669, 670, - 671, 320, 554, 673, 674, 675, 676, 677, 678, 679, - 672, 524, 617, 593, 620, 533, 596, 595, 0, 0, - 631, 550, 632, 633, 441, 442, 443, 444, 382, 657, - 342, 553, 471, 0, 618, 0, 0, 0, 0, 0, - 0, 0, 0, 623, 624, 621, 735, 0, 680, 681, - 0, 0, 547, 548, 377, 0, 566, 385, 341, 456, - 379, 531, 408, 0, 559, 625, 560, 473, 474, 683, - 688, 684, 685, 687, 707, 448, 399, 404, 488, 410, - 424, 476, 530, 454, 481, 339, 520, 490, 429, 610, - 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 665, 664, - 663, 662, 661, 660, 659, 658, 0, 0, 607, 507, - 355, 307, 351, 352, 359, 724, 720, 725, 708, 711, - 710, 686, 0, 315, 587, 422, 470, 376, 652, 653, - 0, 706, 259, 260, 261, 262, 263, 264, 265, 266, - 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, - 280, 281, 282, 283, 284, 285, 655, 276, 277, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 298, 299, 0, 0, 0, 0, 309, 712, 713, - 714, 715, 716, 0, 0, 310, 311, 312, 0, 0, - 274, 275, 302, 498, 303, 304, 305, 306, 0, 0, - 537, 538, 539, 562, 0, 540, 522, 586, 386, 316, - 502, 529, 722, 0, 0, 0, 0, 0, 0, 0, - 637, 648, 682, 0, 694, 695, 697, 699, 698, 701, - 495, 496, 709, 0, 0, 703, 704, 705, 702, 426, - 482, 503, 489, 0, 728, 577, 578, 729, 690, 317, - 453, 0, 0, 592, 626, 615, 700, 580, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, - 0, 421, 630, 611, 622, 612, 597, 598, 599, 606, - 381, 600, 601, 602, 572, 603, 573, 604, 605, 0, - 629, 579, 491, 437, 0, 646, 0, 0, 0, 0, + 0, 0, 0, 670, 669, 668, 667, 666, 665, 664, + 663, 0, 0, 612, 512, 355, 307, 351, 352, 359, + 729, 725, 730, 713, 716, 715, 691, 0, 315, 592, + 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, + 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, + 271, 272, 273, 278, 279, 280, 281, 282, 283, 284, + 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 298, 299, 0, 0, + 0, 0, 309, 717, 718, 719, 720, 721, 0, 0, + 310, 311, 312, 0, 0, 274, 275, 302, 503, 303, + 304, 305, 306, 0, 0, 542, 543, 544, 567, 0, + 545, 527, 591, 386, 316, 507, 534, 727, 0, 0, + 0, 0, 0, 0, 0, 642, 653, 687, 0, 699, + 700, 702, 704, 703, 706, 500, 501, 714, 0, 0, + 708, 709, 710, 707, 431, 487, 508, 494, 0, 733, + 582, 583, 734, 695, 317, 458, 0, 0, 597, 631, + 620, 705, 585, 0, 0, 3905, 0, 0, 0, 0, + 0, 0, 0, 369, 0, 0, 426, 635, 616, 627, + 617, 602, 603, 604, 611, 381, 605, 606, 607, 577, + 608, 578, 609, 610, 0, 634, 584, 496, 442, 0, + 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 2292, 0, 0, 245, 0, 0, 0, - 0, 0, 0, 337, 246, 574, 696, 576, 575, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, + 0, 245, 0, 0, 0, 0, 0, 0, 337, 246, + 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 492, 521, - 0, 534, 0, 406, 407, 0, 0, 0, 0, 0, - 0, 0, 324, 499, 518, 338, 486, 532, 343, 494, - 511, 333, 452, 483, 0, 0, 326, 516, 493, 434, - 325, 0, 477, 366, 383, 363, 450, 0, 0, 515, - 545, 362, 535, 0, 526, 328, 0, 525, 449, 512, - 517, 435, 428, 0, 327, 514, 433, 427, 412, 373, - 561, 413, 414, 387, 464, 425, 465, 388, 439, 438, - 440, 389, 390, 391, 392, 393, 394, 395, 396, 397, - 398, 0, 0, 0, 0, 0, 556, 557, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 689, 0, 0, 693, 0, 528, 0, - 0, 0, 0, 0, 0, 497, 0, 0, 415, 0, - 0, 0, 546, 0, 480, 455, 731, 0, 0, 478, - 423, 513, 466, 519, 500, 527, 472, 467, 318, 501, - 365, 436, 334, 336, 721, 367, 370, 374, 375, 445, - 446, 460, 485, 504, 505, 506, 364, 348, 479, 349, - 384, 350, 319, 356, 354, 357, 487, 358, 321, 461, - 510, 0, 380, 475, 431, 322, 430, 462, 509, 508, - 335, 536, 543, 544, 634, 0, 549, 732, 733, 734, - 558, 0, 468, 331, 330, 0, 0, 0, 360, 463, - 344, 346, 347, 345, 458, 459, 563, 564, 565, 567, - 0, 568, 569, 0, 0, 0, 0, 570, 635, 651, - 619, 588, 551, 643, 585, 589, 590, 401, 402, 403, - 654, 0, 0, 0, 542, 416, 417, 0, 372, 371, - 432, 323, 0, 0, 409, 400, 469, 329, 368, 411, - 405, 418, 419, 420, 378, 313, 314, 727, 361, 451, - 656, 691, 692, 581, 0, 644, 582, 591, 353, 616, - 628, 627, 447, 541, 0, 639, 642, 571, 726, 0, - 636, 650, 730, 649, 723, 457, 0, 484, 647, 594, - 0, 640, 613, 614, 0, 641, 609, 645, 0, 583, - 0, 552, 555, 584, 669, 670, 671, 320, 554, 673, - 674, 675, 676, 677, 678, 679, 672, 524, 617, 593, - 620, 533, 596, 595, 0, 0, 631, 550, 632, 633, - 441, 442, 443, 444, 382, 657, 342, 553, 471, 0, - 618, 0, 0, 0, 0, 0, 0, 0, 0, 623, - 624, 621, 735, 0, 680, 681, 0, 0, 547, 548, - 377, 0, 566, 385, 341, 456, 379, 531, 408, 0, - 559, 625, 560, 473, 474, 683, 688, 684, 685, 687, - 707, 448, 399, 404, 488, 410, 424, 476, 530, 454, - 481, 339, 520, 490, 429, 610, 638, 0, 0, 0, + 0, 0, 0, 497, 526, 0, 539, 0, 407, 408, + 0, 0, 0, 0, 0, 0, 0, 324, 504, 523, + 338, 491, 537, 343, 499, 516, 333, 457, 488, 0, + 0, 326, 521, 498, 439, 325, 0, 482, 366, 383, + 363, 455, 0, 0, 520, 550, 362, 540, 0, 531, + 328, 0, 530, 454, 517, 522, 440, 433, 0, 327, + 519, 438, 432, 413, 373, 566, 414, 415, 416, 417, + 418, 419, 387, 469, 430, 470, 388, 444, 443, 445, + 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, + 0, 0, 0, 0, 0, 561, 562, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 694, 0, 0, 698, 0, 533, 0, 0, + 0, 0, 0, 0, 502, 0, 0, 420, 0, 0, + 0, 551, 0, 485, 460, 736, 0, 0, 483, 428, + 518, 471, 524, 505, 532, 477, 472, 318, 506, 365, + 441, 334, 336, 726, 367, 370, 374, 375, 450, 451, + 465, 490, 509, 510, 511, 364, 348, 484, 349, 384, + 350, 319, 356, 354, 357, 492, 358, 321, 466, 515, + 0, 380, 480, 436, 322, 435, 467, 514, 513, 335, + 541, 548, 549, 639, 0, 554, 737, 738, 739, 563, + 0, 473, 331, 330, 0, 0, 0, 360, 468, 344, + 346, 347, 345, 463, 464, 568, 569, 570, 572, 0, + 573, 574, 0, 0, 0, 0, 575, 640, 656, 624, + 593, 556, 648, 590, 594, 595, 401, 402, 403, 404, + 659, 0, 0, 0, 547, 421, 422, 0, 372, 371, + 437, 323, 0, 0, 410, 400, 474, 329, 368, 412, + 406, 423, 424, 425, 378, 313, 314, 732, 361, 456, + 661, 696, 697, 586, 0, 649, 587, 596, 353, 621, + 633, 632, 452, 546, 0, 644, 647, 576, 731, 0, + 641, 655, 735, 654, 728, 462, 0, 489, 652, 599, + 0, 645, 618, 619, 0, 646, 614, 650, 0, 588, + 0, 557, 560, 589, 674, 675, 676, 320, 559, 678, + 679, 680, 681, 682, 683, 684, 677, 529, 622, 598, + 625, 538, 601, 600, 0, 0, 636, 555, 637, 638, + 446, 447, 448, 449, 382, 662, 342, 558, 476, 0, + 623, 0, 0, 0, 0, 0, 0, 0, 0, 628, + 629, 626, 740, 0, 685, 686, 0, 0, 552, 553, + 377, 0, 571, 385, 341, 461, 379, 536, 409, 0, + 564, 630, 565, 478, 479, 688, 693, 689, 690, 692, + 712, 453, 399, 405, 493, 411, 429, 481, 535, 459, + 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 665, 664, 663, 662, 661, 660, - 659, 658, 0, 0, 607, 507, 355, 307, 351, 352, - 359, 724, 720, 725, 708, 711, 710, 686, 0, 315, - 587, 422, 470, 376, 652, 653, 0, 706, 259, 260, + 0, 0, 0, 0, 670, 669, 668, 667, 666, 665, + 664, 663, 0, 0, 612, 512, 355, 307, 351, 352, + 359, 729, 725, 730, 713, 716, 715, 691, 0, 315, + 592, 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, 283, - 284, 285, 655, 276, 277, 286, 287, 288, 289, 290, + 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, - 0, 0, 0, 309, 712, 713, 714, 715, 716, 0, - 0, 310, 311, 312, 0, 0, 274, 275, 302, 498, - 303, 304, 305, 306, 0, 0, 537, 538, 539, 562, - 0, 540, 522, 586, 386, 316, 502, 529, 722, 0, - 0, 0, 0, 0, 0, 0, 637, 648, 682, 0, - 694, 695, 697, 699, 698, 701, 495, 496, 709, 0, - 0, 703, 704, 705, 702, 426, 482, 503, 489, 0, - 728, 577, 578, 729, 690, 317, 453, 0, 0, 592, - 626, 615, 700, 580, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 369, 0, 0, 421, 630, 611, - 622, 612, 597, 598, 599, 606, 381, 600, 601, 602, - 572, 603, 573, 604, 605, 0, 629, 579, 491, 437, - 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 309, 717, 718, 719, 720, 721, 0, + 0, 310, 311, 312, 0, 0, 274, 275, 302, 503, + 303, 304, 305, 306, 0, 0, 542, 543, 544, 567, + 0, 545, 527, 591, 386, 316, 507, 534, 727, 0, + 0, 0, 0, 0, 0, 0, 642, 653, 687, 0, + 699, 700, 702, 704, 703, 706, 500, 501, 714, 0, + 0, 708, 709, 710, 707, 431, 487, 508, 494, 0, + 733, 582, 583, 734, 695, 317, 458, 0, 0, 597, + 631, 620, 705, 585, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 369, 0, 0, 426, 635, 616, + 627, 617, 602, 603, 604, 611, 381, 605, 606, 607, + 577, 608, 578, 609, 610, 0, 634, 584, 496, 442, + 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, 337, - 246, 574, 696, 576, 575, 0, 0, 0, 0, 0, + 246, 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 3635, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 492, 521, 0, 534, 0, 406, - 407, 0, 0, 0, 0, 0, 0, 0, 324, 499, - 518, 338, 486, 532, 343, 494, 511, 333, 452, 483, - 0, 0, 326, 516, 493, 434, 325, 0, 477, 366, - 383, 363, 450, 0, 0, 515, 545, 362, 535, 0, - 526, 328, 0, 525, 449, 512, 517, 435, 428, 0, - 327, 514, 433, 427, 412, 373, 561, 413, 414, 387, - 464, 425, 465, 388, 439, 438, 440, 389, 390, 391, - 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, - 0, 0, 556, 557, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, - 0, 0, 693, 0, 528, 0, 0, 0, 0, 0, - 0, 497, 0, 0, 415, 0, 0, 0, 546, 0, - 480, 455, 731, 0, 0, 478, 423, 513, 466, 519, - 500, 527, 472, 467, 318, 501, 365, 436, 334, 336, - 721, 367, 370, 374, 375, 445, 446, 460, 485, 504, - 505, 506, 364, 348, 479, 349, 384, 350, 319, 356, - 354, 357, 487, 358, 321, 461, 510, 0, 380, 475, - 431, 322, 430, 462, 509, 508, 335, 536, 543, 544, - 634, 0, 549, 732, 733, 734, 558, 0, 468, 331, - 330, 0, 0, 0, 360, 463, 344, 346, 347, 345, - 458, 459, 563, 564, 565, 567, 0, 568, 569, 0, - 0, 0, 0, 570, 635, 651, 619, 588, 551, 643, - 585, 589, 590, 401, 402, 403, 654, 0, 0, 0, - 542, 416, 417, 0, 372, 371, 432, 323, 0, 0, - 409, 400, 469, 329, 368, 411, 405, 418, 419, 420, - 378, 313, 314, 727, 361, 451, 656, 691, 692, 581, - 0, 644, 582, 591, 353, 616, 628, 627, 447, 541, - 0, 639, 642, 571, 726, 0, 636, 650, 730, 649, - 723, 457, 0, 484, 647, 594, 0, 640, 613, 614, - 0, 641, 609, 645, 0, 583, 0, 552, 555, 584, - 669, 670, 671, 320, 554, 673, 674, 675, 676, 677, - 678, 679, 672, 524, 617, 593, 620, 533, 596, 595, - 0, 0, 631, 550, 632, 633, 441, 442, 443, 444, - 382, 657, 342, 553, 471, 0, 618, 0, 0, 0, - 0, 0, 0, 0, 0, 623, 624, 621, 735, 0, - 680, 681, 0, 0, 547, 548, 377, 0, 566, 385, - 341, 456, 379, 531, 408, 0, 559, 625, 560, 473, - 474, 683, 688, 684, 685, 687, 707, 448, 399, 404, - 488, 410, 424, 476, 530, 454, 481, 339, 520, 490, - 429, 610, 638, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 665, 664, 663, 662, 661, 660, 659, 658, 0, 0, - 607, 507, 355, 307, 351, 352, 359, 724, 720, 725, - 708, 711, 710, 686, 0, 315, 587, 422, 470, 376, - 652, 653, 0, 706, 259, 260, 261, 262, 263, 264, - 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, - 278, 279, 280, 281, 282, 283, 284, 285, 655, 276, - 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 298, 299, 0, 0, 0, 0, 309, - 712, 713, 714, 715, 716, 0, 0, 310, 311, 312, - 0, 0, 274, 275, 302, 498, 303, 304, 305, 306, - 0, 0, 537, 538, 539, 562, 0, 540, 522, 586, - 386, 316, 502, 529, 722, 0, 0, 0, 0, 0, - 0, 0, 637, 648, 682, 0, 694, 695, 697, 699, - 698, 701, 495, 496, 709, 0, 0, 703, 704, 705, - 702, 426, 482, 503, 489, 0, 728, 577, 578, 729, - 690, 317, 453, 0, 0, 592, 626, 615, 700, 580, - 0, 0, 3879, 0, 0, 0, 0, 0, 0, 0, - 369, 0, 0, 421, 630, 611, 622, 612, 597, 598, - 599, 606, 381, 600, 601, 602, 572, 603, 573, 604, - 605, 0, 629, 579, 491, 437, 0, 646, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 0, 0, 0, 0, 337, 246, 574, 696, 576, - 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 3783, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 497, 526, 0, 539, 0, 407, + 408, 0, 0, 0, 0, 0, 0, 0, 324, 504, + 523, 338, 491, 537, 343, 499, 516, 333, 457, 488, + 0, 0, 326, 521, 498, 439, 325, 0, 482, 366, + 383, 363, 455, 0, 0, 520, 550, 362, 540, 0, + 531, 328, 0, 530, 454, 517, 522, 440, 433, 0, + 327, 519, 438, 432, 413, 373, 566, 414, 415, 416, + 417, 418, 419, 387, 469, 430, 470, 388, 444, 443, + 445, 389, 390, 391, 392, 393, 394, 395, 396, 397, + 398, 0, 0, 0, 0, 0, 561, 562, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 694, 0, 0, 698, 0, 533, 0, + 0, 0, 0, 0, 0, 502, 0, 0, 420, 0, + 0, 0, 551, 0, 485, 460, 736, 0, 0, 483, + 428, 518, 471, 524, 505, 532, 477, 472, 318, 506, + 365, 441, 334, 336, 726, 367, 370, 374, 375, 450, + 451, 465, 490, 509, 510, 511, 364, 348, 484, 349, + 384, 350, 319, 356, 354, 357, 492, 358, 321, 466, + 515, 0, 380, 480, 436, 322, 435, 467, 514, 513, + 335, 541, 548, 549, 639, 0, 554, 737, 738, 739, + 563, 0, 473, 331, 330, 0, 0, 0, 360, 468, + 344, 346, 347, 345, 463, 464, 568, 569, 570, 572, + 0, 573, 574, 0, 0, 0, 0, 575, 640, 656, + 624, 593, 556, 648, 590, 594, 595, 401, 402, 403, + 404, 659, 0, 0, 0, 547, 421, 422, 0, 372, + 371, 437, 323, 0, 0, 410, 400, 474, 329, 368, + 412, 406, 423, 424, 425, 378, 313, 314, 732, 361, + 456, 661, 696, 697, 586, 0, 649, 587, 596, 353, + 621, 633, 632, 452, 546, 0, 644, 647, 576, 731, + 0, 641, 655, 735, 654, 728, 462, 0, 489, 652, + 599, 0, 645, 618, 619, 0, 646, 614, 650, 0, + 588, 0, 557, 560, 589, 674, 675, 676, 320, 559, + 678, 679, 680, 681, 682, 683, 684, 677, 529, 622, + 598, 625, 538, 601, 600, 0, 0, 636, 555, 637, + 638, 446, 447, 448, 449, 382, 662, 342, 558, 476, + 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, + 628, 629, 626, 740, 0, 685, 686, 0, 0, 552, + 553, 377, 0, 571, 385, 341, 461, 379, 536, 409, + 0, 564, 630, 565, 478, 479, 688, 693, 689, 690, + 692, 712, 453, 399, 405, 493, 411, 429, 481, 535, + 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, + 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 670, 669, 668, 667, 666, + 665, 664, 663, 0, 0, 612, 512, 355, 307, 351, + 352, 359, 729, 725, 730, 713, 716, 715, 691, 0, + 315, 592, 427, 475, 376, 657, 658, 0, 711, 259, + 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, + 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, + 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, + 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, + 0, 0, 0, 0, 309, 717, 718, 719, 720, 721, + 0, 0, 310, 311, 312, 0, 0, 274, 275, 302, + 503, 303, 304, 305, 306, 0, 0, 542, 543, 544, + 567, 0, 545, 527, 591, 386, 316, 507, 534, 727, + 0, 0, 0, 0, 0, 0, 0, 642, 653, 687, + 0, 699, 700, 702, 704, 703, 706, 500, 501, 714, + 0, 0, 708, 709, 710, 707, 431, 487, 508, 494, + 0, 733, 582, 583, 734, 695, 317, 458, 0, 0, + 597, 631, 620, 705, 585, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 369, 0, 0, 426, 635, + 616, 627, 617, 602, 603, 604, 611, 381, 605, 606, + 607, 577, 608, 578, 609, 610, 0, 634, 584, 496, + 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 245, 0, 0, 3632, 0, 0, 0, + 337, 246, 579, 701, 581, 580, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 492, 521, 0, 534, 0, 406, 407, 0, 0, 0, - 0, 0, 0, 0, 324, 499, 518, 338, 486, 532, - 343, 494, 511, 333, 452, 483, 0, 0, 326, 516, - 493, 434, 325, 0, 477, 366, 383, 363, 450, 0, - 0, 515, 545, 362, 535, 0, 526, 328, 0, 525, - 449, 512, 517, 435, 428, 0, 327, 514, 433, 427, - 412, 373, 561, 413, 414, 387, 464, 425, 465, 388, - 439, 438, 440, 389, 390, 391, 392, 393, 394, 395, - 396, 397, 398, 0, 0, 0, 0, 0, 556, 557, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 689, 0, 0, 693, 0, - 528, 0, 0, 0, 0, 0, 0, 497, 0, 0, - 415, 0, 0, 0, 546, 0, 480, 455, 731, 0, - 0, 478, 423, 513, 466, 519, 500, 527, 472, 467, - 318, 501, 365, 436, 334, 336, 721, 367, 370, 374, - 375, 445, 446, 460, 485, 504, 505, 506, 364, 348, - 479, 349, 384, 350, 319, 356, 354, 357, 487, 358, - 321, 461, 510, 0, 380, 475, 431, 322, 430, 462, - 509, 508, 335, 536, 543, 544, 634, 0, 549, 732, - 733, 734, 558, 0, 468, 331, 330, 0, 0, 0, - 360, 463, 344, 346, 347, 345, 458, 459, 563, 564, - 565, 567, 0, 568, 569, 0, 0, 0, 0, 570, - 635, 651, 619, 588, 551, 643, 585, 589, 590, 401, - 402, 403, 654, 0, 0, 0, 542, 416, 417, 0, - 372, 371, 432, 323, 0, 0, 409, 400, 469, 329, - 368, 411, 405, 418, 419, 420, 378, 313, 314, 727, - 361, 451, 656, 691, 692, 581, 0, 644, 582, 591, - 353, 616, 628, 627, 447, 541, 0, 639, 642, 571, - 726, 0, 636, 650, 730, 649, 723, 457, 0, 484, - 647, 594, 0, 640, 613, 614, 0, 641, 609, 645, - 0, 583, 0, 552, 555, 584, 669, 670, 671, 320, - 554, 673, 674, 675, 676, 677, 678, 679, 672, 524, - 617, 593, 620, 533, 596, 595, 0, 0, 631, 550, - 632, 633, 441, 442, 443, 444, 382, 657, 342, 553, - 471, 0, 618, 0, 0, 0, 0, 0, 0, 0, - 0, 623, 624, 621, 735, 0, 680, 681, 0, 0, - 547, 548, 377, 0, 566, 385, 341, 456, 379, 531, - 408, 0, 559, 625, 560, 473, 474, 683, 688, 684, - 685, 687, 707, 448, 399, 404, 488, 410, 424, 476, - 530, 454, 481, 339, 520, 490, 429, 610, 638, 0, + 0, 0, 0, 0, 0, 497, 526, 0, 539, 0, + 407, 408, 0, 0, 0, 0, 0, 0, 0, 324, + 504, 523, 338, 491, 537, 343, 499, 516, 333, 457, + 488, 0, 0, 326, 521, 498, 439, 325, 0, 482, + 366, 383, 363, 455, 0, 0, 520, 550, 362, 540, + 0, 531, 328, 0, 530, 454, 517, 522, 440, 433, + 0, 327, 519, 438, 432, 413, 373, 566, 414, 415, + 416, 417, 418, 419, 387, 469, 430, 470, 388, 444, + 443, 445, 389, 390, 391, 392, 393, 394, 395, 396, + 397, 398, 0, 0, 0, 0, 0, 561, 562, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 694, 0, 0, 698, 0, 533, + 0, 0, 0, 0, 0, 0, 502, 0, 0, 420, + 0, 0, 0, 551, 0, 485, 460, 736, 0, 0, + 483, 428, 518, 471, 524, 505, 532, 477, 472, 318, + 506, 365, 441, 334, 336, 726, 367, 370, 374, 375, + 450, 451, 465, 490, 509, 510, 511, 364, 348, 484, + 349, 384, 350, 319, 356, 354, 357, 492, 358, 321, + 466, 515, 0, 380, 480, 436, 322, 435, 467, 514, + 513, 335, 541, 548, 549, 639, 0, 554, 737, 738, + 739, 563, 0, 473, 331, 330, 0, 0, 0, 360, + 468, 344, 346, 347, 345, 463, 464, 568, 569, 570, + 572, 0, 573, 574, 0, 0, 0, 0, 575, 640, + 656, 624, 593, 556, 648, 590, 594, 595, 401, 402, + 403, 404, 659, 0, 0, 0, 547, 421, 422, 0, + 372, 371, 437, 323, 0, 0, 410, 400, 474, 329, + 368, 412, 406, 423, 424, 425, 378, 313, 314, 732, + 361, 456, 661, 696, 697, 586, 0, 649, 587, 596, + 353, 621, 633, 632, 452, 546, 0, 644, 647, 576, + 731, 0, 641, 655, 735, 654, 728, 462, 0, 489, + 652, 599, 0, 645, 618, 619, 0, 646, 614, 650, + 0, 588, 0, 557, 560, 589, 674, 675, 676, 320, + 559, 678, 679, 680, 681, 682, 683, 684, 677, 529, + 622, 598, 625, 538, 601, 600, 0, 0, 636, 555, + 637, 638, 446, 447, 448, 449, 382, 662, 342, 558, + 476, 0, 623, 0, 0, 0, 0, 0, 0, 0, + 0, 628, 629, 626, 740, 0, 685, 686, 0, 0, + 552, 553, 377, 0, 571, 385, 341, 461, 379, 536, + 409, 0, 564, 630, 565, 478, 479, 688, 693, 689, + 690, 692, 712, 453, 399, 405, 493, 411, 429, 481, + 535, 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 665, 664, 663, 662, - 661, 660, 659, 658, 0, 0, 607, 507, 355, 307, - 351, 352, 359, 724, 720, 725, 708, 711, 710, 686, - 0, 315, 587, 422, 470, 376, 652, 653, 0, 706, + 0, 0, 0, 0, 0, 0, 670, 669, 668, 667, + 666, 665, 664, 663, 0, 0, 612, 512, 355, 307, + 351, 352, 359, 729, 725, 730, 713, 716, 715, 691, + 0, 315, 592, 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, - 282, 283, 284, 285, 655, 276, 277, 286, 287, 288, + 282, 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, - 299, 0, 0, 0, 0, 309, 712, 713, 714, 715, - 716, 0, 0, 310, 311, 312, 0, 0, 274, 275, - 302, 498, 303, 304, 305, 306, 0, 0, 537, 538, - 539, 562, 0, 540, 522, 586, 386, 316, 502, 529, - 722, 0, 0, 0, 0, 0, 0, 0, 637, 648, - 682, 0, 694, 695, 697, 699, 698, 701, 495, 496, - 709, 0, 0, 703, 704, 705, 702, 426, 482, 503, - 489, 0, 728, 577, 578, 729, 690, 317, 453, 0, - 0, 592, 626, 615, 700, 580, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 369, 0, 0, 421, - 630, 611, 622, 612, 597, 598, 599, 606, 381, 600, - 601, 602, 572, 603, 573, 604, 605, 0, 629, 579, - 491, 437, 0, 646, 0, 0, 0, 0, 0, 0, + 299, 0, 0, 0, 0, 309, 717, 718, 719, 720, + 721, 0, 0, 310, 311, 312, 0, 0, 274, 275, + 302, 503, 303, 304, 305, 306, 0, 0, 542, 543, + 544, 567, 0, 545, 527, 591, 386, 316, 507, 534, + 727, 0, 0, 0, 0, 0, 0, 0, 642, 653, + 687, 0, 699, 700, 702, 704, 703, 706, 500, 501, + 714, 0, 0, 708, 709, 710, 707, 431, 487, 508, + 494, 0, 733, 582, 583, 734, 695, 317, 3561, 0, + 0, 0, 0, 0, 458, 0, 0, 597, 631, 620, + 705, 585, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 369, 0, 0, 426, 635, 616, 627, 617, + 602, 603, 604, 611, 381, 605, 606, 607, 577, 608, + 578, 609, 610, 0, 634, 584, 496, 442, 0, 651, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 245, 0, 0, 0, 0, 0, 0, 337, 246, 579, + 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, - 0, 337, 246, 574, 696, 576, 575, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 3758, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 492, 521, 0, 534, - 0, 406, 407, 0, 0, 0, 0, 0, 0, 0, - 324, 499, 518, 338, 486, 532, 343, 494, 511, 333, - 452, 483, 0, 0, 326, 516, 493, 434, 325, 0, - 477, 366, 383, 363, 450, 0, 0, 515, 545, 362, - 535, 0, 526, 328, 0, 525, 449, 512, 517, 435, - 428, 0, 327, 514, 433, 427, 412, 373, 561, 413, - 414, 387, 464, 425, 465, 388, 439, 438, 440, 389, + 0, 0, 497, 526, 0, 539, 0, 407, 408, 0, + 0, 0, 0, 0, 0, 0, 324, 504, 523, 338, + 491, 537, 343, 499, 516, 333, 457, 488, 0, 0, + 326, 521, 498, 439, 325, 0, 482, 366, 383, 363, + 455, 0, 0, 520, 550, 362, 540, 0, 531, 328, + 0, 530, 454, 517, 522, 440, 433, 0, 327, 519, + 438, 432, 413, 373, 566, 414, 415, 416, 417, 418, + 419, 387, 469, 430, 470, 388, 444, 443, 445, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, - 0, 0, 0, 0, 556, 557, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 689, 0, 0, 693, 0, 528, 0, 0, 0, - 0, 0, 0, 497, 0, 0, 415, 0, 0, 0, - 546, 0, 480, 455, 731, 0, 0, 478, 423, 513, - 466, 519, 500, 527, 472, 467, 318, 501, 365, 436, - 334, 336, 721, 367, 370, 374, 375, 445, 446, 460, - 485, 504, 505, 506, 364, 348, 479, 349, 384, 350, - 319, 356, 354, 357, 487, 358, 321, 461, 510, 0, - 380, 475, 431, 322, 430, 462, 509, 508, 335, 536, - 543, 544, 634, 0, 549, 732, 733, 734, 558, 0, - 468, 331, 330, 0, 0, 0, 360, 463, 344, 346, - 347, 345, 458, 459, 563, 564, 565, 567, 0, 568, - 569, 0, 0, 0, 0, 570, 635, 651, 619, 588, - 551, 643, 585, 589, 590, 401, 402, 403, 654, 0, - 0, 0, 542, 416, 417, 0, 372, 371, 432, 323, - 0, 0, 409, 400, 469, 329, 368, 411, 405, 418, - 419, 420, 378, 313, 314, 727, 361, 451, 656, 691, - 692, 581, 0, 644, 582, 591, 353, 616, 628, 627, - 447, 541, 0, 639, 642, 571, 726, 0, 636, 650, - 730, 649, 723, 457, 0, 484, 647, 594, 0, 640, - 613, 614, 0, 641, 609, 645, 0, 583, 0, 552, - 555, 584, 669, 670, 671, 320, 554, 673, 674, 675, - 676, 677, 678, 679, 672, 524, 617, 593, 620, 533, - 596, 595, 0, 0, 631, 550, 632, 633, 441, 442, - 443, 444, 382, 657, 342, 553, 471, 0, 618, 0, - 0, 0, 0, 0, 0, 0, 0, 623, 624, 621, - 735, 0, 680, 681, 0, 0, 547, 548, 377, 0, - 566, 385, 341, 456, 379, 531, 408, 0, 559, 625, - 560, 473, 474, 683, 688, 684, 685, 687, 707, 448, - 399, 404, 488, 410, 424, 476, 530, 454, 481, 339, - 520, 490, 429, 610, 638, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, + 0, 0, 0, 0, 561, 562, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 694, 0, 0, 698, 0, 533, 0, 0, 0, + 0, 0, 0, 502, 0, 0, 420, 0, 0, 0, + 551, 0, 485, 460, 736, 0, 0, 483, 428, 518, + 471, 524, 505, 532, 477, 472, 318, 506, 365, 441, + 334, 336, 726, 367, 370, 374, 375, 450, 451, 465, + 490, 509, 510, 511, 364, 348, 484, 349, 384, 350, + 319, 356, 354, 357, 492, 358, 321, 466, 515, 0, + 380, 480, 436, 322, 435, 467, 514, 513, 335, 541, + 548, 549, 639, 0, 554, 737, 738, 739, 563, 0, + 473, 331, 330, 0, 0, 0, 360, 468, 344, 346, + 347, 345, 463, 464, 568, 569, 570, 572, 0, 573, + 574, 0, 0, 0, 0, 575, 640, 656, 624, 593, + 556, 648, 590, 594, 595, 401, 402, 403, 404, 659, + 0, 0, 0, 547, 421, 422, 0, 372, 371, 437, + 323, 0, 0, 410, 400, 474, 329, 368, 412, 406, + 423, 424, 425, 378, 313, 314, 732, 361, 456, 661, + 696, 697, 586, 0, 649, 587, 596, 353, 621, 633, + 632, 452, 546, 0, 644, 647, 576, 731, 0, 641, + 655, 735, 654, 728, 462, 0, 489, 652, 599, 0, + 645, 618, 619, 0, 646, 614, 650, 0, 588, 0, + 557, 560, 589, 674, 675, 676, 320, 559, 678, 679, + 680, 681, 682, 683, 684, 677, 529, 622, 598, 625, + 538, 601, 600, 0, 0, 636, 555, 637, 638, 446, + 447, 448, 449, 382, 662, 342, 558, 476, 0, 623, + 0, 0, 0, 0, 0, 0, 0, 0, 628, 629, + 626, 740, 0, 685, 686, 0, 0, 552, 553, 377, + 0, 571, 385, 341, 461, 379, 536, 409, 0, 564, + 630, 565, 478, 479, 688, 693, 689, 690, 692, 712, + 453, 399, 405, 493, 411, 429, 481, 535, 459, 486, + 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 665, 664, 663, 662, 661, 660, 659, 658, - 0, 0, 607, 507, 355, 307, 351, 352, 359, 724, - 720, 725, 708, 711, 710, 686, 0, 315, 587, 422, - 470, 376, 652, 653, 0, 706, 259, 260, 261, 262, - 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, - 272, 273, 278, 279, 280, 281, 282, 283, 284, 285, - 655, 276, 277, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, - 0, 309, 712, 713, 714, 715, 716, 0, 0, 310, - 311, 312, 0, 0, 274, 275, 302, 498, 303, 304, - 305, 306, 0, 0, 537, 538, 539, 562, 0, 540, - 522, 586, 386, 316, 502, 529, 722, 0, 0, 0, - 0, 0, 0, 0, 637, 648, 682, 0, 694, 695, - 697, 699, 698, 701, 495, 496, 709, 0, 0, 703, - 704, 705, 702, 426, 482, 503, 489, 0, 728, 577, - 578, 729, 690, 317, 453, 0, 0, 592, 626, 615, - 700, 580, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 369, 0, 0, 421, 630, 611, 622, 612, - 597, 598, 599, 606, 381, 600, 601, 602, 572, 603, - 573, 604, 605, 0, 629, 579, 491, 437, 0, 646, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 3608, 0, 0, 0, 337, 246, 574, - 696, 576, 575, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 670, 669, 668, 667, 666, 665, 664, + 663, 0, 0, 612, 512, 355, 307, 351, 352, 359, + 729, 725, 730, 713, 716, 715, 691, 0, 315, 592, + 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, + 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, + 271, 272, 273, 278, 279, 280, 281, 282, 283, 284, + 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 298, 299, 0, 0, + 0, 0, 309, 717, 718, 719, 720, 721, 0, 0, + 310, 311, 312, 0, 0, 274, 275, 302, 503, 303, + 304, 305, 306, 0, 0, 542, 543, 544, 567, 0, + 545, 527, 591, 386, 316, 507, 534, 727, 0, 0, + 0, 0, 0, 0, 0, 642, 653, 687, 0, 699, + 700, 702, 704, 703, 706, 500, 501, 714, 0, 0, + 708, 709, 710, 707, 431, 487, 508, 494, 0, 733, + 582, 583, 734, 695, 317, 458, 0, 0, 597, 631, + 620, 705, 585, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 369, 0, 0, 426, 635, 616, 627, + 617, 602, 603, 604, 611, 381, 605, 606, 607, 577, + 608, 578, 609, 610, 0, 634, 584, 496, 442, 0, + 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 245, 0, 0, 0, 0, 0, 0, 337, 246, + 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 492, 521, 0, 534, 0, 406, 407, 0, - 0, 0, 0, 0, 0, 0, 324, 499, 518, 338, - 486, 532, 343, 494, 511, 333, 452, 483, 0, 0, - 326, 516, 493, 434, 325, 0, 477, 366, 383, 363, - 450, 0, 0, 515, 545, 362, 535, 0, 526, 328, - 0, 525, 449, 512, 517, 435, 428, 0, 327, 514, - 433, 427, 412, 373, 561, 413, 414, 387, 464, 425, - 465, 388, 439, 438, 440, 389, 390, 391, 392, 393, - 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, - 556, 557, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 689, 0, 0, - 693, 0, 528, 0, 0, 0, 0, 0, 0, 497, - 0, 0, 415, 0, 0, 0, 546, 0, 480, 455, - 731, 0, 0, 478, 423, 513, 466, 519, 500, 527, - 472, 467, 318, 501, 365, 436, 334, 336, 721, 367, - 370, 374, 375, 445, 446, 460, 485, 504, 505, 506, - 364, 348, 479, 349, 384, 350, 319, 356, 354, 357, - 487, 358, 321, 461, 510, 0, 380, 475, 431, 322, - 430, 462, 509, 508, 335, 536, 543, 544, 634, 0, - 549, 732, 733, 734, 558, 0, 468, 331, 330, 0, - 0, 0, 360, 463, 344, 346, 347, 345, 458, 459, - 563, 564, 565, 567, 0, 568, 569, 0, 0, 0, - 0, 570, 635, 651, 619, 588, 551, 643, 585, 589, - 590, 401, 402, 403, 654, 0, 0, 0, 542, 416, - 417, 0, 372, 371, 432, 323, 0, 0, 409, 400, - 469, 329, 368, 411, 405, 418, 419, 420, 378, 313, - 314, 727, 361, 451, 656, 691, 692, 581, 0, 644, - 582, 591, 353, 616, 628, 627, 447, 541, 0, 639, - 642, 571, 726, 0, 636, 650, 730, 649, 723, 457, - 0, 484, 647, 594, 0, 640, 613, 614, 0, 641, - 609, 645, 0, 583, 0, 552, 555, 584, 669, 670, - 671, 320, 554, 673, 674, 675, 676, 677, 678, 679, - 672, 524, 617, 593, 620, 533, 596, 595, 0, 0, - 631, 550, 632, 633, 441, 442, 443, 444, 382, 657, - 342, 553, 471, 0, 618, 0, 0, 0, 0, 0, - 0, 0, 0, 623, 624, 621, 735, 0, 680, 681, - 0, 0, 547, 548, 377, 0, 566, 385, 341, 456, - 379, 531, 408, 0, 559, 625, 560, 473, 474, 683, - 688, 684, 685, 687, 707, 448, 399, 404, 488, 410, - 424, 476, 530, 454, 481, 339, 520, 490, 429, 610, - 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 665, 664, - 663, 662, 661, 660, 659, 658, 0, 0, 607, 507, - 355, 307, 351, 352, 359, 724, 720, 725, 708, 711, - 710, 686, 0, 315, 587, 422, 470, 376, 652, 653, - 0, 706, 259, 260, 261, 262, 263, 264, 265, 266, - 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, - 280, 281, 282, 283, 284, 285, 655, 276, 277, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 298, 299, 0, 0, 0, 0, 309, 712, 713, - 714, 715, 716, 0, 0, 310, 311, 312, 0, 0, - 274, 275, 302, 498, 303, 304, 305, 306, 0, 0, - 537, 538, 539, 562, 0, 540, 522, 586, 386, 316, - 502, 529, 722, 0, 0, 0, 0, 0, 0, 0, - 637, 648, 682, 0, 694, 695, 697, 699, 698, 701, - 495, 496, 709, 0, 0, 703, 704, 705, 702, 426, - 482, 503, 489, 0, 728, 577, 578, 729, 690, 317, - 3537, 0, 0, 0, 0, 0, 453, 0, 0, 592, - 626, 615, 700, 580, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 369, 0, 0, 421, 630, 611, - 622, 612, 597, 598, 599, 606, 381, 600, 601, 602, - 572, 603, 573, 604, 605, 0, 629, 579, 491, 437, - 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 3454, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 497, 526, 0, 539, 0, 407, 408, + 0, 0, 0, 0, 0, 0, 0, 324, 504, 523, + 338, 491, 537, 343, 499, 516, 333, 457, 488, 0, + 0, 326, 521, 498, 439, 325, 0, 482, 366, 383, + 363, 455, 0, 0, 520, 550, 362, 540, 0, 531, + 328, 0, 530, 454, 517, 522, 440, 433, 0, 327, + 519, 438, 432, 413, 373, 566, 414, 415, 416, 417, + 418, 419, 387, 469, 430, 470, 388, 444, 443, 445, + 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, + 0, 0, 0, 0, 0, 561, 562, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 694, 0, 0, 698, 0, 533, 0, 0, + 0, 0, 0, 0, 502, 0, 0, 420, 0, 0, + 0, 551, 0, 485, 460, 736, 0, 0, 483, 428, + 518, 471, 524, 505, 532, 477, 472, 318, 506, 365, + 441, 334, 336, 726, 367, 370, 374, 375, 450, 451, + 465, 490, 509, 510, 511, 364, 348, 484, 349, 384, + 350, 319, 356, 354, 357, 492, 358, 321, 466, 515, + 0, 380, 480, 436, 322, 435, 467, 514, 513, 335, + 541, 548, 549, 639, 0, 554, 737, 738, 739, 563, + 0, 473, 331, 330, 0, 0, 0, 360, 468, 344, + 346, 347, 345, 463, 464, 568, 569, 570, 572, 0, + 573, 574, 0, 0, 0, 0, 575, 640, 656, 624, + 593, 556, 648, 590, 594, 595, 401, 402, 403, 404, + 659, 0, 0, 0, 547, 421, 422, 0, 372, 371, + 437, 323, 0, 0, 410, 400, 474, 329, 368, 412, + 406, 423, 424, 425, 378, 313, 314, 732, 361, 456, + 661, 696, 697, 586, 0, 649, 587, 596, 353, 621, + 633, 632, 452, 546, 0, 644, 647, 576, 731, 0, + 641, 655, 735, 654, 728, 462, 0, 489, 652, 599, + 0, 645, 618, 619, 0, 646, 614, 650, 0, 588, + 0, 557, 560, 589, 674, 675, 676, 320, 559, 678, + 679, 680, 681, 682, 683, 684, 677, 529, 622, 598, + 625, 538, 601, 600, 0, 0, 636, 555, 637, 638, + 446, 447, 448, 449, 382, 662, 342, 558, 476, 0, + 623, 0, 0, 0, 0, 0, 0, 0, 0, 628, + 629, 626, 740, 0, 685, 686, 0, 0, 552, 553, + 377, 0, 571, 385, 341, 461, 379, 536, 409, 0, + 564, 630, 565, 478, 479, 688, 693, 689, 690, 692, + 712, 453, 399, 405, 493, 411, 429, 481, 535, 459, + 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 245, 0, 0, 0, 0, 0, 0, 337, - 246, 574, 696, 576, 575, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 670, 669, 668, 667, 666, 665, + 664, 663, 0, 0, 612, 512, 355, 307, 351, 352, + 359, 729, 725, 730, 713, 716, 715, 691, 0, 315, + 592, 427, 475, 376, 657, 658, 0, 711, 259, 260, + 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, + 270, 271, 272, 273, 278, 279, 280, 281, 282, 283, + 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, + 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, + 0, 0, 0, 309, 717, 718, 719, 720, 721, 0, + 0, 310, 311, 312, 0, 0, 274, 275, 302, 503, + 303, 304, 305, 306, 0, 0, 542, 543, 544, 567, + 0, 545, 527, 591, 386, 316, 507, 534, 727, 0, + 0, 0, 0, 0, 0, 0, 642, 653, 687, 0, + 699, 700, 702, 704, 703, 706, 500, 501, 714, 0, + 0, 708, 709, 710, 707, 431, 487, 508, 494, 0, + 733, 582, 583, 734, 695, 317, 458, 0, 0, 597, + 631, 620, 705, 585, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 369, 0, 0, 426, 635, 616, + 627, 617, 602, 603, 604, 611, 381, 605, 606, 607, + 577, 608, 578, 609, 610, 0, 634, 584, 496, 442, + 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 245, 0, 0, 1734, 0, 0, 0, 337, + 246, 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 492, 521, 0, 534, 0, 406, - 407, 0, 0, 0, 0, 0, 0, 0, 324, 499, - 518, 338, 486, 532, 343, 494, 511, 333, 452, 483, - 0, 0, 326, 516, 493, 434, 325, 0, 477, 366, - 383, 363, 450, 0, 0, 515, 545, 362, 535, 0, - 526, 328, 0, 525, 449, 512, 517, 435, 428, 0, - 327, 514, 433, 427, 412, 373, 561, 413, 414, 387, - 464, 425, 465, 388, 439, 438, 440, 389, 390, 391, - 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, - 0, 0, 556, 557, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, - 0, 0, 693, 0, 528, 0, 0, 0, 0, 0, - 0, 497, 0, 0, 415, 0, 0, 0, 546, 0, - 480, 455, 731, 0, 0, 478, 423, 513, 466, 519, - 500, 527, 472, 467, 318, 501, 365, 436, 334, 336, - 721, 367, 370, 374, 375, 445, 446, 460, 485, 504, - 505, 506, 364, 348, 479, 349, 384, 350, 319, 356, - 354, 357, 487, 358, 321, 461, 510, 0, 380, 475, - 431, 322, 430, 462, 509, 508, 335, 536, 543, 544, - 634, 0, 549, 732, 733, 734, 558, 0, 468, 331, - 330, 0, 0, 0, 360, 463, 344, 346, 347, 345, - 458, 459, 563, 564, 565, 567, 0, 568, 569, 0, - 0, 0, 0, 570, 635, 651, 619, 588, 551, 643, - 585, 589, 590, 401, 402, 403, 654, 0, 0, 0, - 542, 416, 417, 0, 372, 371, 432, 323, 0, 0, - 409, 400, 469, 329, 368, 411, 405, 418, 419, 420, - 378, 313, 314, 727, 361, 451, 656, 691, 692, 581, - 0, 644, 582, 591, 353, 616, 628, 627, 447, 541, - 0, 639, 642, 571, 726, 0, 636, 650, 730, 649, - 723, 457, 0, 484, 647, 594, 0, 640, 613, 614, - 0, 641, 609, 645, 0, 583, 0, 552, 555, 584, - 669, 670, 671, 320, 554, 673, 674, 675, 676, 677, - 678, 679, 672, 524, 617, 593, 620, 533, 596, 595, - 0, 0, 631, 550, 632, 633, 441, 442, 443, 444, - 382, 657, 342, 553, 471, 0, 618, 0, 0, 0, - 0, 0, 0, 0, 0, 623, 624, 621, 735, 0, - 680, 681, 0, 0, 547, 548, 377, 0, 566, 385, - 341, 456, 379, 531, 408, 0, 559, 625, 560, 473, - 474, 683, 688, 684, 685, 687, 707, 448, 399, 404, - 488, 410, 424, 476, 530, 454, 481, 339, 520, 490, - 429, 610, 638, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, + 0, 0, 0, 0, 497, 526, 0, 539, 0, 407, + 408, 0, 0, 0, 0, 0, 0, 0, 324, 504, + 523, 338, 491, 537, 343, 499, 516, 333, 457, 488, + 0, 0, 326, 521, 498, 439, 325, 0, 482, 366, + 383, 363, 455, 0, 0, 520, 550, 362, 540, 0, + 531, 328, 0, 530, 454, 517, 522, 440, 433, 0, + 327, 519, 438, 432, 413, 373, 566, 414, 415, 416, + 417, 418, 419, 387, 469, 430, 470, 388, 444, 443, + 445, 389, 390, 391, 392, 393, 394, 395, 396, 397, + 398, 0, 0, 0, 0, 0, 561, 562, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 694, 0, 0, 698, 0, 533, 0, + 0, 0, 0, 0, 0, 502, 0, 0, 420, 0, + 0, 0, 551, 0, 485, 460, 736, 0, 0, 483, + 428, 518, 471, 524, 505, 532, 477, 472, 318, 506, + 365, 441, 334, 336, 726, 367, 370, 374, 375, 450, + 451, 465, 490, 509, 510, 511, 364, 348, 484, 349, + 384, 350, 319, 356, 354, 357, 492, 358, 321, 466, + 515, 0, 380, 480, 436, 322, 435, 467, 514, 513, + 335, 541, 548, 549, 639, 0, 554, 737, 738, 739, + 563, 0, 473, 331, 330, 0, 0, 0, 360, 468, + 344, 346, 347, 345, 463, 464, 568, 569, 570, 572, + 0, 573, 574, 0, 0, 0, 0, 575, 640, 656, + 624, 593, 556, 648, 590, 594, 595, 401, 402, 403, + 404, 659, 0, 0, 0, 547, 421, 422, 0, 372, + 371, 437, 323, 0, 0, 410, 400, 474, 329, 368, + 412, 406, 423, 424, 425, 378, 313, 314, 732, 361, + 456, 661, 696, 697, 586, 0, 649, 587, 596, 353, + 621, 633, 632, 452, 546, 0, 644, 647, 576, 731, + 0, 641, 655, 735, 654, 728, 462, 0, 489, 652, + 599, 0, 645, 618, 619, 0, 646, 614, 650, 0, + 588, 0, 557, 560, 589, 674, 675, 676, 320, 559, + 678, 679, 680, 681, 682, 683, 684, 677, 529, 622, + 598, 625, 538, 601, 600, 0, 0, 636, 555, 637, + 638, 446, 447, 448, 449, 382, 662, 342, 558, 476, + 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, + 628, 629, 626, 740, 0, 685, 686, 0, 0, 552, + 553, 377, 0, 571, 385, 341, 461, 379, 536, 409, + 0, 564, 630, 565, 478, 479, 688, 693, 689, 690, + 692, 712, 453, 399, 405, 493, 411, 429, 481, 535, + 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, + 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 670, 669, 668, 667, 666, + 665, 664, 663, 0, 0, 612, 512, 355, 307, 351, + 352, 359, 729, 725, 730, 713, 716, 715, 691, 0, + 315, 592, 427, 475, 376, 657, 658, 0, 711, 259, + 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, + 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, + 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, + 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, + 0, 0, 0, 0, 309, 717, 718, 719, 720, 721, + 0, 0, 310, 311, 312, 0, 0, 274, 275, 302, + 503, 303, 304, 305, 306, 0, 0, 542, 543, 544, + 567, 0, 545, 527, 591, 386, 316, 507, 534, 727, + 0, 0, 0, 0, 0, 0, 0, 642, 653, 687, + 0, 699, 700, 702, 704, 703, 706, 500, 501, 714, + 0, 0, 708, 709, 710, 707, 431, 487, 508, 494, + 0, 733, 582, 583, 734, 695, 317, 458, 0, 0, + 597, 631, 620, 705, 585, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 369, 0, 0, 426, 635, + 616, 627, 617, 602, 603, 604, 611, 381, 605, 606, + 607, 577, 608, 578, 609, 610, 0, 634, 584, 496, + 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 245, 0, 0, 2825, 0, 0, 0, + 337, 246, 579, 701, 581, 580, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 665, 664, 663, 662, 661, 660, 659, 658, 0, 0, - 607, 507, 355, 307, 351, 352, 359, 724, 720, 725, - 708, 711, 710, 686, 0, 315, 587, 422, 470, 376, - 652, 653, 0, 706, 259, 260, 261, 262, 263, 264, - 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, - 278, 279, 280, 281, 282, 283, 284, 285, 655, 276, - 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 298, 299, 0, 0, 0, 0, 309, - 712, 713, 714, 715, 716, 0, 0, 310, 311, 312, - 0, 0, 274, 275, 302, 498, 303, 304, 305, 306, - 0, 0, 537, 538, 539, 562, 0, 540, 522, 586, - 386, 316, 502, 529, 722, 0, 0, 0, 0, 0, - 0, 0, 637, 648, 682, 0, 694, 695, 697, 699, - 698, 701, 495, 496, 709, 0, 0, 703, 704, 705, - 702, 426, 482, 503, 489, 0, 728, 577, 578, 729, - 690, 317, 453, 0, 0, 592, 626, 615, 700, 580, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 369, 0, 0, 421, 630, 611, 622, 612, 597, 598, - 599, 606, 381, 600, 601, 602, 572, 603, 573, 604, - 605, 0, 629, 579, 491, 437, 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 0, 0, 0, 0, 337, 246, 574, 696, 576, - 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 3431, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 492, 521, 0, 534, 0, 406, 407, 0, 0, 0, - 0, 0, 0, 0, 324, 499, 518, 338, 486, 532, - 343, 494, 511, 333, 452, 483, 0, 0, 326, 516, - 493, 434, 325, 0, 477, 366, 383, 363, 450, 0, - 0, 515, 545, 362, 535, 0, 526, 328, 0, 525, - 449, 512, 517, 435, 428, 0, 327, 514, 433, 427, - 412, 373, 561, 413, 414, 387, 464, 425, 465, 388, - 439, 438, 440, 389, 390, 391, 392, 393, 394, 395, - 396, 397, 398, 0, 0, 0, 0, 0, 556, 557, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 689, 0, 0, 693, 0, - 528, 0, 0, 0, 0, 0, 0, 497, 0, 0, - 415, 0, 0, 0, 546, 0, 480, 455, 731, 0, - 0, 478, 423, 513, 466, 519, 500, 527, 472, 467, - 318, 501, 365, 436, 334, 336, 721, 367, 370, 374, - 375, 445, 446, 460, 485, 504, 505, 506, 364, 348, - 479, 349, 384, 350, 319, 356, 354, 357, 487, 358, - 321, 461, 510, 0, 380, 475, 431, 322, 430, 462, - 509, 508, 335, 536, 543, 544, 634, 0, 549, 732, - 733, 734, 558, 0, 468, 331, 330, 0, 0, 0, - 360, 463, 344, 346, 347, 345, 458, 459, 563, 564, - 565, 567, 0, 568, 569, 0, 0, 0, 0, 570, - 635, 651, 619, 588, 551, 643, 585, 589, 590, 401, - 402, 403, 654, 0, 0, 0, 542, 416, 417, 0, - 372, 371, 432, 323, 0, 0, 409, 400, 469, 329, - 368, 411, 405, 418, 419, 420, 378, 313, 314, 727, - 361, 451, 656, 691, 692, 581, 0, 644, 582, 591, - 353, 616, 628, 627, 447, 541, 0, 639, 642, 571, - 726, 0, 636, 650, 730, 649, 723, 457, 0, 484, - 647, 594, 0, 640, 613, 614, 0, 641, 609, 645, - 0, 583, 0, 552, 555, 584, 669, 670, 671, 320, - 554, 673, 674, 675, 676, 677, 678, 679, 672, 524, - 617, 593, 620, 533, 596, 595, 0, 0, 631, 550, - 632, 633, 441, 442, 443, 444, 382, 657, 342, 553, - 471, 0, 618, 0, 0, 0, 0, 0, 0, 0, - 0, 623, 624, 621, 735, 0, 680, 681, 0, 0, - 547, 548, 377, 0, 566, 385, 341, 456, 379, 531, - 408, 0, 559, 625, 560, 473, 474, 683, 688, 684, - 685, 687, 707, 448, 399, 404, 488, 410, 424, 476, - 530, 454, 481, 339, 520, 490, 429, 610, 638, 0, + 0, 0, 0, 0, 0, 497, 526, 0, 539, 0, + 407, 408, 0, 0, 0, 0, 0, 0, 0, 324, + 504, 523, 338, 491, 537, 343, 499, 516, 333, 457, + 488, 0, 0, 326, 521, 498, 439, 325, 0, 482, + 366, 383, 363, 455, 0, 0, 520, 550, 362, 540, + 0, 531, 328, 0, 530, 454, 517, 522, 440, 433, + 0, 327, 519, 438, 432, 413, 373, 566, 414, 415, + 416, 417, 418, 419, 387, 469, 430, 470, 388, 444, + 443, 445, 389, 390, 391, 392, 393, 394, 395, 396, + 397, 398, 0, 0, 0, 0, 0, 561, 562, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 694, 0, 0, 698, 0, 533, + 0, 0, 0, 0, 0, 0, 502, 0, 0, 420, + 0, 0, 0, 551, 0, 485, 460, 736, 0, 0, + 483, 428, 518, 471, 524, 505, 532, 477, 472, 318, + 506, 365, 441, 334, 336, 726, 367, 370, 374, 375, + 450, 451, 465, 490, 509, 510, 511, 364, 348, 484, + 349, 384, 350, 319, 356, 354, 357, 492, 358, 321, + 466, 515, 0, 380, 480, 436, 322, 435, 467, 514, + 513, 335, 541, 548, 549, 639, 0, 554, 737, 738, + 739, 563, 0, 473, 331, 330, 0, 0, 0, 360, + 468, 344, 346, 347, 345, 463, 464, 568, 569, 570, + 572, 0, 573, 574, 0, 0, 0, 0, 575, 640, + 656, 624, 593, 556, 648, 590, 594, 595, 401, 402, + 403, 404, 659, 0, 0, 0, 547, 421, 422, 0, + 372, 371, 437, 323, 0, 0, 410, 400, 474, 329, + 368, 412, 406, 423, 424, 425, 378, 313, 314, 732, + 361, 456, 661, 696, 697, 586, 0, 649, 587, 596, + 353, 621, 633, 632, 452, 546, 0, 644, 647, 576, + 731, 0, 641, 655, 735, 654, 728, 462, 0, 489, + 652, 599, 0, 645, 618, 619, 0, 646, 614, 650, + 0, 588, 0, 557, 560, 589, 674, 675, 676, 320, + 559, 678, 679, 680, 681, 682, 683, 684, 677, 529, + 622, 598, 625, 538, 601, 600, 0, 0, 636, 555, + 637, 638, 446, 447, 448, 449, 382, 662, 342, 558, + 476, 0, 623, 0, 0, 0, 0, 0, 0, 0, + 0, 628, 629, 626, 740, 0, 685, 686, 0, 0, + 552, 553, 377, 0, 571, 385, 341, 461, 379, 536, + 409, 0, 564, 630, 565, 478, 479, 688, 693, 689, + 690, 692, 712, 453, 399, 405, 493, 411, 429, 481, + 535, 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 665, 664, 663, 662, - 661, 660, 659, 658, 0, 0, 607, 507, 355, 307, - 351, 352, 359, 724, 720, 725, 708, 711, 710, 686, - 0, 315, 587, 422, 470, 376, 652, 653, 0, 706, + 0, 0, 0, 0, 0, 0, 670, 669, 668, 667, + 666, 665, 664, 663, 0, 0, 612, 512, 355, 307, + 351, 352, 359, 729, 725, 730, 713, 716, 715, 691, + 0, 315, 592, 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, - 282, 283, 284, 285, 655, 276, 277, 286, 287, 288, + 282, 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, - 299, 0, 0, 0, 0, 309, 712, 713, 714, 715, - 716, 0, 0, 310, 311, 312, 0, 0, 274, 275, - 302, 498, 303, 304, 305, 306, 0, 0, 537, 538, - 539, 562, 0, 540, 522, 586, 386, 316, 502, 529, - 722, 0, 0, 0, 0, 0, 0, 0, 637, 648, - 682, 0, 694, 695, 697, 699, 698, 701, 495, 496, - 709, 0, 0, 703, 704, 705, 702, 426, 482, 503, - 489, 0, 728, 577, 578, 729, 690, 317, 453, 0, - 0, 592, 626, 615, 700, 580, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 369, 0, 0, 421, - 630, 611, 622, 612, 597, 598, 599, 606, 381, 600, - 601, 602, 572, 603, 573, 604, 605, 0, 629, 579, - 491, 437, 0, 646, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 245, 0, 0, 1723, 0, 0, - 0, 337, 246, 574, 696, 576, 575, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 299, 0, 0, 0, 0, 309, 717, 718, 719, 720, + 721, 0, 0, 310, 311, 312, 0, 0, 274, 275, + 302, 503, 303, 304, 305, 306, 0, 0, 542, 543, + 544, 567, 0, 545, 527, 591, 386, 316, 507, 534, + 727, 0, 0, 0, 0, 0, 0, 0, 642, 653, + 687, 0, 699, 700, 702, 704, 703, 706, 500, 501, + 714, 0, 0, 708, 709, 710, 707, 431, 487, 508, + 494, 0, 733, 582, 583, 734, 695, 317, 458, 0, + 0, 597, 631, 620, 705, 585, 0, 0, 3264, 0, + 0, 0, 0, 0, 0, 0, 369, 0, 0, 426, + 635, 616, 627, 617, 602, 603, 604, 611, 381, 605, + 606, 607, 577, 608, 578, 609, 610, 0, 634, 584, + 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, + 0, 337, 246, 579, 701, 581, 580, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 492, 521, 0, 534, - 0, 406, 407, 0, 0, 0, 0, 0, 0, 0, - 324, 499, 518, 338, 486, 532, 343, 494, 511, 333, - 452, 483, 0, 0, 326, 516, 493, 434, 325, 0, - 477, 366, 383, 363, 450, 0, 0, 515, 545, 362, - 535, 0, 526, 328, 0, 525, 449, 512, 517, 435, - 428, 0, 327, 514, 433, 427, 412, 373, 561, 413, - 414, 387, 464, 425, 465, 388, 439, 438, 440, 389, - 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, - 0, 0, 0, 0, 556, 557, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 689, 0, 0, 693, 0, 528, 0, 0, 0, - 0, 0, 0, 497, 0, 0, 415, 0, 0, 0, - 546, 0, 480, 455, 731, 0, 0, 478, 423, 513, - 466, 519, 500, 527, 472, 467, 318, 501, 365, 436, - 334, 336, 721, 367, 370, 374, 375, 445, 446, 460, - 485, 504, 505, 506, 364, 348, 479, 349, 384, 350, - 319, 356, 354, 357, 487, 358, 321, 461, 510, 0, - 380, 475, 431, 322, 430, 462, 509, 508, 335, 536, - 543, 544, 634, 0, 549, 732, 733, 734, 558, 0, - 468, 331, 330, 0, 0, 0, 360, 463, 344, 346, - 347, 345, 458, 459, 563, 564, 565, 567, 0, 568, - 569, 0, 0, 0, 0, 570, 635, 651, 619, 588, - 551, 643, 585, 589, 590, 401, 402, 403, 654, 0, - 0, 0, 542, 416, 417, 0, 372, 371, 432, 323, - 0, 0, 409, 400, 469, 329, 368, 411, 405, 418, - 419, 420, 378, 313, 314, 727, 361, 451, 656, 691, - 692, 581, 0, 644, 582, 591, 353, 616, 628, 627, - 447, 541, 0, 639, 642, 571, 726, 0, 636, 650, - 730, 649, 723, 457, 0, 484, 647, 594, 0, 640, - 613, 614, 0, 641, 609, 645, 0, 583, 0, 552, - 555, 584, 669, 670, 671, 320, 554, 673, 674, 675, - 676, 677, 678, 679, 672, 524, 617, 593, 620, 533, - 596, 595, 0, 0, 631, 550, 632, 633, 441, 442, - 443, 444, 382, 657, 342, 553, 471, 0, 618, 0, - 0, 0, 0, 0, 0, 0, 0, 623, 624, 621, - 735, 0, 680, 681, 0, 0, 547, 548, 377, 0, - 566, 385, 341, 456, 379, 531, 408, 0, 559, 625, - 560, 473, 474, 683, 688, 684, 685, 687, 707, 448, - 399, 404, 488, 410, 424, 476, 530, 454, 481, 339, - 520, 490, 429, 610, 638, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 665, 664, 663, 662, 661, 660, 659, 658, - 0, 0, 607, 507, 355, 307, 351, 352, 359, 724, - 720, 725, 708, 711, 710, 686, 0, 315, 587, 422, - 470, 376, 652, 653, 0, 706, 259, 260, 261, 262, - 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, - 272, 273, 278, 279, 280, 281, 282, 283, 284, 285, - 655, 276, 277, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, - 0, 309, 712, 713, 714, 715, 716, 0, 0, 310, - 311, 312, 0, 0, 274, 275, 302, 498, 303, 304, - 305, 306, 0, 0, 537, 538, 539, 562, 0, 540, - 522, 586, 386, 316, 502, 529, 722, 0, 0, 0, - 0, 0, 0, 0, 637, 648, 682, 0, 694, 695, - 697, 699, 698, 701, 495, 496, 709, 0, 0, 703, - 704, 705, 702, 426, 482, 503, 489, 0, 728, 577, - 578, 729, 690, 317, 453, 0, 0, 592, 626, 615, - 700, 580, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 369, 0, 0, 421, 630, 611, 622, 612, - 597, 598, 599, 606, 381, 600, 601, 602, 572, 603, - 573, 604, 605, 0, 629, 579, 491, 437, 0, 646, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 2808, 0, 0, 0, 337, 246, 574, - 696, 576, 575, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 497, 526, 0, 539, + 0, 407, 408, 0, 0, 0, 0, 0, 0, 0, + 324, 504, 523, 338, 491, 537, 343, 499, 516, 333, + 457, 488, 0, 0, 326, 521, 498, 439, 325, 0, + 482, 366, 383, 363, 455, 0, 0, 520, 550, 362, + 540, 0, 531, 328, 0, 530, 454, 517, 522, 440, + 433, 0, 327, 519, 438, 432, 413, 373, 566, 414, + 415, 416, 417, 418, 419, 387, 469, 430, 470, 388, + 444, 443, 445, 389, 390, 391, 392, 393, 394, 395, + 396, 397, 398, 0, 0, 0, 0, 0, 561, 562, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 694, 0, 0, 698, 0, + 533, 0, 0, 0, 0, 0, 0, 502, 0, 0, + 420, 0, 0, 0, 551, 0, 485, 460, 736, 0, + 0, 483, 428, 518, 471, 524, 505, 532, 477, 472, + 318, 506, 365, 441, 334, 336, 726, 367, 370, 374, + 375, 450, 451, 465, 490, 509, 510, 511, 364, 348, + 484, 349, 384, 350, 319, 356, 354, 357, 492, 358, + 321, 466, 515, 0, 380, 480, 436, 322, 435, 467, + 514, 513, 335, 541, 548, 549, 639, 0, 554, 737, + 738, 739, 563, 0, 473, 331, 330, 0, 0, 0, + 360, 468, 344, 346, 347, 345, 463, 464, 568, 569, + 570, 572, 0, 573, 574, 0, 0, 0, 0, 575, + 640, 656, 624, 593, 556, 648, 590, 594, 595, 401, + 402, 403, 404, 659, 0, 0, 0, 547, 421, 422, + 0, 372, 371, 437, 323, 0, 0, 410, 400, 474, + 329, 368, 412, 406, 423, 424, 425, 378, 313, 314, + 732, 361, 456, 661, 696, 697, 586, 0, 649, 587, + 596, 353, 621, 633, 632, 452, 546, 0, 644, 647, + 576, 731, 0, 641, 655, 735, 654, 728, 462, 0, + 489, 652, 599, 0, 645, 618, 619, 0, 646, 614, + 650, 0, 588, 0, 557, 560, 589, 674, 675, 676, + 320, 559, 678, 679, 680, 681, 682, 683, 684, 677, + 529, 622, 598, 625, 538, 601, 600, 0, 0, 636, + 555, 637, 638, 446, 447, 448, 449, 382, 662, 342, + 558, 476, 0, 623, 0, 0, 0, 0, 0, 0, + 0, 0, 628, 629, 626, 740, 0, 685, 686, 0, + 0, 552, 553, 377, 0, 571, 385, 341, 461, 379, + 536, 409, 0, 564, 630, 565, 478, 479, 688, 693, + 689, 690, 692, 712, 453, 399, 405, 493, 411, 429, + 481, 535, 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 492, 521, 0, 534, 0, 406, 407, 0, - 0, 0, 0, 0, 0, 0, 324, 499, 518, 338, - 486, 532, 343, 494, 511, 333, 452, 483, 0, 0, - 326, 516, 493, 434, 325, 0, 477, 366, 383, 363, - 450, 0, 0, 515, 545, 362, 535, 0, 526, 328, - 0, 525, 449, 512, 517, 435, 428, 0, 327, 514, - 433, 427, 412, 373, 561, 413, 414, 387, 464, 425, - 465, 388, 439, 438, 440, 389, 390, 391, 392, 393, - 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, - 556, 557, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 689, 0, 0, - 693, 0, 528, 0, 0, 0, 0, 0, 0, 497, - 0, 0, 415, 0, 0, 0, 546, 0, 480, 455, - 731, 0, 0, 478, 423, 513, 466, 519, 500, 527, - 472, 467, 318, 501, 365, 436, 334, 336, 721, 367, - 370, 374, 375, 445, 446, 460, 485, 504, 505, 506, - 364, 348, 479, 349, 384, 350, 319, 356, 354, 357, - 487, 358, 321, 461, 510, 0, 380, 475, 431, 322, - 430, 462, 509, 508, 335, 536, 543, 544, 634, 0, - 549, 732, 733, 734, 558, 0, 468, 331, 330, 0, - 0, 0, 360, 463, 344, 346, 347, 345, 458, 459, - 563, 564, 565, 567, 0, 568, 569, 0, 0, 0, - 0, 570, 635, 651, 619, 588, 551, 643, 585, 589, - 590, 401, 402, 403, 654, 0, 0, 0, 542, 416, - 417, 0, 372, 371, 432, 323, 0, 0, 409, 400, - 469, 329, 368, 411, 405, 418, 419, 420, 378, 313, - 314, 727, 361, 451, 656, 691, 692, 581, 0, 644, - 582, 591, 353, 616, 628, 627, 447, 541, 0, 639, - 642, 571, 726, 0, 636, 650, 730, 649, 723, 457, - 0, 484, 647, 594, 0, 640, 613, 614, 0, 641, - 609, 645, 0, 583, 0, 552, 555, 584, 669, 670, - 671, 320, 554, 673, 674, 675, 676, 677, 678, 679, - 672, 524, 617, 593, 620, 533, 596, 595, 0, 0, - 631, 550, 632, 633, 441, 442, 443, 444, 382, 657, - 342, 553, 471, 0, 618, 0, 0, 0, 0, 0, - 0, 0, 0, 623, 624, 621, 735, 0, 680, 681, - 0, 0, 547, 548, 377, 0, 566, 385, 341, 456, - 379, 531, 408, 0, 559, 625, 560, 473, 474, 683, - 688, 684, 685, 687, 707, 448, 399, 404, 488, 410, - 424, 476, 530, 454, 481, 339, 520, 490, 429, 610, - 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 670, 669, 668, + 667, 666, 665, 664, 663, 0, 0, 612, 512, 355, + 307, 351, 352, 359, 729, 725, 730, 713, 716, 715, + 691, 0, 315, 592, 427, 475, 376, 657, 658, 0, + 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, + 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, + 281, 282, 283, 284, 285, 660, 276, 277, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 298, 299, 0, 0, 0, 0, 309, 717, 718, 719, + 720, 721, 0, 0, 310, 311, 312, 0, 0, 274, + 275, 302, 503, 303, 304, 305, 306, 0, 0, 542, + 543, 544, 567, 0, 545, 527, 591, 386, 316, 507, + 534, 727, 0, 0, 0, 0, 0, 0, 0, 642, + 653, 687, 0, 699, 700, 702, 704, 703, 706, 500, + 501, 714, 0, 0, 708, 709, 710, 707, 431, 487, + 508, 494, 0, 733, 582, 583, 734, 695, 317, 458, + 0, 0, 597, 631, 620, 705, 585, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, + 426, 635, 616, 627, 617, 602, 603, 604, 611, 381, + 605, 606, 607, 577, 608, 578, 609, 610, 0, 634, + 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 245, 0, 0, 3178, 0, + 0, 0, 337, 246, 579, 701, 581, 580, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 497, 526, 0, + 539, 0, 407, 408, 0, 0, 0, 0, 0, 0, + 0, 324, 504, 523, 338, 491, 537, 343, 499, 516, + 333, 457, 488, 0, 0, 326, 521, 498, 439, 325, + 0, 482, 366, 383, 363, 455, 0, 0, 520, 550, + 362, 540, 0, 531, 328, 0, 530, 454, 517, 522, + 440, 433, 0, 327, 519, 438, 432, 413, 373, 566, + 414, 415, 416, 417, 418, 419, 387, 469, 430, 470, + 388, 444, 443, 445, 389, 390, 391, 392, 393, 394, + 395, 396, 397, 398, 0, 0, 0, 0, 0, 561, + 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 694, 0, 0, 698, + 0, 533, 0, 0, 0, 0, 0, 0, 502, 0, + 0, 420, 0, 0, 0, 551, 0, 485, 460, 736, + 0, 0, 483, 428, 518, 471, 524, 505, 532, 477, + 472, 318, 506, 365, 441, 334, 336, 726, 367, 370, + 374, 375, 450, 451, 465, 490, 509, 510, 511, 364, + 348, 484, 349, 384, 350, 319, 356, 354, 357, 492, + 358, 321, 466, 515, 0, 380, 480, 436, 322, 435, + 467, 514, 513, 335, 541, 548, 549, 639, 0, 554, + 737, 738, 739, 563, 0, 473, 331, 330, 0, 0, + 0, 360, 468, 344, 346, 347, 345, 463, 464, 568, + 569, 570, 572, 0, 573, 574, 0, 0, 0, 0, + 575, 640, 656, 624, 593, 556, 648, 590, 594, 595, + 401, 402, 403, 404, 659, 0, 0, 0, 547, 421, + 422, 0, 372, 371, 437, 323, 0, 0, 410, 400, + 474, 329, 368, 412, 406, 423, 424, 425, 378, 313, + 314, 732, 361, 456, 661, 696, 697, 586, 0, 649, + 587, 596, 353, 621, 633, 632, 452, 546, 0, 644, + 647, 576, 731, 0, 641, 655, 735, 654, 728, 462, + 0, 489, 652, 599, 0, 645, 618, 619, 0, 646, + 614, 650, 0, 588, 0, 557, 560, 589, 674, 675, + 676, 320, 559, 678, 679, 680, 681, 682, 683, 684, + 677, 529, 622, 598, 625, 538, 601, 600, 0, 0, + 636, 555, 637, 638, 446, 447, 448, 449, 382, 662, + 342, 558, 476, 0, 623, 0, 0, 0, 0, 0, + 0, 0, 0, 628, 629, 626, 740, 0, 685, 686, + 0, 0, 552, 553, 377, 0, 571, 385, 341, 461, + 379, 536, 409, 0, 564, 630, 565, 478, 479, 688, + 693, 689, 690, 692, 712, 453, 399, 405, 493, 411, + 429, 481, 535, 459, 486, 339, 525, 495, 434, 615, + 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 665, 664, - 663, 662, 661, 660, 659, 658, 0, 0, 607, 507, - 355, 307, 351, 352, 359, 724, 720, 725, 708, 711, - 710, 686, 0, 315, 587, 422, 470, 376, 652, 653, - 0, 706, 259, 260, 261, 262, 263, 264, 265, 266, + 0, 0, 0, 0, 0, 0, 0, 0, 670, 669, + 668, 667, 666, 665, 664, 663, 0, 0, 612, 512, + 355, 307, 351, 352, 359, 729, 725, 730, 713, 716, + 715, 691, 0, 315, 592, 427, 475, 376, 657, 658, + 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, - 280, 281, 282, 283, 284, 285, 655, 276, 277, 286, + 280, 281, 282, 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 298, 299, 0, 0, 0, 0, 309, 712, 713, - 714, 715, 716, 0, 0, 310, 311, 312, 0, 0, - 274, 275, 302, 498, 303, 304, 305, 306, 0, 0, - 537, 538, 539, 562, 0, 540, 522, 586, 386, 316, - 502, 529, 722, 0, 0, 0, 0, 0, 0, 0, - 637, 648, 682, 0, 694, 695, 697, 699, 698, 701, - 495, 496, 709, 0, 0, 703, 704, 705, 702, 426, - 482, 503, 489, 0, 728, 577, 578, 729, 690, 317, - 453, 0, 0, 592, 626, 615, 700, 580, 0, 0, - 3241, 0, 0, 0, 0, 0, 0, 0, 369, 0, - 0, 421, 630, 611, 622, 612, 597, 598, 599, 606, - 381, 600, 601, 602, 572, 603, 573, 604, 605, 0, - 629, 579, 491, 437, 0, 646, 0, 0, 0, 0, + 297, 298, 299, 0, 0, 0, 0, 309, 717, 718, + 719, 720, 721, 0, 0, 310, 311, 312, 0, 0, + 274, 275, 302, 503, 303, 304, 305, 306, 0, 0, + 542, 543, 544, 567, 0, 545, 527, 591, 386, 316, + 507, 534, 727, 0, 0, 0, 0, 0, 0, 0, + 642, 653, 687, 0, 699, 700, 702, 704, 703, 706, + 500, 501, 714, 0, 0, 708, 709, 710, 707, 431, + 487, 508, 494, 0, 733, 582, 583, 734, 695, 317, + 458, 0, 0, 597, 631, 620, 705, 585, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, + 0, 426, 635, 616, 627, 617, 602, 603, 604, 611, + 381, 605, 606, 607, 577, 608, 578, 609, 610, 0, + 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, - 0, 0, 0, 337, 246, 574, 696, 576, 575, 0, + 0, 0, 0, 337, 246, 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 3158, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 497, 526, + 0, 539, 0, 407, 408, 0, 0, 0, 0, 0, + 0, 0, 324, 504, 523, 338, 491, 537, 343, 499, + 516, 333, 457, 488, 0, 0, 326, 521, 498, 439, + 325, 0, 482, 366, 383, 363, 455, 0, 0, 520, + 550, 362, 540, 0, 531, 328, 0, 530, 454, 517, + 522, 440, 433, 0, 327, 519, 438, 432, 413, 373, + 566, 414, 415, 416, 417, 418, 419, 387, 469, 430, + 470, 388, 444, 443, 445, 389, 390, 391, 392, 393, + 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, + 561, 562, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 694, 0, 0, + 698, 0, 533, 0, 0, 0, 0, 0, 0, 502, + 0, 0, 420, 0, 0, 0, 551, 0, 485, 460, + 736, 0, 0, 483, 428, 518, 471, 524, 505, 532, + 477, 472, 318, 506, 365, 441, 334, 336, 726, 367, + 370, 374, 375, 450, 451, 465, 490, 509, 510, 511, + 364, 348, 484, 349, 384, 350, 319, 356, 354, 357, + 492, 358, 321, 466, 515, 0, 380, 480, 436, 322, + 435, 467, 514, 513, 335, 541, 548, 549, 639, 0, + 554, 737, 738, 739, 563, 0, 473, 331, 330, 0, + 0, 0, 360, 468, 344, 346, 347, 345, 463, 464, + 568, 569, 570, 572, 0, 573, 574, 0, 0, 0, + 0, 575, 640, 656, 624, 593, 556, 648, 590, 594, + 595, 401, 402, 403, 404, 659, 0, 0, 0, 547, + 421, 422, 0, 372, 371, 437, 323, 0, 0, 410, + 400, 474, 329, 368, 412, 406, 423, 424, 425, 378, + 313, 314, 732, 361, 456, 661, 696, 697, 586, 0, + 649, 587, 596, 353, 621, 633, 632, 452, 546, 0, + 644, 647, 576, 731, 0, 641, 655, 735, 654, 728, + 462, 0, 489, 652, 599, 0, 645, 618, 619, 0, + 646, 614, 650, 0, 588, 0, 557, 560, 589, 674, + 675, 676, 320, 559, 678, 679, 680, 681, 682, 683, + 684, 677, 529, 622, 598, 625, 538, 601, 600, 0, + 0, 636, 555, 637, 638, 446, 447, 448, 449, 382, + 662, 342, 558, 476, 0, 623, 0, 0, 0, 0, + 0, 0, 0, 0, 628, 629, 626, 740, 0, 685, + 686, 0, 0, 552, 553, 377, 0, 571, 385, 341, + 461, 379, 536, 409, 0, 564, 630, 565, 478, 479, + 688, 693, 689, 690, 692, 712, 453, 399, 405, 493, + 411, 429, 481, 535, 459, 486, 339, 525, 495, 434, + 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 670, + 669, 668, 667, 666, 665, 664, 663, 0, 0, 612, + 512, 355, 307, 351, 352, 359, 729, 725, 730, 713, + 716, 715, 691, 0, 315, 592, 427, 475, 376, 657, + 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, + 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, + 279, 280, 281, 282, 283, 284, 285, 660, 276, 277, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 298, 299, 0, 0, 0, 0, 309, 717, + 718, 719, 720, 721, 0, 0, 310, 311, 312, 0, + 0, 274, 275, 302, 503, 303, 304, 305, 306, 0, + 0, 542, 543, 544, 567, 0, 545, 527, 591, 386, + 316, 507, 534, 727, 0, 0, 0, 0, 0, 0, + 0, 642, 653, 687, 0, 699, 700, 702, 704, 703, + 706, 500, 501, 714, 0, 0, 708, 709, 710, 707, + 431, 487, 508, 494, 0, 733, 582, 583, 734, 695, + 317, 458, 0, 0, 597, 631, 620, 705, 585, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, + 0, 0, 426, 635, 616, 627, 617, 602, 603, 604, + 611, 381, 605, 606, 607, 577, 608, 578, 609, 610, + 0, 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 492, 521, - 0, 534, 0, 406, 407, 0, 0, 0, 0, 0, - 0, 0, 324, 499, 518, 338, 486, 532, 343, 494, - 511, 333, 452, 483, 0, 0, 326, 516, 493, 434, - 325, 0, 477, 366, 383, 363, 450, 0, 0, 515, - 545, 362, 535, 0, 526, 328, 0, 525, 449, 512, - 517, 435, 428, 0, 327, 514, 433, 427, 412, 373, - 561, 413, 414, 387, 464, 425, 465, 388, 439, 438, - 440, 389, 390, 391, 392, 393, 394, 395, 396, 397, - 398, 0, 0, 0, 0, 0, 556, 557, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 689, 0, 0, 693, 0, 528, 0, - 0, 0, 0, 0, 0, 497, 0, 0, 415, 0, - 0, 0, 546, 0, 480, 455, 731, 0, 0, 478, - 423, 513, 466, 519, 500, 527, 472, 467, 318, 501, - 365, 436, 334, 336, 721, 367, 370, 374, 375, 445, - 446, 460, 485, 504, 505, 506, 364, 348, 479, 349, - 384, 350, 319, 356, 354, 357, 487, 358, 321, 461, - 510, 0, 380, 475, 431, 322, 430, 462, 509, 508, - 335, 536, 543, 544, 634, 0, 549, 732, 733, 734, - 558, 0, 468, 331, 330, 0, 0, 0, 360, 463, - 344, 346, 347, 345, 458, 459, 563, 564, 565, 567, - 0, 568, 569, 0, 0, 0, 0, 570, 635, 651, - 619, 588, 551, 643, 585, 589, 590, 401, 402, 403, - 654, 0, 0, 0, 542, 416, 417, 0, 372, 371, - 432, 323, 0, 0, 409, 400, 469, 329, 368, 411, - 405, 418, 419, 420, 378, 313, 314, 727, 361, 451, - 656, 691, 692, 581, 0, 644, 582, 591, 353, 616, - 628, 627, 447, 541, 0, 639, 642, 571, 726, 0, - 636, 650, 730, 649, 723, 457, 0, 484, 647, 594, - 0, 640, 613, 614, 0, 641, 609, 645, 0, 583, - 0, 552, 555, 584, 669, 670, 671, 320, 554, 673, - 674, 675, 676, 677, 678, 679, 672, 524, 617, 593, - 620, 533, 596, 595, 0, 0, 631, 550, 632, 633, - 441, 442, 443, 444, 382, 657, 342, 553, 471, 0, - 618, 0, 0, 0, 0, 0, 0, 0, 0, 623, - 624, 621, 735, 0, 680, 681, 0, 0, 547, 548, - 377, 0, 566, 385, 341, 456, 379, 531, 408, 0, - 559, 625, 560, 473, 474, 683, 688, 684, 685, 687, - 707, 448, 399, 404, 488, 410, 424, 476, 530, 454, - 481, 339, 520, 490, 429, 610, 638, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 665, 664, 663, 662, 661, 660, - 659, 658, 0, 0, 607, 507, 355, 307, 351, 352, - 359, 724, 720, 725, 708, 711, 710, 686, 0, 315, - 587, 422, 470, 376, 652, 653, 0, 706, 259, 260, - 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, - 270, 271, 272, 273, 278, 279, 280, 281, 282, 283, - 284, 285, 655, 276, 277, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, - 0, 0, 0, 309, 712, 713, 714, 715, 716, 0, - 0, 310, 311, 312, 0, 0, 274, 275, 302, 498, - 303, 304, 305, 306, 0, 0, 537, 538, 539, 562, - 0, 540, 522, 586, 386, 316, 502, 529, 722, 0, - 0, 0, 0, 0, 0, 0, 637, 648, 682, 0, - 694, 695, 697, 699, 698, 701, 495, 496, 709, 0, - 0, 703, 704, 705, 702, 426, 482, 503, 489, 0, - 728, 577, 578, 729, 690, 317, 453, 0, 0, 592, - 626, 615, 700, 580, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 369, 0, 0, 421, 630, 611, - 622, 612, 597, 598, 599, 606, 381, 600, 601, 602, - 572, 603, 573, 604, 605, 0, 629, 579, 491, 437, - 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 245, 0, 0, 3155, 0, 0, 0, 337, - 246, 574, 696, 576, 575, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, + 3103, 0, 0, 0, 337, 246, 579, 701, 581, 580, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 492, 521, 0, 534, 0, 406, - 407, 0, 0, 0, 0, 0, 0, 0, 324, 499, - 518, 338, 486, 532, 343, 494, 511, 333, 452, 483, - 0, 0, 326, 516, 493, 434, 325, 0, 477, 366, - 383, 363, 450, 0, 0, 515, 545, 362, 535, 0, - 526, 328, 0, 525, 449, 512, 517, 435, 428, 0, - 327, 514, 433, 427, 412, 373, 561, 413, 414, 387, - 464, 425, 465, 388, 439, 438, 440, 389, 390, 391, - 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, - 0, 0, 556, 557, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, - 0, 0, 693, 0, 528, 0, 0, 0, 0, 0, - 0, 497, 0, 0, 415, 0, 0, 0, 546, 0, - 480, 455, 731, 0, 0, 478, 423, 513, 466, 519, - 500, 527, 472, 467, 318, 501, 365, 436, 334, 336, - 721, 367, 370, 374, 375, 445, 446, 460, 485, 504, - 505, 506, 364, 348, 479, 349, 384, 350, 319, 356, - 354, 357, 487, 358, 321, 461, 510, 0, 380, 475, - 431, 322, 430, 462, 509, 508, 335, 536, 543, 544, - 634, 0, 549, 732, 733, 734, 558, 0, 468, 331, - 330, 0, 0, 0, 360, 463, 344, 346, 347, 345, - 458, 459, 563, 564, 565, 567, 0, 568, 569, 0, - 0, 0, 0, 570, 635, 651, 619, 588, 551, 643, - 585, 589, 590, 401, 402, 403, 654, 0, 0, 0, - 542, 416, 417, 0, 372, 371, 432, 323, 0, 0, - 409, 400, 469, 329, 368, 411, 405, 418, 419, 420, - 378, 313, 314, 727, 361, 451, 656, 691, 692, 581, - 0, 644, 582, 591, 353, 616, 628, 627, 447, 541, - 0, 639, 642, 571, 726, 0, 636, 650, 730, 649, - 723, 457, 0, 484, 647, 594, 0, 640, 613, 614, - 0, 641, 609, 645, 0, 583, 0, 552, 555, 584, - 669, 670, 671, 320, 554, 673, 674, 675, 676, 677, - 678, 679, 672, 524, 617, 593, 620, 533, 596, 595, - 0, 0, 631, 550, 632, 633, 441, 442, 443, 444, - 382, 657, 342, 553, 471, 0, 618, 0, 0, 0, - 0, 0, 0, 0, 0, 623, 624, 621, 735, 0, - 680, 681, 0, 0, 547, 548, 377, 0, 566, 385, - 341, 456, 379, 531, 408, 0, 559, 625, 560, 473, - 474, 683, 688, 684, 685, 687, 707, 448, 399, 404, - 488, 410, 424, 476, 530, 454, 481, 339, 520, 490, - 429, 610, 638, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, + 526, 0, 539, 0, 407, 408, 0, 0, 0, 0, + 0, 0, 0, 324, 504, 523, 338, 491, 537, 343, + 499, 516, 333, 457, 488, 0, 0, 326, 521, 498, + 439, 325, 0, 482, 366, 383, 363, 455, 0, 0, + 520, 550, 362, 540, 0, 531, 328, 0, 530, 454, + 517, 522, 440, 433, 0, 327, 519, 438, 432, 413, + 373, 566, 414, 415, 416, 417, 418, 419, 387, 469, + 430, 470, 388, 444, 443, 445, 389, 390, 391, 392, + 393, 394, 395, 396, 397, 398, 0, 0, 0, 0, + 0, 561, 562, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 694, 0, + 0, 698, 0, 533, 0, 0, 0, 0, 0, 0, + 502, 0, 0, 420, 0, 0, 0, 551, 0, 485, + 460, 736, 0, 0, 483, 428, 518, 471, 524, 505, + 532, 477, 472, 318, 506, 365, 441, 334, 336, 726, + 367, 370, 374, 375, 450, 451, 465, 490, 509, 510, + 511, 364, 348, 484, 349, 384, 350, 319, 356, 354, + 357, 492, 358, 321, 466, 515, 0, 380, 480, 436, + 322, 435, 467, 514, 513, 335, 541, 548, 549, 639, + 0, 554, 737, 738, 739, 563, 0, 473, 331, 330, + 0, 0, 0, 360, 468, 344, 346, 347, 345, 463, + 464, 568, 569, 570, 572, 0, 573, 574, 0, 0, + 0, 0, 575, 640, 656, 624, 593, 556, 648, 590, + 594, 595, 401, 402, 403, 404, 659, 0, 0, 0, + 547, 421, 422, 0, 372, 371, 437, 323, 0, 0, + 410, 400, 474, 329, 368, 412, 406, 423, 424, 425, + 378, 313, 314, 732, 361, 456, 661, 696, 697, 586, + 0, 649, 587, 596, 353, 621, 633, 632, 452, 546, + 0, 644, 647, 576, 731, 0, 641, 655, 735, 654, + 728, 462, 0, 489, 652, 599, 0, 645, 618, 619, + 0, 646, 614, 650, 0, 588, 0, 557, 560, 589, + 674, 675, 676, 320, 559, 678, 679, 680, 681, 682, + 683, 684, 677, 529, 622, 598, 625, 538, 601, 600, + 0, 0, 636, 555, 637, 638, 446, 447, 448, 449, + 382, 662, 342, 558, 476, 0, 623, 0, 0, 0, + 0, 0, 0, 0, 0, 628, 629, 626, 740, 0, + 685, 686, 0, 0, 552, 553, 377, 0, 571, 385, + 341, 461, 379, 536, 409, 0, 564, 630, 565, 478, + 479, 688, 693, 689, 690, 692, 712, 453, 399, 405, + 493, 411, 429, 481, 535, 459, 486, 339, 525, 495, + 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 665, 664, 663, 662, 661, 660, 659, 658, 0, 0, - 607, 507, 355, 307, 351, 352, 359, 724, 720, 725, - 708, 711, 710, 686, 0, 315, 587, 422, 470, 376, - 652, 653, 0, 706, 259, 260, 261, 262, 263, 264, + 670, 669, 668, 667, 666, 665, 664, 663, 0, 0, + 612, 512, 355, 307, 351, 352, 359, 729, 725, 730, + 713, 716, 715, 691, 0, 315, 592, 427, 475, 376, + 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, - 278, 279, 280, 281, 282, 283, 284, 285, 655, 276, + 278, 279, 280, 281, 282, 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, 0, 309, - 712, 713, 714, 715, 716, 0, 0, 310, 311, 312, - 0, 0, 274, 275, 302, 498, 303, 304, 305, 306, - 0, 0, 537, 538, 539, 562, 0, 540, 522, 586, - 386, 316, 502, 529, 722, 0, 0, 0, 0, 0, - 0, 0, 637, 648, 682, 0, 694, 695, 697, 699, - 698, 701, 495, 496, 709, 0, 0, 703, 704, 705, - 702, 426, 482, 503, 489, 0, 728, 577, 578, 729, - 690, 317, 453, 0, 0, 592, 626, 615, 700, 580, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 369, 0, 0, 421, 630, 611, 622, 612, 597, 598, - 599, 606, 381, 600, 601, 602, 572, 603, 573, 604, - 605, 0, 629, 579, 491, 437, 0, 646, 0, 0, + 717, 718, 719, 720, 721, 0, 0, 310, 311, 312, + 0, 0, 274, 275, 302, 503, 303, 304, 305, 306, + 0, 0, 542, 543, 544, 567, 0, 545, 527, 591, + 386, 316, 507, 534, 727, 0, 0, 0, 0, 0, + 0, 0, 642, 653, 687, 0, 699, 700, 702, 704, + 703, 706, 500, 501, 714, 0, 0, 708, 709, 710, + 707, 431, 487, 508, 494, 0, 733, 582, 583, 734, + 695, 317, 458, 0, 0, 597, 631, 620, 705, 585, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 369, 0, 0, 426, 635, 616, 627, 617, 602, 603, + 604, 611, 381, 605, 606, 607, 577, 608, 578, 609, + 610, 0, 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 0, 0, 0, 0, 337, 246, 574, 696, 576, - 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 337, 246, 579, 701, 581, + 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 3136, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 492, 521, 0, 534, 0, 406, 407, 0, 0, 0, - 0, 0, 0, 0, 324, 499, 518, 338, 486, 532, - 343, 494, 511, 333, 452, 483, 0, 0, 326, 516, - 493, 434, 325, 0, 477, 366, 383, 363, 450, 0, - 0, 515, 545, 362, 535, 0, 526, 328, 0, 525, - 449, 512, 517, 435, 428, 0, 327, 514, 433, 427, - 412, 373, 561, 413, 414, 387, 464, 425, 465, 388, - 439, 438, 440, 389, 390, 391, 392, 393, 394, 395, - 396, 397, 398, 0, 0, 0, 0, 0, 556, 557, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 689, 0, 0, 693, 0, - 528, 0, 0, 0, 0, 0, 0, 497, 0, 0, - 415, 0, 0, 0, 546, 0, 480, 455, 731, 0, - 0, 478, 423, 513, 466, 519, 500, 527, 472, 467, - 318, 501, 365, 436, 334, 336, 721, 367, 370, 374, - 375, 445, 446, 460, 485, 504, 505, 506, 364, 348, - 479, 349, 384, 350, 319, 356, 354, 357, 487, 358, - 321, 461, 510, 0, 380, 475, 431, 322, 430, 462, - 509, 508, 335, 536, 543, 544, 634, 0, 549, 732, - 733, 734, 558, 0, 468, 331, 330, 0, 0, 0, - 360, 463, 344, 346, 347, 345, 458, 459, 563, 564, - 565, 567, 0, 568, 569, 0, 0, 0, 0, 570, - 635, 651, 619, 588, 551, 643, 585, 589, 590, 401, - 402, 403, 654, 0, 0, 0, 542, 416, 417, 0, - 372, 371, 432, 323, 0, 0, 409, 400, 469, 329, - 368, 411, 405, 418, 419, 420, 378, 313, 314, 727, - 361, 451, 656, 691, 692, 581, 0, 644, 582, 591, - 353, 616, 628, 627, 447, 541, 0, 639, 642, 571, - 726, 0, 636, 650, 730, 649, 723, 457, 0, 484, - 647, 594, 0, 640, 613, 614, 0, 641, 609, 645, - 0, 583, 0, 552, 555, 584, 669, 670, 671, 320, - 554, 673, 674, 675, 676, 677, 678, 679, 672, 524, - 617, 593, 620, 533, 596, 595, 0, 0, 631, 550, - 632, 633, 441, 442, 443, 444, 382, 657, 342, 553, - 471, 0, 618, 0, 0, 0, 0, 0, 0, 0, - 0, 623, 624, 621, 735, 0, 680, 681, 0, 0, - 547, 548, 377, 0, 566, 385, 341, 456, 379, 531, - 408, 0, 559, 625, 560, 473, 474, 683, 688, 684, - 685, 687, 707, 448, 399, 404, 488, 410, 424, 476, - 530, 454, 481, 339, 520, 490, 429, 610, 638, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 665, 664, 663, 662, - 661, 660, 659, 658, 0, 0, 607, 507, 355, 307, - 351, 352, 359, 724, 720, 725, 708, 711, 710, 686, - 0, 315, 587, 422, 470, 376, 652, 653, 0, 706, - 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, - 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, - 282, 283, 284, 285, 655, 276, 277, 286, 287, 288, - 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, - 299, 0, 0, 0, 0, 309, 712, 713, 714, 715, - 716, 0, 0, 310, 311, 312, 0, 0, 274, 275, - 302, 498, 303, 304, 305, 306, 0, 0, 537, 538, - 539, 562, 0, 540, 522, 586, 386, 316, 502, 529, - 722, 0, 0, 0, 0, 0, 0, 0, 637, 648, - 682, 0, 694, 695, 697, 699, 698, 701, 495, 496, - 709, 0, 0, 703, 704, 705, 702, 426, 482, 503, - 489, 0, 728, 577, 578, 729, 690, 317, 453, 0, - 0, 592, 626, 615, 700, 580, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 369, 0, 0, 421, - 630, 611, 622, 612, 597, 598, 599, 606, 381, 600, - 601, 602, 572, 603, 573, 604, 605, 0, 629, 579, - 491, 437, 0, 646, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 245, 0, 0, 3081, 0, 0, - 0, 337, 246, 574, 696, 576, 575, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, + 497, 526, 0, 539, 0, 407, 408, 0, 0, 0, + 0, 0, 0, 0, 324, 504, 523, 338, 491, 537, + 343, 499, 516, 333, 457, 488, 0, 0, 326, 521, + 498, 439, 325, 0, 482, 366, 383, 363, 455, 0, + 0, 520, 550, 362, 540, 0, 531, 328, 0, 530, + 454, 517, 522, 440, 433, 0, 327, 519, 438, 432, + 413, 373, 566, 414, 415, 416, 417, 418, 419, 387, + 469, 430, 470, 388, 444, 443, 445, 389, 390, 391, + 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, + 0, 0, 561, 562, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, + 0, 0, 698, 0, 533, 0, 0, 0, 0, 0, + 0, 502, 0, 0, 420, 0, 0, 0, 551, 0, + 485, 460, 736, 0, 0, 483, 428, 518, 471, 524, + 505, 532, 477, 472, 318, 506, 365, 441, 334, 336, + 726, 367, 370, 374, 375, 450, 451, 465, 490, 509, + 510, 511, 364, 348, 484, 349, 384, 350, 319, 356, + 354, 357, 492, 358, 321, 466, 515, 0, 380, 480, + 436, 322, 435, 467, 514, 513, 335, 541, 548, 549, + 639, 0, 554, 737, 738, 739, 563, 0, 473, 331, + 330, 0, 0, 0, 360, 468, 344, 346, 347, 345, + 463, 464, 568, 569, 570, 572, 0, 573, 574, 0, + 0, 0, 0, 575, 640, 656, 624, 593, 556, 648, + 590, 594, 595, 401, 402, 403, 404, 659, 0, 0, + 0, 547, 421, 422, 0, 372, 371, 437, 323, 0, + 0, 410, 400, 474, 329, 368, 412, 406, 423, 424, + 425, 378, 313, 314, 732, 361, 456, 661, 696, 697, + 586, 0, 649, 587, 596, 353, 621, 633, 632, 452, + 546, 0, 644, 647, 576, 731, 0, 641, 655, 735, + 654, 728, 462, 0, 489, 652, 599, 0, 645, 618, + 619, 0, 646, 614, 650, 0, 588, 0, 557, 560, + 589, 674, 675, 676, 320, 559, 678, 679, 680, 681, + 682, 683, 684, 677, 529, 622, 598, 625, 538, 601, + 600, 0, 0, 636, 555, 637, 638, 446, 447, 448, + 449, 382, 662, 342, 558, 476, 0, 623, 0, 0, + 0, 0, 0, 0, 0, 0, 628, 629, 626, 740, + 0, 685, 686, 0, 0, 552, 553, 377, 0, 571, + 385, 341, 461, 379, 536, 409, 0, 564, 630, 565, + 478, 479, 688, 693, 689, 690, 692, 712, 453, 399, + 405, 493, 411, 429, 481, 535, 459, 486, 339, 525, + 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 670, 669, 668, 667, 666, 665, 664, 663, 0, + 0, 612, 512, 355, 307, 351, 352, 359, 729, 725, + 730, 713, 716, 715, 691, 0, 315, 592, 427, 475, + 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, + 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, + 273, 278, 279, 280, 281, 282, 283, 284, 285, 660, + 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 298, 299, 0, 0, 0, 0, + 309, 717, 718, 719, 720, 721, 0, 0, 310, 311, + 312, 0, 0, 274, 275, 302, 503, 303, 304, 305, + 306, 0, 0, 542, 543, 544, 567, 0, 545, 527, + 591, 386, 316, 507, 534, 727, 0, 0, 0, 0, + 0, 0, 0, 642, 653, 687, 0, 699, 700, 702, + 704, 703, 706, 500, 501, 714, 0, 0, 708, 709, + 710, 707, 431, 487, 508, 494, 0, 733, 582, 583, + 734, 695, 317, 458, 0, 0, 597, 631, 620, 705, + 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 369, 0, 0, 426, 635, 616, 627, 617, 602, + 603, 604, 611, 381, 605, 606, 607, 577, 608, 578, + 609, 610, 0, 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, + 0, 0, 2974, 0, 0, 0, 337, 246, 579, 701, + 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 492, 521, 0, 534, - 0, 406, 407, 0, 0, 0, 0, 0, 0, 0, - 324, 499, 518, 338, 486, 532, 343, 494, 511, 333, - 452, 483, 0, 0, 326, 516, 493, 434, 325, 0, - 477, 366, 383, 363, 450, 0, 0, 515, 545, 362, - 535, 0, 526, 328, 0, 525, 449, 512, 517, 435, - 428, 0, 327, 514, 433, 427, 412, 373, 561, 413, - 414, 387, 464, 425, 465, 388, 439, 438, 440, 389, - 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, - 0, 0, 0, 0, 556, 557, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 689, 0, 0, 693, 0, 528, 0, 0, 0, - 0, 0, 0, 497, 0, 0, 415, 0, 0, 0, - 546, 0, 480, 455, 731, 0, 0, 478, 423, 513, - 466, 519, 500, 527, 472, 467, 318, 501, 365, 436, - 334, 336, 721, 367, 370, 374, 375, 445, 446, 460, - 485, 504, 505, 506, 364, 348, 479, 349, 384, 350, - 319, 356, 354, 357, 487, 358, 321, 461, 510, 0, - 380, 475, 431, 322, 430, 462, 509, 508, 335, 536, - 543, 544, 634, 0, 549, 732, 733, 734, 558, 0, - 468, 331, 330, 0, 0, 0, 360, 463, 344, 346, - 347, 345, 458, 459, 563, 564, 565, 567, 0, 568, - 569, 0, 0, 0, 0, 570, 635, 651, 619, 588, - 551, 643, 585, 589, 590, 401, 402, 403, 654, 0, - 0, 0, 542, 416, 417, 0, 372, 371, 432, 323, - 0, 0, 409, 400, 469, 329, 368, 411, 405, 418, - 419, 420, 378, 313, 314, 727, 361, 451, 656, 691, - 692, 581, 0, 644, 582, 591, 353, 616, 628, 627, - 447, 541, 0, 639, 642, 571, 726, 0, 636, 650, - 730, 649, 723, 457, 0, 484, 647, 594, 0, 640, - 613, 614, 0, 641, 609, 645, 0, 583, 0, 552, - 555, 584, 669, 670, 671, 320, 554, 673, 674, 675, - 676, 677, 678, 679, 672, 524, 617, 593, 620, 533, - 596, 595, 0, 0, 631, 550, 632, 633, 441, 442, - 443, 444, 382, 657, 342, 553, 471, 0, 618, 0, - 0, 0, 0, 0, 0, 0, 0, 623, 624, 621, - 735, 0, 680, 681, 0, 0, 547, 548, 377, 0, - 566, 385, 341, 456, 379, 531, 408, 0, 559, 625, - 560, 473, 474, 683, 688, 684, 685, 687, 707, 448, - 399, 404, 488, 410, 424, 476, 530, 454, 481, 339, - 520, 490, 429, 610, 638, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 497, 526, 0, 539, 0, 407, 408, 0, 0, + 0, 0, 0, 0, 0, 324, 504, 523, 338, 491, + 537, 343, 499, 516, 333, 457, 488, 0, 0, 326, + 521, 498, 439, 325, 0, 482, 366, 383, 363, 455, + 0, 0, 520, 550, 362, 540, 0, 531, 328, 0, + 530, 454, 517, 522, 440, 433, 0, 327, 519, 438, + 432, 413, 373, 566, 414, 415, 416, 417, 418, 419, + 387, 469, 430, 470, 388, 444, 443, 445, 389, 390, + 391, 392, 393, 394, 395, 396, 397, 398, 0, 0, + 0, 0, 0, 561, 562, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 694, 0, 0, 698, 0, 533, 0, 0, 0, 0, + 0, 0, 502, 0, 0, 420, 0, 0, 0, 551, + 0, 485, 460, 736, 0, 0, 483, 428, 518, 471, + 524, 505, 532, 477, 472, 318, 506, 365, 441, 334, + 336, 726, 367, 370, 374, 375, 450, 451, 465, 490, + 509, 510, 511, 364, 348, 484, 349, 384, 350, 319, + 356, 354, 357, 492, 358, 321, 466, 515, 0, 380, + 480, 436, 322, 435, 467, 514, 513, 335, 541, 548, + 549, 639, 0, 554, 737, 738, 739, 563, 0, 473, + 331, 330, 0, 0, 0, 360, 468, 344, 346, 347, + 345, 463, 464, 568, 569, 570, 572, 0, 573, 574, + 0, 0, 0, 0, 575, 640, 656, 624, 593, 556, + 648, 590, 594, 595, 401, 402, 403, 404, 659, 0, + 0, 0, 547, 421, 422, 0, 372, 371, 437, 323, + 0, 0, 410, 400, 474, 329, 368, 412, 406, 423, + 424, 425, 378, 313, 314, 732, 361, 456, 661, 696, + 697, 586, 0, 649, 587, 596, 353, 621, 633, 632, + 452, 546, 0, 644, 647, 576, 731, 0, 641, 655, + 735, 654, 728, 462, 0, 489, 652, 599, 0, 645, + 618, 619, 0, 646, 614, 650, 0, 588, 0, 557, + 560, 589, 674, 675, 676, 320, 559, 678, 679, 680, + 681, 682, 683, 684, 677, 529, 622, 598, 625, 538, + 601, 600, 0, 0, 636, 555, 637, 638, 446, 447, + 448, 449, 382, 662, 342, 558, 476, 0, 623, 0, + 0, 0, 0, 0, 0, 0, 0, 628, 629, 626, + 740, 0, 685, 686, 0, 0, 552, 553, 377, 0, + 571, 385, 341, 461, 379, 536, 409, 0, 564, 630, + 565, 478, 479, 688, 693, 689, 690, 692, 712, 453, + 399, 405, 493, 411, 429, 481, 535, 459, 486, 339, + 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 665, 664, 663, 662, 661, 660, 659, 658, - 0, 0, 607, 507, 355, 307, 351, 352, 359, 724, - 720, 725, 708, 711, 710, 686, 0, 315, 587, 422, - 470, 376, 652, 653, 0, 706, 259, 260, 261, 262, + 0, 0, 670, 669, 668, 667, 666, 665, 664, 663, + 0, 0, 612, 512, 355, 307, 351, 352, 359, 729, + 725, 730, 713, 716, 715, 691, 0, 315, 592, 427, + 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, 283, 284, 285, - 655, 276, 277, 286, 287, 288, 289, 290, 291, 292, + 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, - 0, 309, 712, 713, 714, 715, 716, 0, 0, 310, - 311, 312, 0, 0, 274, 275, 302, 498, 303, 304, - 305, 306, 0, 0, 537, 538, 539, 562, 0, 540, - 522, 586, 386, 316, 502, 529, 722, 0, 0, 0, - 0, 0, 0, 0, 637, 648, 682, 0, 694, 695, - 697, 699, 698, 701, 495, 496, 709, 0, 0, 703, - 704, 705, 702, 426, 482, 503, 489, 0, 728, 577, - 578, 729, 690, 317, 453, 0, 0, 592, 626, 615, - 700, 580, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 369, 0, 0, 421, 630, 611, 622, 612, - 597, 598, 599, 606, 381, 600, 601, 602, 572, 603, - 573, 604, 605, 0, 629, 579, 491, 437, 0, 646, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 0, 0, 0, 0, 337, 246, 574, - 696, 576, 575, 0, 0, 0, 0, 0, 0, 0, + 0, 309, 717, 718, 719, 720, 721, 0, 0, 310, + 311, 312, 0, 0, 274, 275, 302, 503, 303, 304, + 305, 306, 0, 0, 542, 543, 544, 567, 0, 545, + 527, 591, 386, 316, 507, 534, 727, 0, 0, 0, + 0, 0, 0, 0, 642, 653, 687, 0, 699, 700, + 702, 704, 703, 706, 500, 501, 714, 0, 0, 708, + 709, 710, 707, 431, 487, 508, 494, 0, 733, 582, + 583, 734, 695, 317, 458, 0, 0, 597, 631, 620, + 705, 585, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 369, 0, 0, 426, 635, 616, 627, 617, + 602, 603, 604, 611, 381, 605, 606, 607, 577, 608, + 578, 609, 610, 0, 634, 584, 496, 442, 0, 651, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 245, 0, 0, 0, 0, 0, 0, 337, 246, 579, + 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2428, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 492, 521, 0, 534, 0, 406, 407, 0, - 0, 0, 0, 0, 0, 0, 324, 499, 518, 338, - 486, 532, 343, 494, 511, 333, 452, 483, 0, 0, - 326, 516, 493, 434, 325, 0, 477, 366, 383, 363, - 450, 0, 0, 515, 545, 362, 535, 0, 526, 328, - 0, 525, 449, 512, 517, 435, 428, 0, 327, 514, - 433, 427, 412, 373, 561, 413, 414, 387, 464, 425, - 465, 388, 439, 438, 440, 389, 390, 391, 392, 393, - 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, - 556, 557, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 689, 0, 0, - 693, 0, 528, 0, 0, 0, 0, 0, 0, 497, - 0, 0, 415, 0, 0, 0, 546, 0, 480, 455, - 731, 0, 0, 478, 423, 513, 466, 519, 500, 527, - 472, 467, 318, 501, 365, 436, 334, 336, 721, 367, - 370, 374, 375, 445, 446, 460, 485, 504, 505, 506, - 364, 348, 479, 349, 384, 350, 319, 356, 354, 357, - 487, 358, 321, 461, 510, 0, 380, 475, 431, 322, - 430, 462, 509, 508, 335, 536, 543, 544, 634, 0, - 549, 732, 733, 734, 558, 0, 468, 331, 330, 0, - 0, 0, 360, 463, 344, 346, 347, 345, 458, 459, - 563, 564, 565, 567, 0, 568, 569, 0, 0, 0, - 0, 570, 635, 651, 619, 588, 551, 643, 585, 589, - 590, 401, 402, 403, 654, 0, 0, 0, 542, 416, - 417, 0, 372, 371, 432, 323, 0, 0, 409, 400, - 469, 329, 368, 411, 405, 418, 419, 420, 378, 313, - 314, 727, 361, 451, 656, 691, 692, 581, 0, 644, - 582, 591, 353, 616, 628, 627, 447, 541, 0, 639, - 642, 571, 726, 0, 636, 650, 730, 649, 723, 457, - 0, 484, 647, 594, 0, 640, 613, 614, 0, 641, - 609, 645, 0, 583, 0, 552, 555, 584, 669, 670, - 671, 320, 554, 673, 674, 675, 676, 677, 678, 679, - 672, 524, 617, 593, 620, 533, 596, 595, 0, 0, - 631, 550, 632, 633, 441, 442, 443, 444, 382, 657, - 342, 553, 471, 0, 618, 0, 0, 0, 0, 0, - 0, 0, 0, 623, 624, 621, 735, 0, 680, 681, - 0, 0, 547, 548, 377, 0, 566, 385, 341, 456, - 379, 531, 408, 0, 559, 625, 560, 473, 474, 683, - 688, 684, 685, 687, 707, 448, 399, 404, 488, 410, - 424, 476, 530, 454, 481, 339, 520, 490, 429, 610, - 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 665, 664, - 663, 662, 661, 660, 659, 658, 0, 0, 607, 507, - 355, 307, 351, 352, 359, 724, 720, 725, 708, 711, - 710, 686, 0, 315, 587, 422, 470, 376, 652, 653, - 0, 706, 259, 260, 261, 262, 263, 264, 265, 266, - 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, - 280, 281, 282, 283, 284, 285, 655, 276, 277, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 298, 299, 0, 0, 0, 0, 309, 712, 713, - 714, 715, 716, 0, 0, 310, 311, 312, 0, 0, - 274, 275, 302, 498, 303, 304, 305, 306, 0, 0, - 537, 538, 539, 562, 0, 540, 522, 586, 386, 316, - 502, 529, 722, 0, 0, 0, 0, 0, 0, 0, - 637, 648, 682, 0, 694, 695, 697, 699, 698, 701, - 495, 496, 709, 0, 0, 703, 704, 705, 702, 426, - 482, 503, 489, 0, 728, 577, 578, 729, 690, 317, - 453, 0, 0, 592, 626, 615, 700, 580, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, - 0, 421, 630, 611, 622, 612, 597, 598, 599, 606, - 381, 600, 601, 602, 572, 603, 573, 604, 605, 0, - 629, 579, 491, 437, 0, 646, 0, 0, 0, 0, + 2931, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 497, 526, 0, 539, 0, 407, 408, 0, + 0, 0, 0, 0, 0, 0, 324, 504, 523, 338, + 491, 537, 343, 499, 516, 333, 457, 488, 0, 0, + 326, 521, 498, 439, 325, 0, 482, 366, 383, 363, + 455, 0, 0, 520, 550, 362, 540, 0, 531, 328, + 0, 530, 454, 517, 522, 440, 433, 0, 327, 519, + 438, 432, 413, 373, 566, 414, 415, 416, 417, 418, + 419, 387, 469, 430, 470, 388, 444, 443, 445, 389, + 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, + 0, 0, 0, 0, 561, 562, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 694, 0, 0, 698, 0, 533, 0, 0, 0, + 0, 0, 0, 502, 0, 0, 420, 0, 0, 0, + 551, 0, 485, 460, 736, 0, 0, 483, 428, 518, + 471, 524, 505, 532, 477, 472, 318, 506, 365, 441, + 334, 336, 726, 367, 370, 374, 375, 450, 451, 465, + 490, 509, 510, 511, 364, 348, 484, 349, 384, 350, + 319, 356, 354, 357, 492, 358, 321, 466, 515, 0, + 380, 480, 436, 322, 435, 467, 514, 513, 335, 541, + 548, 549, 639, 0, 554, 737, 738, 739, 563, 0, + 473, 331, 330, 0, 0, 0, 360, 468, 344, 346, + 347, 345, 463, 464, 568, 569, 570, 572, 0, 573, + 574, 0, 0, 0, 0, 575, 640, 656, 624, 593, + 556, 648, 590, 594, 595, 401, 402, 403, 404, 659, + 0, 0, 0, 547, 421, 422, 0, 372, 371, 437, + 323, 0, 0, 410, 400, 474, 329, 368, 412, 406, + 423, 424, 425, 378, 313, 314, 732, 361, 456, 661, + 696, 697, 586, 0, 649, 587, 596, 353, 621, 633, + 632, 452, 546, 0, 644, 647, 576, 731, 0, 641, + 655, 735, 654, 728, 462, 0, 489, 652, 599, 0, + 645, 618, 619, 0, 646, 614, 650, 0, 588, 0, + 557, 560, 589, 674, 675, 676, 320, 559, 678, 679, + 680, 681, 682, 683, 684, 677, 529, 622, 598, 625, + 538, 601, 600, 0, 0, 636, 555, 637, 638, 446, + 447, 448, 449, 382, 662, 342, 558, 476, 0, 623, + 0, 0, 0, 0, 0, 0, 0, 0, 628, 629, + 626, 740, 0, 685, 686, 0, 0, 552, 553, 377, + 0, 571, 385, 341, 461, 379, 536, 409, 0, 564, + 630, 565, 478, 479, 688, 693, 689, 690, 692, 712, + 453, 399, 405, 493, 411, 429, 481, 535, 459, 486, + 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 245, 0, 0, 2953, - 0, 0, 0, 337, 246, 574, 696, 576, 575, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, + 0, 0, 0, 670, 669, 668, 667, 666, 665, 664, + 663, 0, 0, 612, 512, 355, 307, 351, 352, 359, + 729, 725, 730, 713, 716, 715, 691, 0, 315, 592, + 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, + 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, + 271, 272, 273, 278, 279, 280, 281, 282, 283, 284, + 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 298, 299, 0, 0, + 0, 0, 309, 717, 718, 719, 720, 721, 0, 0, + 310, 311, 312, 0, 0, 274, 275, 302, 503, 303, + 304, 305, 306, 0, 0, 542, 543, 544, 567, 0, + 545, 527, 591, 386, 316, 507, 534, 727, 0, 0, + 0, 0, 0, 0, 0, 642, 653, 687, 0, 699, + 700, 702, 704, 703, 706, 500, 501, 714, 0, 0, + 708, 709, 710, 707, 431, 487, 508, 494, 0, 733, + 582, 583, 734, 695, 317, 458, 0, 0, 597, 631, + 620, 705, 585, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 369, 0, 0, 426, 635, 616, 627, + 617, 602, 603, 604, 611, 381, 605, 606, 607, 577, + 608, 578, 609, 610, 0, 634, 584, 496, 442, 0, + 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 245, 0, 0, 2929, 0, 0, 0, 337, 246, + 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 492, 521, - 0, 534, 0, 406, 407, 0, 0, 0, 0, 0, - 0, 0, 324, 499, 518, 338, 486, 532, 343, 494, - 511, 333, 452, 483, 0, 0, 326, 516, 493, 434, - 325, 0, 477, 366, 383, 363, 450, 0, 0, 515, - 545, 362, 535, 0, 526, 328, 0, 525, 449, 512, - 517, 435, 428, 0, 327, 514, 433, 427, 412, 373, - 561, 413, 414, 387, 464, 425, 465, 388, 439, 438, - 440, 389, 390, 391, 392, 393, 394, 395, 396, 397, - 398, 0, 0, 0, 0, 0, 556, 557, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 689, 0, 0, 693, 0, 528, 0, - 0, 0, 0, 0, 0, 497, 0, 0, 415, 0, - 0, 0, 546, 0, 480, 455, 731, 0, 0, 478, - 423, 513, 466, 519, 500, 527, 472, 467, 318, 501, - 365, 436, 334, 336, 721, 367, 370, 374, 375, 445, - 446, 460, 485, 504, 505, 506, 364, 348, 479, 349, - 384, 350, 319, 356, 354, 357, 487, 358, 321, 461, - 510, 0, 380, 475, 431, 322, 430, 462, 509, 508, - 335, 536, 543, 544, 634, 0, 549, 732, 733, 734, - 558, 0, 468, 331, 330, 0, 0, 0, 360, 463, - 344, 346, 347, 345, 458, 459, 563, 564, 565, 567, - 0, 568, 569, 0, 0, 0, 0, 570, 635, 651, - 619, 588, 551, 643, 585, 589, 590, 401, 402, 403, - 654, 0, 0, 0, 542, 416, 417, 0, 372, 371, - 432, 323, 0, 0, 409, 400, 469, 329, 368, 411, - 405, 418, 419, 420, 378, 313, 314, 727, 361, 451, - 656, 691, 692, 581, 0, 644, 582, 591, 353, 616, - 628, 627, 447, 541, 0, 639, 642, 571, 726, 0, - 636, 650, 730, 649, 723, 457, 0, 484, 647, 594, - 0, 640, 613, 614, 0, 641, 609, 645, 0, 583, - 0, 552, 555, 584, 669, 670, 671, 320, 554, 673, - 674, 675, 676, 677, 678, 679, 672, 524, 617, 593, - 620, 533, 596, 595, 0, 0, 631, 550, 632, 633, - 441, 442, 443, 444, 382, 657, 342, 553, 471, 0, - 618, 0, 0, 0, 0, 0, 0, 0, 0, 623, - 624, 621, 735, 0, 680, 681, 0, 0, 547, 548, - 377, 0, 566, 385, 341, 456, 379, 531, 408, 0, - 559, 625, 560, 473, 474, 683, 688, 684, 685, 687, - 707, 448, 399, 404, 488, 410, 424, 476, 530, 454, - 481, 339, 520, 490, 429, 610, 638, 0, 0, 0, + 0, 0, 0, 497, 526, 0, 539, 0, 407, 408, + 0, 0, 0, 0, 0, 0, 0, 324, 504, 523, + 338, 491, 537, 343, 499, 516, 333, 457, 488, 0, + 0, 326, 521, 498, 439, 325, 0, 482, 366, 383, + 363, 455, 0, 0, 520, 550, 362, 540, 0, 531, + 328, 0, 530, 454, 517, 522, 440, 433, 0, 327, + 519, 438, 432, 413, 373, 566, 414, 415, 416, 417, + 418, 419, 387, 469, 430, 470, 388, 444, 443, 445, + 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, + 0, 0, 0, 0, 0, 561, 562, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 694, 0, 0, 698, 0, 533, 0, 0, + 0, 0, 0, 0, 502, 0, 0, 420, 0, 0, + 0, 551, 0, 485, 460, 736, 0, 0, 483, 428, + 518, 471, 524, 505, 532, 477, 472, 318, 506, 365, + 441, 334, 336, 726, 367, 370, 374, 375, 450, 451, + 465, 490, 509, 510, 511, 364, 348, 484, 349, 384, + 350, 319, 356, 354, 357, 492, 358, 321, 466, 515, + 0, 380, 480, 436, 322, 435, 467, 514, 513, 335, + 541, 548, 549, 639, 0, 554, 737, 738, 739, 563, + 0, 473, 331, 330, 0, 0, 0, 360, 468, 344, + 346, 347, 345, 463, 464, 568, 569, 570, 572, 0, + 573, 574, 0, 0, 0, 0, 575, 640, 656, 624, + 593, 556, 648, 590, 594, 595, 401, 402, 403, 404, + 659, 0, 0, 0, 547, 421, 422, 0, 372, 371, + 437, 323, 0, 0, 410, 400, 474, 329, 368, 412, + 406, 423, 424, 425, 378, 313, 314, 732, 361, 456, + 661, 696, 697, 586, 0, 649, 587, 596, 353, 621, + 633, 632, 452, 546, 0, 644, 647, 576, 731, 0, + 641, 655, 735, 654, 728, 462, 0, 489, 652, 599, + 0, 645, 618, 619, 0, 646, 614, 650, 0, 588, + 0, 557, 560, 589, 674, 675, 676, 320, 559, 678, + 679, 680, 681, 682, 683, 684, 677, 529, 622, 598, + 625, 538, 601, 600, 0, 0, 636, 555, 637, 638, + 446, 447, 448, 449, 382, 662, 342, 558, 476, 0, + 623, 0, 0, 0, 0, 0, 0, 0, 0, 628, + 629, 626, 740, 0, 685, 686, 0, 0, 552, 553, + 377, 0, 571, 385, 341, 461, 379, 536, 409, 0, + 564, 630, 565, 478, 479, 688, 693, 689, 690, 692, + 712, 453, 399, 405, 493, 411, 429, 481, 535, 459, + 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 665, 664, 663, 662, 661, 660, - 659, 658, 0, 0, 607, 507, 355, 307, 351, 352, - 359, 724, 720, 725, 708, 711, 710, 686, 0, 315, - 587, 422, 470, 376, 652, 653, 0, 706, 259, 260, + 0, 0, 0, 0, 670, 669, 668, 667, 666, 665, + 664, 663, 0, 0, 612, 512, 355, 307, 351, 352, + 359, 729, 725, 730, 713, 716, 715, 691, 0, 315, + 592, 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, 283, - 284, 285, 655, 276, 277, 286, 287, 288, 289, 290, + 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, - 0, 0, 0, 309, 712, 713, 714, 715, 716, 0, - 0, 310, 311, 312, 0, 0, 274, 275, 302, 498, - 303, 304, 305, 306, 0, 0, 537, 538, 539, 562, - 0, 540, 522, 586, 386, 316, 502, 529, 722, 0, - 0, 0, 0, 0, 0, 0, 637, 648, 682, 0, - 694, 695, 697, 699, 698, 701, 495, 496, 709, 0, - 0, 703, 704, 705, 702, 426, 482, 503, 489, 0, - 728, 577, 578, 729, 690, 317, 453, 0, 0, 592, - 626, 615, 700, 580, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 369, 0, 0, 421, 630, 611, - 622, 612, 597, 598, 599, 606, 381, 600, 601, 602, - 572, 603, 573, 604, 605, 0, 629, 579, 491, 437, - 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 309, 717, 718, 719, 720, 721, 0, + 0, 310, 311, 312, 0, 0, 274, 275, 302, 503, + 303, 304, 305, 306, 0, 0, 542, 543, 544, 567, + 0, 545, 527, 591, 386, 316, 507, 534, 727, 0, + 0, 0, 0, 0, 0, 0, 642, 653, 687, 0, + 699, 700, 702, 704, 703, 706, 500, 501, 714, 0, + 0, 708, 709, 710, 707, 431, 487, 508, 494, 0, + 733, 582, 583, 734, 695, 317, 2662, 0, 0, 0, + 0, 0, 458, 0, 0, 597, 631, 620, 705, 585, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 369, 0, 0, 426, 635, 616, 627, 617, 602, 603, + 604, 611, 381, 605, 606, 607, 577, 608, 578, 609, + 610, 0, 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 245, 0, 0, 0, 0, 0, 0, 337, - 246, 574, 696, 576, 575, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, + 0, 0, 0, 0, 0, 337, 246, 579, 701, 581, + 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 2910, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 492, 521, 0, 534, 0, 406, - 407, 0, 0, 0, 0, 0, 0, 0, 324, 499, - 518, 338, 486, 532, 343, 494, 511, 333, 452, 483, - 0, 0, 326, 516, 493, 434, 325, 0, 477, 366, - 383, 363, 450, 0, 0, 515, 545, 362, 535, 0, - 526, 328, 0, 525, 449, 512, 517, 435, 428, 0, - 327, 514, 433, 427, 412, 373, 561, 413, 414, 387, - 464, 425, 465, 388, 439, 438, 440, 389, 390, 391, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 497, 526, 0, 539, 0, 407, 408, 0, 0, 0, + 0, 0, 0, 0, 324, 504, 523, 338, 491, 537, + 343, 499, 516, 333, 457, 488, 0, 0, 326, 521, + 498, 439, 325, 0, 482, 366, 383, 363, 455, 0, + 0, 520, 550, 362, 540, 0, 531, 328, 0, 530, + 454, 517, 522, 440, 433, 0, 327, 519, 438, 432, + 413, 373, 566, 414, 415, 416, 417, 418, 419, 387, + 469, 430, 470, 388, 444, 443, 445, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, - 0, 0, 556, 557, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, - 0, 0, 693, 0, 528, 0, 0, 0, 0, 0, - 0, 497, 0, 0, 415, 0, 0, 0, 546, 0, - 480, 455, 731, 0, 0, 478, 423, 513, 466, 519, - 500, 527, 472, 467, 318, 501, 365, 436, 334, 336, - 721, 367, 370, 374, 375, 445, 446, 460, 485, 504, - 505, 506, 364, 348, 479, 349, 384, 350, 319, 356, - 354, 357, 487, 358, 321, 461, 510, 0, 380, 475, - 431, 322, 430, 462, 509, 508, 335, 536, 543, 544, - 634, 0, 549, 732, 733, 734, 558, 0, 468, 331, - 330, 0, 0, 0, 360, 463, 344, 346, 347, 345, - 458, 459, 563, 564, 565, 567, 0, 568, 569, 0, - 0, 0, 0, 570, 635, 651, 619, 588, 551, 643, - 585, 589, 590, 401, 402, 403, 654, 0, 0, 0, - 542, 416, 417, 0, 372, 371, 432, 323, 0, 0, - 409, 400, 469, 329, 368, 411, 405, 418, 419, 420, - 378, 313, 314, 727, 361, 451, 656, 691, 692, 581, - 0, 644, 582, 591, 353, 616, 628, 627, 447, 541, - 0, 639, 642, 571, 726, 0, 636, 650, 730, 649, - 723, 457, 0, 484, 647, 594, 0, 640, 613, 614, - 0, 641, 609, 645, 0, 583, 0, 552, 555, 584, - 669, 670, 671, 320, 554, 673, 674, 675, 676, 677, - 678, 679, 672, 524, 617, 593, 620, 533, 596, 595, - 0, 0, 631, 550, 632, 633, 441, 442, 443, 444, - 382, 657, 342, 553, 471, 0, 618, 0, 0, 0, - 0, 0, 0, 0, 0, 623, 624, 621, 735, 0, - 680, 681, 0, 0, 547, 548, 377, 0, 566, 385, - 341, 456, 379, 531, 408, 0, 559, 625, 560, 473, - 474, 683, 688, 684, 685, 687, 707, 448, 399, 404, - 488, 410, 424, 476, 530, 454, 481, 339, 520, 490, - 429, 610, 638, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, + 0, 0, 561, 562, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, + 0, 0, 698, 0, 533, 0, 0, 0, 0, 0, + 0, 502, 0, 0, 420, 0, 0, 0, 551, 0, + 485, 460, 736, 0, 0, 483, 428, 518, 471, 524, + 505, 532, 477, 472, 318, 506, 365, 441, 334, 336, + 726, 367, 370, 374, 375, 450, 451, 465, 490, 509, + 510, 511, 364, 348, 484, 349, 384, 350, 319, 356, + 354, 357, 492, 358, 321, 466, 515, 0, 380, 480, + 436, 322, 435, 467, 514, 513, 335, 541, 548, 549, + 639, 0, 554, 737, 738, 739, 563, 0, 473, 331, + 330, 0, 0, 0, 360, 468, 344, 346, 347, 345, + 463, 464, 568, 569, 570, 572, 0, 573, 574, 0, + 0, 0, 0, 575, 640, 656, 624, 593, 556, 648, + 590, 594, 595, 401, 402, 403, 404, 659, 0, 0, + 0, 547, 421, 422, 0, 372, 371, 437, 323, 0, + 0, 410, 400, 474, 329, 368, 412, 406, 423, 424, + 425, 378, 313, 314, 732, 361, 456, 661, 696, 697, + 586, 0, 649, 587, 596, 353, 621, 633, 632, 452, + 546, 0, 644, 647, 576, 731, 0, 641, 655, 735, + 654, 728, 462, 0, 489, 652, 599, 0, 645, 618, + 619, 0, 646, 614, 650, 0, 588, 0, 557, 560, + 589, 674, 675, 676, 320, 559, 678, 679, 680, 681, + 682, 683, 684, 677, 529, 622, 598, 625, 538, 601, + 600, 0, 0, 636, 555, 637, 638, 446, 447, 448, + 449, 382, 662, 342, 558, 476, 0, 623, 0, 0, + 0, 0, 0, 0, 0, 0, 628, 629, 626, 740, + 0, 685, 686, 0, 0, 552, 553, 377, 0, 571, + 385, 341, 461, 379, 536, 409, 0, 564, 630, 565, + 478, 479, 688, 693, 689, 690, 692, 712, 453, 399, + 405, 493, 411, 429, 481, 535, 459, 486, 339, 525, + 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 665, 664, 663, 662, 661, 660, 659, 658, 0, 0, - 607, 507, 355, 307, 351, 352, 359, 724, 720, 725, - 708, 711, 710, 686, 0, 315, 587, 422, 470, 376, - 652, 653, 0, 706, 259, 260, 261, 262, 263, 264, - 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, - 278, 279, 280, 281, 282, 283, 284, 285, 655, 276, - 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 298, 299, 0, 0, 0, 0, 309, - 712, 713, 714, 715, 716, 0, 0, 310, 311, 312, - 0, 0, 274, 275, 302, 498, 303, 304, 305, 306, - 0, 0, 537, 538, 539, 562, 0, 540, 522, 586, - 386, 316, 502, 529, 722, 0, 0, 0, 0, 0, - 0, 0, 637, 648, 682, 0, 694, 695, 697, 699, - 698, 701, 495, 496, 709, 0, 0, 703, 704, 705, - 702, 426, 482, 503, 489, 0, 728, 577, 578, 729, - 690, 317, 453, 0, 0, 592, 626, 615, 700, 580, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 369, 0, 0, 421, 630, 611, 622, 612, 597, 598, - 599, 606, 381, 600, 601, 602, 572, 603, 573, 604, - 605, 0, 629, 579, 491, 437, 0, 646, 0, 0, + 0, 670, 669, 668, 667, 666, 665, 664, 663, 0, + 0, 612, 512, 355, 307, 351, 352, 359, 729, 725, + 730, 713, 716, 715, 691, 0, 315, 592, 427, 475, + 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, + 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, + 273, 278, 279, 280, 281, 282, 283, 284, 285, 660, + 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 298, 299, 0, 0, 0, 0, + 309, 717, 718, 719, 720, 721, 0, 0, 310, 311, + 312, 0, 0, 274, 275, 302, 503, 303, 304, 305, + 306, 0, 0, 542, 543, 544, 567, 0, 545, 527, + 591, 386, 316, 507, 534, 727, 0, 0, 0, 0, + 0, 0, 0, 642, 653, 687, 0, 699, 700, 702, + 704, 703, 706, 500, 501, 714, 0, 0, 708, 709, + 710, 707, 431, 487, 508, 494, 0, 733, 582, 583, + 734, 695, 317, 458, 0, 0, 597, 631, 620, 705, + 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 369, 0, 0, 426, 635, 616, 627, 617, 602, + 603, 604, 611, 381, 605, 606, 607, 577, 608, 578, + 609, 610, 0, 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 2908, 0, 0, 0, 337, 246, 574, 696, 576, - 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, + 0, 0, 0, 2139, 0, 0, 337, 246, 579, 701, + 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 492, 521, 0, 534, 0, 406, 407, 0, 0, 0, - 0, 0, 0, 0, 324, 499, 518, 338, 486, 532, - 343, 494, 511, 333, 452, 483, 0, 0, 326, 516, - 493, 434, 325, 0, 477, 366, 383, 363, 450, 0, - 0, 515, 545, 362, 535, 0, 526, 328, 0, 525, - 449, 512, 517, 435, 428, 0, 327, 514, 433, 427, - 412, 373, 561, 413, 414, 387, 464, 425, 465, 388, - 439, 438, 440, 389, 390, 391, 392, 393, 394, 395, - 396, 397, 398, 0, 0, 0, 0, 0, 556, 557, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 689, 0, 0, 693, 0, - 528, 0, 0, 0, 0, 0, 0, 497, 0, 0, - 415, 0, 0, 0, 546, 0, 480, 455, 731, 0, - 0, 478, 423, 513, 466, 519, 500, 527, 472, 467, - 318, 501, 365, 436, 334, 336, 721, 367, 370, 374, - 375, 445, 446, 460, 485, 504, 505, 506, 364, 348, - 479, 349, 384, 350, 319, 356, 354, 357, 487, 358, - 321, 461, 510, 0, 380, 475, 431, 322, 430, 462, - 509, 508, 335, 536, 543, 544, 634, 0, 549, 732, - 733, 734, 558, 0, 468, 331, 330, 0, 0, 0, - 360, 463, 344, 346, 347, 345, 458, 459, 563, 564, - 565, 567, 0, 568, 569, 0, 0, 0, 0, 570, - 635, 651, 619, 588, 551, 643, 585, 589, 590, 401, - 402, 403, 654, 0, 0, 0, 542, 416, 417, 0, - 372, 371, 432, 323, 0, 0, 409, 400, 469, 329, - 368, 411, 405, 418, 419, 420, 378, 313, 314, 727, - 361, 451, 656, 691, 692, 581, 0, 644, 582, 591, - 353, 616, 628, 627, 447, 541, 0, 639, 642, 571, - 726, 0, 636, 650, 730, 649, 723, 457, 0, 484, - 647, 594, 0, 640, 613, 614, 0, 641, 609, 645, - 0, 583, 0, 552, 555, 584, 669, 670, 671, 320, - 554, 673, 674, 675, 676, 677, 678, 679, 672, 524, - 617, 593, 620, 533, 596, 595, 0, 0, 631, 550, - 632, 633, 441, 442, 443, 444, 382, 657, 342, 553, - 471, 0, 618, 0, 0, 0, 0, 0, 0, 0, - 0, 623, 624, 621, 735, 0, 680, 681, 0, 0, - 547, 548, 377, 0, 566, 385, 341, 456, 379, 531, - 408, 0, 559, 625, 560, 473, 474, 683, 688, 684, - 685, 687, 707, 448, 399, 404, 488, 410, 424, 476, - 530, 454, 481, 339, 520, 490, 429, 610, 638, 0, + 0, 497, 526, 0, 539, 0, 407, 408, 0, 0, + 0, 0, 0, 0, 0, 324, 504, 523, 338, 491, + 537, 343, 499, 516, 333, 457, 488, 0, 0, 326, + 521, 498, 439, 325, 0, 482, 366, 383, 363, 455, + 0, 0, 520, 550, 362, 540, 0, 531, 328, 0, + 530, 454, 517, 522, 440, 433, 0, 327, 519, 438, + 432, 413, 373, 566, 414, 415, 416, 417, 418, 419, + 387, 469, 430, 470, 388, 444, 443, 445, 389, 390, + 391, 392, 393, 394, 395, 396, 397, 398, 0, 0, + 0, 0, 0, 561, 562, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 694, 0, 0, 698, 0, 533, 0, 0, 0, 0, + 0, 0, 502, 0, 0, 420, 0, 0, 0, 551, + 0, 485, 460, 736, 0, 0, 483, 428, 518, 471, + 524, 505, 532, 477, 472, 318, 506, 365, 441, 334, + 336, 726, 367, 370, 374, 375, 450, 451, 465, 490, + 509, 510, 511, 364, 348, 484, 349, 384, 350, 319, + 356, 354, 357, 492, 358, 321, 466, 515, 0, 380, + 480, 436, 322, 435, 467, 514, 513, 335, 541, 548, + 549, 639, 0, 554, 737, 738, 739, 563, 0, 473, + 331, 330, 0, 0, 0, 360, 468, 344, 346, 347, + 345, 463, 464, 568, 569, 570, 572, 0, 573, 574, + 0, 0, 0, 0, 575, 640, 656, 624, 593, 556, + 648, 590, 594, 595, 401, 402, 403, 404, 659, 0, + 0, 0, 547, 421, 422, 0, 372, 371, 437, 323, + 0, 0, 410, 400, 474, 329, 368, 412, 406, 423, + 424, 425, 378, 313, 314, 732, 361, 456, 661, 696, + 697, 586, 0, 649, 587, 596, 353, 621, 633, 632, + 452, 546, 0, 644, 647, 576, 731, 0, 641, 655, + 735, 654, 728, 462, 0, 489, 652, 599, 0, 645, + 618, 619, 0, 646, 614, 650, 0, 588, 0, 557, + 560, 589, 674, 675, 676, 320, 559, 678, 679, 680, + 681, 682, 683, 684, 677, 529, 622, 598, 625, 538, + 601, 600, 0, 0, 636, 555, 637, 638, 446, 447, + 448, 449, 382, 662, 342, 558, 476, 0, 623, 0, + 0, 0, 0, 0, 0, 0, 0, 628, 629, 626, + 740, 0, 685, 686, 0, 0, 552, 553, 377, 0, + 571, 385, 341, 461, 379, 536, 409, 0, 564, 630, + 565, 478, 479, 688, 693, 689, 690, 692, 712, 453, + 399, 405, 493, 411, 429, 481, 535, 459, 486, 339, + 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 665, 664, 663, 662, - 661, 660, 659, 658, 0, 0, 607, 507, 355, 307, - 351, 352, 359, 724, 720, 725, 708, 711, 710, 686, - 0, 315, 587, 422, 470, 376, 652, 653, 0, 706, - 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, - 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, - 282, 283, 284, 285, 655, 276, 277, 286, 287, 288, - 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, - 299, 0, 0, 0, 0, 309, 712, 713, 714, 715, - 716, 0, 0, 310, 311, 312, 0, 0, 274, 275, - 302, 498, 303, 304, 305, 306, 0, 0, 537, 538, - 539, 562, 0, 540, 522, 586, 386, 316, 502, 529, - 722, 0, 0, 0, 0, 0, 0, 0, 637, 648, - 682, 0, 694, 695, 697, 699, 698, 701, 495, 496, - 709, 0, 0, 703, 704, 705, 702, 426, 482, 503, - 489, 0, 728, 577, 578, 729, 690, 317, 2645, 0, - 0, 0, 0, 0, 453, 0, 0, 592, 626, 615, - 700, 580, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 369, 0, 0, 421, 630, 611, 622, 612, - 597, 598, 599, 606, 381, 600, 601, 602, 572, 603, - 573, 604, 605, 0, 629, 579, 491, 437, 0, 646, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 0, 0, 0, 0, 337, 246, 574, - 696, 576, 575, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 670, 669, 668, 667, 666, 665, 664, 663, + 0, 0, 612, 512, 355, 307, 351, 352, 359, 729, + 725, 730, 713, 716, 715, 691, 0, 315, 592, 427, + 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, + 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, + 272, 273, 278, 279, 280, 281, 282, 283, 284, 285, + 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, + 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, + 0, 309, 717, 718, 719, 720, 721, 0, 0, 310, + 311, 312, 0, 0, 274, 275, 302, 503, 303, 304, + 305, 306, 0, 0, 542, 543, 544, 567, 0, 545, + 527, 591, 386, 316, 507, 534, 727, 0, 0, 0, + 0, 0, 0, 0, 642, 653, 687, 0, 699, 700, + 702, 704, 703, 706, 500, 501, 714, 0, 0, 708, + 709, 710, 707, 431, 487, 508, 494, 0, 733, 582, + 583, 734, 695, 317, 458, 0, 0, 597, 631, 620, + 705, 585, 0, 0, 1557, 0, 0, 0, 0, 0, + 0, 0, 369, 0, 0, 426, 635, 616, 627, 617, + 602, 603, 604, 611, 381, 605, 606, 607, 577, 608, + 578, 609, 610, 0, 634, 584, 496, 442, 0, 651, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 245, 0, 0, 0, 0, 0, 0, 337, 246, 579, + 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 492, 521, 0, 534, 0, 406, 407, 0, - 0, 0, 0, 0, 0, 0, 324, 499, 518, 338, - 486, 532, 343, 494, 511, 333, 452, 483, 0, 0, - 326, 516, 493, 434, 325, 0, 477, 366, 383, 363, - 450, 0, 0, 515, 545, 362, 535, 0, 526, 328, - 0, 525, 449, 512, 517, 435, 428, 0, 327, 514, - 433, 427, 412, 373, 561, 413, 414, 387, 464, 425, - 465, 388, 439, 438, 440, 389, 390, 391, 392, 393, - 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, - 556, 557, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 689, 0, 0, - 693, 0, 528, 0, 0, 0, 0, 0, 0, 497, - 0, 0, 415, 0, 0, 0, 546, 0, 480, 455, - 731, 0, 0, 478, 423, 513, 466, 519, 500, 527, - 472, 467, 318, 501, 365, 436, 334, 336, 721, 367, - 370, 374, 375, 445, 446, 460, 485, 504, 505, 506, - 364, 348, 479, 349, 384, 350, 319, 356, 354, 357, - 487, 358, 321, 461, 510, 0, 380, 475, 431, 322, - 430, 462, 509, 508, 335, 536, 543, 544, 634, 0, - 549, 732, 733, 734, 558, 0, 468, 331, 330, 0, - 0, 0, 360, 463, 344, 346, 347, 345, 458, 459, - 563, 564, 565, 567, 0, 568, 569, 0, 0, 0, - 0, 570, 635, 651, 619, 588, 551, 643, 585, 589, - 590, 401, 402, 403, 654, 0, 0, 0, 542, 416, - 417, 0, 372, 371, 432, 323, 0, 0, 409, 400, - 469, 329, 368, 411, 405, 418, 419, 420, 378, 313, - 314, 727, 361, 451, 656, 691, 692, 581, 0, 644, - 582, 591, 353, 616, 628, 627, 447, 541, 0, 639, - 642, 571, 726, 0, 636, 650, 730, 649, 723, 457, - 0, 484, 647, 594, 0, 640, 613, 614, 0, 641, - 609, 645, 0, 583, 0, 552, 555, 584, 669, 670, - 671, 320, 554, 673, 674, 675, 676, 677, 678, 679, - 672, 524, 617, 593, 620, 533, 596, 595, 0, 0, - 631, 550, 632, 633, 441, 442, 443, 444, 382, 657, - 342, 553, 471, 0, 618, 0, 0, 0, 0, 0, - 0, 0, 0, 623, 624, 621, 735, 0, 680, 681, - 0, 0, 547, 548, 377, 0, 566, 385, 341, 456, - 379, 531, 408, 0, 559, 625, 560, 473, 474, 683, - 688, 684, 685, 687, 707, 448, 399, 404, 488, 410, - 424, 476, 530, 454, 481, 339, 520, 490, 429, 610, - 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 665, 664, - 663, 662, 661, 660, 659, 658, 0, 0, 607, 507, - 355, 307, 351, 352, 359, 724, 720, 725, 708, 711, - 710, 686, 0, 315, 587, 422, 470, 376, 652, 653, - 0, 706, 259, 260, 261, 262, 263, 264, 265, 266, - 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, - 280, 281, 282, 283, 284, 285, 655, 276, 277, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 298, 299, 0, 0, 0, 0, 309, 712, 713, - 714, 715, 716, 0, 0, 310, 311, 312, 0, 0, - 274, 275, 302, 498, 303, 304, 305, 306, 0, 0, - 537, 538, 539, 562, 0, 540, 522, 586, 386, 316, - 502, 529, 722, 0, 0, 0, 0, 0, 0, 0, - 637, 648, 682, 0, 694, 695, 697, 699, 698, 701, - 495, 496, 709, 0, 0, 703, 704, 705, 702, 426, - 482, 503, 489, 0, 728, 577, 578, 729, 690, 317, - 453, 0, 0, 592, 626, 615, 700, 580, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, - 0, 421, 630, 611, 622, 612, 597, 598, 599, 606, - 381, 600, 601, 602, 572, 603, 573, 604, 605, 0, - 629, 579, 491, 437, 0, 646, 0, 0, 0, 0, + 0, 0, 497, 526, 0, 539, 0, 407, 408, 0, + 0, 0, 0, 0, 0, 0, 324, 504, 523, 338, + 491, 537, 343, 499, 516, 333, 457, 488, 0, 0, + 326, 521, 498, 439, 325, 0, 482, 366, 383, 363, + 455, 0, 0, 520, 550, 362, 540, 0, 531, 328, + 0, 530, 454, 517, 522, 440, 433, 0, 327, 519, + 438, 432, 413, 373, 566, 414, 415, 416, 417, 418, + 419, 387, 469, 430, 470, 388, 444, 443, 445, 389, + 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, + 0, 0, 0, 0, 561, 562, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 694, 0, 0, 698, 0, 533, 0, 0, 0, + 0, 0, 0, 502, 0, 0, 420, 0, 0, 0, + 551, 0, 485, 460, 736, 0, 0, 483, 428, 518, + 471, 524, 505, 532, 477, 472, 318, 506, 365, 441, + 334, 336, 726, 367, 370, 374, 375, 450, 451, 465, + 490, 509, 510, 511, 364, 348, 484, 349, 384, 350, + 319, 356, 354, 357, 492, 358, 321, 466, 515, 0, + 380, 480, 436, 322, 435, 467, 514, 513, 335, 541, + 548, 549, 639, 0, 554, 737, 738, 739, 563, 0, + 473, 331, 330, 0, 0, 0, 360, 468, 344, 346, + 347, 345, 463, 464, 568, 569, 570, 572, 0, 573, + 574, 0, 0, 0, 0, 575, 640, 656, 624, 593, + 556, 648, 590, 594, 595, 401, 402, 403, 404, 659, + 0, 0, 0, 547, 421, 422, 0, 372, 371, 437, + 323, 0, 0, 410, 400, 474, 329, 368, 412, 406, + 423, 424, 425, 378, 313, 314, 732, 361, 456, 661, + 696, 697, 586, 0, 649, 587, 596, 353, 621, 633, + 632, 452, 546, 0, 644, 647, 576, 731, 0, 641, + 655, 735, 654, 728, 462, 0, 489, 652, 599, 0, + 645, 618, 619, 0, 646, 614, 650, 0, 588, 0, + 557, 560, 589, 674, 675, 676, 320, 559, 678, 679, + 680, 681, 682, 683, 684, 677, 529, 622, 598, 625, + 538, 601, 600, 0, 0, 636, 555, 637, 638, 446, + 447, 448, 449, 382, 662, 342, 558, 476, 0, 623, + 0, 0, 0, 0, 0, 0, 0, 0, 628, 629, + 626, 740, 0, 685, 686, 0, 0, 552, 553, 377, + 0, 571, 385, 341, 461, 379, 536, 409, 0, 564, + 630, 565, 478, 479, 688, 693, 689, 690, 692, 712, + 453, 399, 405, 493, 411, 429, 481, 535, 459, 486, + 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, - 2123, 0, 0, 337, 246, 574, 696, 576, 575, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, + 0, 0, 0, 670, 669, 668, 667, 666, 665, 664, + 663, 0, 0, 612, 512, 355, 307, 351, 352, 359, + 729, 725, 730, 713, 716, 715, 2354, 0, 315, 592, + 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, + 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, + 271, 272, 273, 278, 279, 280, 281, 282, 283, 284, + 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 298, 299, 0, 0, + 0, 0, 309, 717, 718, 719, 720, 721, 0, 0, + 310, 311, 312, 0, 0, 274, 275, 302, 503, 303, + 304, 305, 306, 0, 0, 542, 543, 544, 567, 0, + 545, 527, 591, 386, 316, 507, 534, 727, 0, 0, + 0, 0, 0, 0, 0, 642, 653, 687, 0, 699, + 700, 702, 704, 703, 706, 500, 501, 714, 0, 0, + 708, 709, 710, 707, 431, 487, 508, 494, 0, 733, + 582, 583, 734, 695, 317, 458, 0, 0, 597, 631, + 620, 705, 585, 0, 2290, 0, 0, 0, 0, 0, + 0, 0, 0, 369, 0, 0, 426, 635, 616, 627, + 617, 602, 603, 604, 611, 381, 605, 606, 607, 577, + 608, 578, 609, 610, 0, 634, 584, 496, 442, 0, + 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 245, 0, 0, 0, 0, 0, 0, 337, 246, + 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 492, 521, - 0, 534, 0, 406, 407, 0, 0, 0, 0, 0, - 0, 0, 324, 499, 518, 338, 486, 532, 343, 494, - 511, 333, 452, 483, 0, 0, 326, 516, 493, 434, - 325, 0, 477, 366, 383, 363, 450, 0, 0, 515, - 545, 362, 535, 0, 526, 328, 0, 525, 449, 512, - 517, 435, 428, 0, 327, 514, 433, 427, 412, 373, - 561, 413, 414, 387, 464, 425, 465, 388, 439, 438, - 440, 389, 390, 391, 392, 393, 394, 395, 396, 397, - 398, 0, 0, 0, 0, 0, 556, 557, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 689, 0, 0, 693, 0, 528, 0, - 0, 0, 0, 0, 0, 497, 0, 0, 415, 0, - 0, 0, 546, 0, 480, 455, 731, 0, 0, 478, - 423, 513, 466, 519, 500, 527, 472, 467, 318, 501, - 365, 436, 334, 336, 721, 367, 370, 374, 375, 445, - 446, 460, 485, 504, 505, 506, 364, 348, 479, 349, - 384, 350, 319, 356, 354, 357, 487, 358, 321, 461, - 510, 0, 380, 475, 431, 322, 430, 462, 509, 508, - 335, 536, 543, 544, 634, 0, 549, 732, 733, 734, - 558, 0, 468, 331, 330, 0, 0, 0, 360, 463, - 344, 346, 347, 345, 458, 459, 563, 564, 565, 567, - 0, 568, 569, 0, 0, 0, 0, 570, 635, 651, - 619, 588, 551, 643, 585, 589, 590, 401, 402, 403, - 654, 0, 0, 0, 542, 416, 417, 0, 372, 371, - 432, 323, 0, 0, 409, 400, 469, 329, 368, 411, - 405, 418, 419, 420, 378, 313, 314, 727, 361, 451, - 656, 691, 692, 581, 0, 644, 582, 591, 353, 616, - 628, 627, 447, 541, 0, 639, 642, 571, 726, 0, - 636, 650, 730, 649, 723, 457, 0, 484, 647, 594, - 0, 640, 613, 614, 0, 641, 609, 645, 0, 583, - 0, 552, 555, 584, 669, 670, 671, 320, 554, 673, - 674, 675, 676, 677, 678, 679, 672, 524, 617, 593, - 620, 533, 596, 595, 0, 0, 631, 550, 632, 633, - 441, 442, 443, 444, 382, 657, 342, 553, 471, 0, - 618, 0, 0, 0, 0, 0, 0, 0, 0, 623, - 624, 621, 735, 0, 680, 681, 0, 0, 547, 548, - 377, 0, 566, 385, 341, 456, 379, 531, 408, 0, - 559, 625, 560, 473, 474, 683, 688, 684, 685, 687, - 707, 448, 399, 404, 488, 410, 424, 476, 530, 454, - 481, 339, 520, 490, 429, 610, 638, 0, 0, 0, + 0, 0, 0, 497, 526, 0, 539, 0, 407, 408, + 0, 0, 0, 0, 0, 0, 0, 324, 504, 523, + 338, 491, 537, 343, 499, 516, 333, 457, 488, 0, + 0, 326, 521, 498, 439, 325, 0, 482, 366, 383, + 363, 455, 0, 0, 520, 550, 362, 540, 0, 531, + 328, 0, 530, 454, 517, 522, 440, 433, 0, 327, + 519, 438, 432, 413, 373, 566, 414, 415, 416, 417, + 418, 419, 387, 469, 430, 470, 388, 444, 443, 445, + 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, + 0, 0, 0, 0, 0, 561, 562, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 694, 0, 0, 698, 0, 533, 0, 0, + 0, 0, 0, 0, 502, 0, 0, 420, 0, 0, + 0, 551, 0, 485, 460, 736, 0, 0, 483, 428, + 518, 471, 524, 505, 532, 477, 472, 318, 506, 365, + 441, 334, 336, 726, 367, 370, 374, 375, 450, 451, + 465, 490, 509, 510, 511, 364, 348, 484, 349, 384, + 350, 319, 356, 354, 357, 492, 358, 321, 466, 515, + 0, 380, 480, 436, 322, 435, 467, 514, 513, 335, + 541, 548, 549, 639, 0, 554, 737, 738, 739, 563, + 0, 473, 331, 330, 0, 0, 0, 360, 468, 344, + 346, 347, 345, 463, 464, 568, 569, 570, 572, 0, + 573, 574, 0, 0, 0, 0, 575, 640, 656, 624, + 593, 556, 648, 590, 594, 595, 401, 402, 403, 404, + 659, 0, 0, 0, 547, 421, 422, 0, 372, 371, + 437, 323, 0, 0, 410, 400, 474, 329, 368, 412, + 406, 423, 424, 425, 378, 313, 314, 732, 361, 456, + 661, 696, 697, 586, 0, 649, 587, 596, 353, 621, + 633, 632, 452, 546, 0, 644, 647, 576, 731, 0, + 641, 655, 735, 654, 728, 462, 0, 489, 652, 599, + 0, 645, 618, 619, 0, 646, 614, 650, 0, 588, + 0, 557, 560, 589, 674, 675, 676, 320, 559, 678, + 679, 680, 681, 682, 683, 684, 677, 529, 622, 598, + 625, 538, 601, 600, 0, 0, 636, 555, 637, 638, + 446, 447, 448, 449, 382, 662, 342, 558, 476, 0, + 623, 0, 0, 0, 0, 0, 0, 0, 0, 628, + 629, 626, 740, 0, 685, 686, 0, 0, 552, 553, + 377, 0, 571, 385, 341, 461, 379, 536, 409, 0, + 564, 630, 565, 478, 479, 688, 693, 689, 690, 692, + 712, 453, 399, 405, 493, 411, 429, 481, 535, 459, + 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 665, 664, 663, 662, 661, 660, - 659, 658, 0, 0, 607, 507, 355, 307, 351, 352, - 359, 724, 720, 725, 708, 711, 710, 686, 0, 315, - 587, 422, 470, 376, 652, 653, 0, 706, 259, 260, + 0, 0, 0, 0, 670, 669, 668, 667, 666, 665, + 664, 663, 0, 0, 612, 512, 355, 307, 351, 352, + 359, 729, 725, 730, 713, 716, 715, 691, 0, 315, + 592, 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, 283, - 284, 285, 655, 276, 277, 286, 287, 288, 289, 290, + 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, - 0, 0, 0, 309, 712, 713, 714, 715, 716, 0, - 0, 310, 311, 312, 0, 0, 274, 275, 302, 498, - 303, 304, 305, 306, 0, 0, 537, 538, 539, 562, - 0, 540, 522, 586, 386, 316, 502, 529, 722, 0, - 0, 0, 0, 0, 0, 0, 637, 648, 682, 0, - 694, 695, 697, 699, 698, 701, 495, 496, 709, 0, - 0, 703, 704, 705, 702, 426, 482, 503, 489, 0, - 728, 577, 578, 729, 690, 317, 453, 0, 0, 592, - 626, 615, 700, 580, 0, 0, 1546, 0, 0, 0, - 0, 0, 0, 0, 369, 0, 0, 421, 630, 611, - 622, 612, 597, 598, 599, 606, 381, 600, 601, 602, - 572, 603, 573, 604, 605, 0, 629, 579, 491, 437, - 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 245, 0, 0, 0, 0, 0, 0, 337, - 246, 574, 696, 576, 575, 0, 0, 0, 0, 0, + 0, 0, 0, 309, 717, 718, 719, 720, 721, 0, + 0, 310, 311, 312, 0, 0, 274, 275, 302, 503, + 303, 304, 305, 306, 0, 0, 542, 543, 544, 567, + 0, 545, 527, 591, 386, 316, 507, 534, 727, 0, + 0, 0, 0, 0, 0, 0, 642, 653, 687, 0, + 699, 700, 702, 704, 703, 706, 500, 501, 714, 0, + 0, 708, 709, 710, 707, 431, 487, 508, 494, 0, + 733, 582, 583, 734, 695, 317, 458, 0, 0, 597, + 631, 620, 705, 585, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 369, 0, 0, 426, 635, 616, + 627, 617, 602, 603, 604, 611, 381, 605, 606, 607, + 577, 608, 578, 609, 610, 0, 634, 584, 496, 442, + 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 245, 0, 0, 1734, 0, 0, 0, 337, + 246, 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 492, 521, 0, 534, 0, 406, - 407, 0, 0, 0, 0, 0, 0, 0, 324, 499, - 518, 338, 486, 532, 343, 494, 511, 333, 452, 483, - 0, 0, 326, 516, 493, 434, 325, 0, 477, 366, - 383, 363, 450, 0, 0, 515, 545, 362, 535, 0, - 526, 328, 0, 525, 449, 512, 517, 435, 428, 0, - 327, 514, 433, 427, 412, 373, 561, 413, 414, 387, - 464, 425, 465, 388, 439, 438, 440, 389, 390, 391, - 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, - 0, 0, 556, 557, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, - 0, 0, 693, 0, 528, 0, 0, 0, 0, 0, - 0, 497, 0, 0, 415, 0, 0, 0, 546, 0, - 480, 455, 731, 0, 0, 478, 423, 513, 466, 519, - 500, 527, 472, 467, 318, 501, 365, 436, 334, 336, - 721, 367, 370, 374, 375, 445, 446, 460, 485, 504, - 505, 506, 364, 348, 479, 349, 384, 350, 319, 356, - 354, 357, 487, 358, 321, 461, 510, 0, 380, 475, - 431, 322, 430, 462, 509, 508, 335, 536, 543, 544, - 634, 0, 549, 732, 733, 734, 558, 0, 468, 331, - 330, 0, 0, 0, 360, 463, 344, 346, 347, 345, - 458, 459, 563, 564, 565, 567, 0, 568, 569, 0, - 0, 0, 0, 570, 635, 651, 619, 588, 551, 643, - 585, 589, 590, 401, 402, 403, 654, 0, 0, 0, - 542, 416, 417, 0, 372, 371, 432, 323, 0, 0, - 409, 400, 469, 329, 368, 411, 405, 418, 419, 420, - 378, 313, 314, 727, 361, 451, 656, 691, 692, 581, - 0, 644, 582, 591, 353, 616, 628, 627, 447, 541, - 0, 639, 642, 571, 726, 0, 636, 650, 730, 649, - 723, 457, 0, 484, 647, 594, 0, 640, 613, 614, - 0, 641, 609, 645, 0, 583, 0, 552, 555, 584, - 669, 670, 671, 320, 554, 673, 674, 675, 676, 677, - 678, 679, 672, 524, 617, 593, 620, 533, 596, 595, - 0, 0, 631, 550, 632, 633, 441, 442, 443, 444, - 382, 657, 342, 553, 471, 0, 618, 0, 0, 0, - 0, 0, 0, 0, 0, 623, 624, 621, 735, 0, - 680, 681, 0, 0, 547, 548, 377, 0, 566, 385, - 341, 456, 379, 531, 408, 0, 559, 625, 560, 473, - 474, 683, 688, 684, 685, 687, 707, 448, 399, 404, - 488, 410, 424, 476, 530, 454, 481, 339, 520, 490, - 429, 610, 638, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 665, 664, 663, 662, 661, 660, 659, 658, 0, 0, - 607, 507, 355, 307, 351, 352, 359, 724, 720, 725, - 708, 711, 710, 2338, 0, 315, 587, 422, 470, 376, - 652, 653, 0, 706, 259, 260, 261, 262, 263, 264, - 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, - 278, 279, 280, 281, 282, 283, 284, 285, 655, 276, - 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 298, 299, 0, 0, 0, 0, 309, - 712, 713, 714, 715, 716, 0, 0, 310, 311, 312, - 0, 0, 274, 275, 302, 498, 303, 304, 305, 306, - 0, 0, 537, 538, 539, 562, 0, 540, 522, 586, - 386, 316, 502, 529, 722, 0, 0, 0, 0, 0, - 0, 0, 637, 648, 682, 0, 694, 695, 697, 699, - 698, 701, 495, 496, 709, 0, 0, 703, 704, 705, - 702, 426, 482, 503, 489, 0, 728, 577, 578, 729, - 690, 317, 453, 0, 0, 592, 626, 615, 700, 580, - 0, 2274, 0, 0, 0, 0, 0, 0, 0, 0, - 369, 0, 0, 421, 630, 611, 622, 612, 597, 598, - 599, 606, 381, 600, 601, 602, 572, 603, 573, 604, - 605, 0, 629, 579, 491, 437, 0, 646, 0, 0, + 0, 0, 0, 0, 497, 526, 0, 539, 0, 407, + 408, 0, 0, 0, 0, 0, 0, 0, 324, 504, + 523, 338, 491, 537, 343, 499, 516, 333, 457, 488, + 0, 0, 326, 521, 498, 439, 325, 0, 482, 366, + 383, 363, 455, 0, 0, 520, 550, 362, 540, 0, + 531, 328, 0, 530, 454, 517, 522, 440, 433, 0, + 327, 519, 438, 432, 413, 373, 566, 414, 415, 416, + 417, 418, 419, 387, 469, 430, 470, 388, 444, 443, + 445, 389, 390, 391, 392, 393, 394, 395, 396, 397, + 398, 0, 0, 0, 0, 0, 561, 562, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 694, 0, 0, 698, 0, 533, 0, + 0, 0, 0, 0, 0, 502, 0, 0, 420, 0, + 0, 0, 551, 0, 485, 460, 736, 0, 0, 483, + 428, 518, 471, 524, 505, 532, 2186, 472, 318, 506, + 365, 441, 334, 336, 726, 367, 370, 374, 375, 450, + 451, 465, 490, 509, 510, 511, 364, 348, 484, 349, + 384, 350, 319, 356, 354, 357, 492, 358, 321, 466, + 515, 0, 380, 480, 436, 322, 435, 467, 514, 513, + 335, 541, 548, 549, 639, 0, 554, 737, 738, 739, + 563, 0, 473, 331, 330, 0, 0, 0, 360, 468, + 344, 346, 347, 345, 463, 464, 568, 569, 570, 572, + 0, 573, 574, 0, 0, 0, 0, 575, 640, 656, + 624, 593, 556, 648, 590, 594, 595, 401, 402, 403, + 404, 659, 0, 0, 0, 547, 421, 422, 0, 372, + 371, 437, 323, 0, 0, 410, 400, 474, 329, 368, + 412, 406, 423, 424, 425, 378, 313, 314, 732, 361, + 456, 661, 696, 697, 586, 0, 649, 587, 596, 353, + 621, 633, 632, 452, 546, 0, 644, 647, 576, 731, + 0, 641, 655, 735, 654, 728, 462, 0, 489, 652, + 599, 0, 645, 618, 619, 0, 646, 614, 650, 0, + 588, 0, 557, 560, 589, 674, 675, 676, 320, 559, + 678, 679, 680, 681, 682, 683, 684, 677, 529, 622, + 598, 625, 538, 601, 600, 0, 0, 636, 555, 637, + 638, 446, 447, 448, 449, 382, 662, 342, 558, 476, + 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, + 628, 629, 626, 740, 0, 685, 686, 0, 0, 552, + 553, 377, 0, 571, 385, 341, 461, 379, 536, 409, + 0, 564, 630, 565, 478, 479, 688, 693, 689, 690, + 692, 712, 453, 399, 405, 493, 411, 429, 481, 535, + 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, + 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 670, 669, 668, 667, 666, + 665, 664, 663, 0, 0, 612, 512, 355, 307, 351, + 352, 359, 729, 725, 730, 713, 716, 715, 691, 0, + 315, 592, 427, 475, 376, 657, 658, 0, 711, 259, + 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, + 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, + 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, + 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, + 0, 0, 0, 0, 309, 717, 718, 719, 720, 721, + 0, 0, 310, 311, 312, 0, 0, 274, 275, 302, + 503, 303, 304, 305, 306, 0, 0, 542, 543, 544, + 567, 0, 545, 527, 591, 386, 316, 507, 534, 727, + 0, 0, 0, 0, 0, 0, 0, 642, 653, 687, + 0, 699, 700, 702, 704, 703, 706, 500, 501, 714, + 0, 0, 708, 709, 710, 707, 431, 487, 508, 494, + 0, 733, 582, 583, 734, 695, 317, 458, 0, 0, + 597, 631, 620, 705, 585, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 369, 0, 0, 426, 635, + 616, 627, 617, 602, 603, 604, 611, 381, 605, 606, + 607, 577, 608, 578, 609, 610, 0, 634, 584, 496, + 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 0, 0, 0, 0, 337, 246, 574, 696, 576, - 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, + 337, 246, 579, 701, 581, 580, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 492, 521, 0, 534, 0, 406, 407, 0, 0, 0, - 0, 0, 0, 0, 324, 499, 518, 338, 486, 532, - 343, 494, 511, 333, 452, 483, 0, 0, 326, 516, - 493, 434, 325, 0, 477, 366, 383, 363, 450, 0, - 0, 515, 545, 362, 535, 0, 526, 328, 0, 525, - 449, 512, 517, 435, 428, 0, 327, 514, 433, 427, - 412, 373, 561, 413, 414, 387, 464, 425, 465, 388, - 439, 438, 440, 389, 390, 391, 392, 393, 394, 395, - 396, 397, 398, 0, 0, 0, 0, 0, 556, 557, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 689, 0, 0, 693, 0, - 528, 0, 0, 0, 0, 0, 0, 497, 0, 0, - 415, 0, 0, 0, 546, 0, 480, 455, 731, 0, - 0, 478, 423, 513, 466, 519, 500, 527, 472, 467, - 318, 501, 365, 436, 334, 336, 721, 367, 370, 374, - 375, 445, 446, 460, 485, 504, 505, 506, 364, 348, - 479, 349, 384, 350, 319, 356, 354, 357, 487, 358, - 321, 461, 510, 0, 380, 475, 431, 322, 430, 462, - 509, 508, 335, 536, 543, 544, 634, 0, 549, 732, - 733, 734, 558, 0, 468, 331, 330, 0, 0, 0, - 360, 463, 344, 346, 347, 345, 458, 459, 563, 564, - 565, 567, 0, 568, 569, 0, 0, 0, 0, 570, - 635, 651, 619, 588, 551, 643, 585, 589, 590, 401, - 402, 403, 654, 0, 0, 0, 542, 416, 417, 0, - 372, 371, 432, 323, 0, 0, 409, 400, 469, 329, - 368, 411, 405, 418, 419, 420, 378, 313, 314, 727, - 361, 451, 656, 691, 692, 581, 0, 644, 582, 591, - 353, 616, 628, 627, 447, 541, 0, 639, 642, 571, - 726, 0, 636, 650, 730, 649, 723, 457, 0, 484, - 647, 594, 0, 640, 613, 614, 0, 641, 609, 645, - 0, 583, 0, 552, 555, 584, 669, 670, 671, 320, - 554, 673, 674, 675, 676, 677, 678, 679, 672, 524, - 617, 593, 620, 533, 596, 595, 0, 0, 631, 550, - 632, 633, 441, 442, 443, 444, 382, 657, 342, 553, - 471, 0, 618, 0, 0, 0, 0, 0, 0, 0, - 0, 623, 624, 621, 735, 0, 680, 681, 0, 0, - 547, 548, 377, 0, 566, 385, 341, 456, 379, 531, - 408, 0, 559, 625, 560, 473, 474, 683, 688, 684, - 685, 687, 707, 448, 399, 404, 488, 410, 424, 476, - 530, 454, 481, 339, 520, 490, 429, 610, 638, 0, + 0, 0, 0, 0, 0, 497, 526, 0, 539, 0, + 407, 408, 0, 0, 0, 0, 0, 0, 0, 324, + 504, 523, 338, 491, 537, 343, 499, 516, 333, 457, + 488, 0, 0, 326, 521, 498, 439, 325, 0, 482, + 366, 383, 363, 455, 0, 0, 520, 550, 362, 540, + 0, 531, 328, 0, 530, 454, 517, 522, 440, 433, + 0, 327, 519, 438, 432, 413, 373, 566, 414, 415, + 416, 417, 418, 419, 387, 469, 430, 470, 388, 444, + 443, 445, 389, 390, 391, 392, 393, 394, 395, 396, + 397, 398, 0, 0, 0, 0, 0, 561, 562, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 694, 0, 0, 698, 0, 533, + 0, 0, 1764, 0, 0, 0, 502, 0, 0, 420, + 0, 0, 0, 551, 0, 485, 460, 736, 0, 0, + 483, 428, 518, 471, 524, 505, 532, 477, 472, 318, + 506, 365, 441, 334, 336, 726, 367, 370, 374, 375, + 450, 451, 465, 490, 509, 510, 511, 364, 348, 484, + 349, 384, 350, 319, 356, 354, 357, 492, 358, 321, + 466, 515, 0, 380, 480, 436, 322, 435, 467, 514, + 513, 335, 541, 548, 549, 639, 0, 554, 737, 738, + 739, 563, 0, 473, 331, 330, 0, 0, 0, 360, + 468, 344, 346, 347, 345, 463, 464, 568, 569, 570, + 572, 0, 573, 574, 0, 0, 0, 0, 575, 640, + 656, 624, 593, 556, 648, 590, 594, 595, 401, 402, + 403, 404, 659, 0, 0, 0, 547, 421, 422, 0, + 372, 371, 437, 323, 0, 0, 410, 400, 474, 329, + 368, 412, 406, 423, 424, 425, 378, 313, 314, 732, + 361, 456, 661, 696, 697, 586, 0, 649, 587, 596, + 353, 621, 633, 632, 452, 546, 0, 644, 647, 576, + 731, 0, 641, 655, 735, 654, 728, 462, 0, 489, + 652, 599, 0, 645, 618, 619, 0, 646, 614, 650, + 0, 588, 0, 557, 560, 589, 674, 675, 676, 320, + 559, 678, 679, 680, 681, 682, 683, 684, 677, 529, + 622, 598, 625, 538, 601, 600, 0, 0, 636, 555, + 637, 638, 446, 447, 448, 449, 382, 662, 342, 558, + 476, 0, 623, 0, 0, 0, 0, 0, 0, 0, + 0, 628, 629, 626, 740, 0, 685, 686, 0, 0, + 552, 553, 377, 0, 571, 385, 341, 461, 379, 536, + 409, 0, 564, 630, 565, 478, 479, 688, 693, 689, + 690, 692, 712, 453, 399, 405, 493, 411, 429, 481, + 535, 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 665, 664, 663, 662, - 661, 660, 659, 658, 0, 0, 607, 507, 355, 307, - 351, 352, 359, 724, 720, 725, 708, 711, 710, 686, - 0, 315, 587, 422, 470, 376, 652, 653, 0, 706, + 0, 0, 0, 0, 0, 0, 670, 669, 668, 667, + 666, 665, 664, 663, 0, 0, 612, 512, 355, 307, + 351, 352, 359, 729, 725, 730, 713, 716, 715, 691, + 0, 315, 592, 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, - 282, 283, 284, 285, 655, 276, 277, 286, 287, 288, + 282, 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, - 299, 0, 0, 0, 0, 309, 712, 713, 714, 715, - 716, 0, 0, 310, 311, 312, 0, 0, 274, 275, - 302, 498, 303, 304, 305, 306, 0, 0, 537, 538, - 539, 562, 0, 540, 522, 586, 386, 316, 502, 529, - 722, 0, 0, 0, 0, 0, 0, 0, 637, 648, - 682, 0, 694, 695, 697, 699, 698, 701, 495, 496, - 709, 0, 0, 703, 704, 705, 702, 426, 482, 503, - 489, 0, 728, 577, 578, 729, 690, 317, 453, 0, - 0, 592, 626, 615, 700, 580, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 369, 0, 0, 421, - 630, 611, 622, 612, 597, 598, 599, 606, 381, 600, - 601, 602, 572, 603, 573, 604, 605, 0, 629, 579, - 491, 437, 0, 646, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 245, 0, 0, 1723, 0, 0, - 0, 337, 246, 574, 696, 576, 575, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 299, 0, 0, 0, 0, 309, 717, 718, 719, 720, + 721, 0, 0, 310, 311, 312, 0, 0, 274, 275, + 302, 503, 303, 304, 305, 306, 0, 0, 542, 543, + 544, 567, 0, 545, 527, 591, 386, 316, 507, 534, + 727, 0, 0, 0, 0, 0, 0, 0, 642, 653, + 687, 0, 699, 700, 702, 704, 703, 706, 500, 501, + 714, 0, 0, 708, 709, 710, 707, 431, 487, 508, + 494, 0, 733, 582, 583, 734, 695, 317, 458, 0, + 0, 597, 631, 620, 705, 585, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1075, 369, 0, 0, 426, + 635, 616, 627, 617, 602, 603, 604, 611, 381, 605, + 606, 607, 577, 608, 578, 609, 610, 0, 634, 584, + 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, + 0, 337, 246, 579, 701, 581, 580, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 492, 521, 0, 534, - 0, 406, 407, 0, 0, 0, 0, 0, 0, 0, - 324, 499, 518, 338, 486, 532, 343, 494, 511, 333, - 452, 483, 0, 0, 326, 516, 493, 434, 325, 0, - 477, 366, 383, 363, 450, 0, 0, 515, 545, 362, - 535, 0, 526, 328, 0, 525, 449, 512, 517, 435, - 428, 0, 327, 514, 433, 427, 412, 373, 561, 413, - 414, 387, 464, 425, 465, 388, 439, 438, 440, 389, - 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, - 0, 0, 0, 0, 556, 557, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 689, 0, 0, 693, 0, 528, 0, 0, 0, - 0, 0, 0, 497, 0, 0, 415, 0, 0, 0, - 546, 0, 480, 455, 731, 0, 0, 478, 423, 513, - 466, 519, 500, 527, 2170, 467, 318, 501, 365, 436, - 334, 336, 721, 367, 370, 374, 375, 445, 446, 460, - 485, 504, 505, 506, 364, 348, 479, 349, 384, 350, - 319, 356, 354, 357, 487, 358, 321, 461, 510, 0, - 380, 475, 431, 322, 430, 462, 509, 508, 335, 536, - 543, 544, 634, 0, 549, 732, 733, 734, 558, 0, - 468, 331, 330, 0, 0, 0, 360, 463, 344, 346, - 347, 345, 458, 459, 563, 564, 565, 567, 0, 568, - 569, 0, 0, 0, 0, 570, 635, 651, 619, 588, - 551, 643, 585, 589, 590, 401, 402, 403, 654, 0, - 0, 0, 542, 416, 417, 0, 372, 371, 432, 323, - 0, 0, 409, 400, 469, 329, 368, 411, 405, 418, - 419, 420, 378, 313, 314, 727, 361, 451, 656, 691, - 692, 581, 0, 644, 582, 591, 353, 616, 628, 627, - 447, 541, 0, 639, 642, 571, 726, 0, 636, 650, - 730, 649, 723, 457, 0, 484, 647, 594, 0, 640, - 613, 614, 0, 641, 609, 645, 0, 583, 0, 552, - 555, 584, 669, 670, 671, 320, 554, 673, 674, 675, - 676, 677, 678, 679, 672, 524, 617, 593, 620, 533, - 596, 595, 0, 0, 631, 550, 632, 633, 441, 442, - 443, 444, 382, 657, 342, 553, 471, 0, 618, 0, - 0, 0, 0, 0, 0, 0, 0, 623, 624, 621, - 735, 0, 680, 681, 0, 0, 547, 548, 377, 0, - 566, 385, 341, 456, 379, 531, 408, 0, 559, 625, - 560, 473, 474, 683, 688, 684, 685, 687, 707, 448, - 399, 404, 488, 410, 424, 476, 530, 454, 481, 339, - 520, 490, 429, 610, 638, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 665, 664, 663, 662, 661, 660, 659, 658, - 0, 0, 607, 507, 355, 307, 351, 352, 359, 724, - 720, 725, 708, 711, 710, 686, 0, 315, 587, 422, - 470, 376, 652, 653, 0, 706, 259, 260, 261, 262, - 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, - 272, 273, 278, 279, 280, 281, 282, 283, 284, 285, - 655, 276, 277, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, - 0, 309, 712, 713, 714, 715, 716, 0, 0, 310, - 311, 312, 0, 0, 274, 275, 302, 498, 303, 304, - 305, 306, 0, 0, 537, 538, 539, 562, 0, 540, - 522, 586, 386, 316, 502, 529, 722, 0, 0, 0, - 0, 0, 0, 0, 637, 648, 682, 0, 694, 695, - 697, 699, 698, 701, 495, 496, 709, 0, 0, 703, - 704, 705, 702, 426, 482, 503, 489, 0, 728, 577, - 578, 729, 690, 317, 453, 0, 0, 592, 626, 615, - 700, 580, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 369, 0, 0, 421, 630, 611, 622, 612, - 597, 598, 599, 606, 381, 600, 601, 602, 572, 603, - 573, 604, 605, 0, 629, 579, 491, 437, 0, 646, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 0, 0, 0, 0, 337, 246, 574, - 696, 576, 575, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 497, 526, 0, 539, + 0, 407, 408, 0, 0, 0, 0, 0, 0, 0, + 324, 504, 523, 338, 491, 537, 343, 499, 516, 333, + 457, 488, 0, 0, 326, 521, 498, 439, 325, 0, + 482, 366, 383, 363, 455, 0, 0, 520, 550, 362, + 540, 0, 531, 328, 0, 530, 454, 517, 522, 440, + 433, 0, 327, 519, 438, 432, 413, 373, 566, 414, + 415, 416, 417, 418, 419, 387, 469, 430, 470, 388, + 444, 443, 445, 389, 390, 391, 392, 393, 394, 395, + 396, 397, 398, 0, 0, 0, 0, 0, 561, 562, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 694, 0, 0, 698, 0, + 533, 0, 0, 0, 0, 0, 0, 502, 0, 0, + 420, 0, 0, 0, 551, 0, 485, 460, 736, 0, + 0, 483, 428, 518, 471, 524, 505, 532, 477, 472, + 318, 506, 365, 441, 334, 336, 726, 367, 370, 374, + 375, 450, 451, 465, 490, 509, 510, 511, 364, 348, + 484, 349, 384, 350, 319, 356, 354, 357, 492, 358, + 321, 466, 515, 0, 380, 480, 436, 322, 435, 467, + 514, 513, 335, 541, 548, 549, 639, 0, 554, 737, + 738, 739, 563, 0, 473, 331, 330, 0, 0, 0, + 360, 468, 344, 346, 347, 345, 463, 464, 568, 569, + 570, 572, 0, 573, 574, 0, 0, 0, 0, 575, + 640, 656, 624, 593, 556, 648, 590, 594, 595, 401, + 402, 403, 404, 659, 0, 0, 0, 547, 421, 422, + 0, 372, 371, 437, 323, 0, 0, 410, 400, 474, + 329, 368, 412, 406, 423, 424, 425, 378, 313, 314, + 732, 361, 456, 661, 696, 697, 586, 0, 649, 587, + 596, 353, 621, 633, 632, 452, 546, 0, 644, 647, + 576, 731, 0, 641, 655, 735, 654, 728, 462, 0, + 489, 652, 599, 0, 645, 618, 619, 0, 646, 614, + 650, 0, 588, 0, 557, 560, 589, 674, 675, 676, + 320, 559, 678, 679, 680, 681, 682, 683, 684, 677, + 529, 622, 598, 625, 538, 601, 600, 0, 0, 636, + 555, 637, 638, 446, 447, 448, 449, 382, 662, 342, + 558, 476, 0, 623, 0, 0, 0, 0, 0, 0, + 0, 0, 628, 629, 626, 740, 0, 685, 686, 0, + 0, 552, 553, 377, 0, 571, 385, 341, 461, 379, + 536, 409, 0, 564, 630, 565, 478, 479, 688, 693, + 689, 690, 692, 712, 453, 399, 405, 493, 411, 429, + 481, 535, 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 670, 669, 668, + 667, 666, 665, 664, 663, 0, 0, 612, 512, 355, + 307, 351, 352, 359, 729, 725, 730, 713, 716, 715, + 691, 0, 315, 592, 427, 475, 376, 657, 658, 0, + 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, + 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, + 281, 282, 283, 284, 285, 660, 276, 277, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 298, 299, 0, 0, 0, 0, 309, 717, 718, 719, + 720, 721, 0, 0, 310, 311, 312, 0, 0, 274, + 275, 302, 503, 303, 304, 305, 306, 0, 0, 542, + 543, 544, 567, 0, 545, 527, 591, 386, 316, 507, + 534, 727, 0, 0, 0, 0, 0, 0, 0, 642, + 653, 687, 0, 699, 700, 702, 704, 703, 706, 500, + 501, 714, 0, 0, 708, 709, 710, 707, 431, 487, + 508, 494, 0, 733, 582, 583, 734, 695, 317, 458, + 0, 0, 597, 631, 620, 705, 585, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, + 426, 635, 616, 627, 617, 602, 603, 604, 611, 381, + 605, 606, 607, 577, 608, 578, 609, 610, 0, 634, + 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 492, 521, 0, 534, 0, 406, 407, 0, - 0, 0, 0, 0, 0, 0, 324, 499, 518, 338, - 486, 532, 343, 494, 511, 333, 452, 483, 0, 0, - 326, 516, 493, 434, 325, 0, 477, 366, 383, 363, - 450, 0, 0, 515, 545, 362, 535, 0, 526, 328, - 0, 525, 449, 512, 517, 435, 428, 0, 327, 514, - 433, 427, 412, 373, 561, 413, 414, 387, 464, 425, - 465, 388, 439, 438, 440, 389, 390, 391, 392, 393, - 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, - 556, 557, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 689, 0, 0, - 693, 0, 528, 0, 0, 1753, 0, 0, 0, 497, - 0, 0, 415, 0, 0, 0, 546, 0, 480, 455, - 731, 0, 0, 478, 423, 513, 466, 519, 500, 527, - 472, 467, 318, 501, 365, 436, 334, 336, 721, 367, - 370, 374, 375, 445, 446, 460, 485, 504, 505, 506, - 364, 348, 479, 349, 384, 350, 319, 356, 354, 357, - 487, 358, 321, 461, 510, 0, 380, 475, 431, 322, - 430, 462, 509, 508, 335, 536, 543, 544, 634, 0, - 549, 732, 733, 734, 558, 0, 468, 331, 330, 0, - 0, 0, 360, 463, 344, 346, 347, 345, 458, 459, - 563, 564, 565, 567, 0, 568, 569, 0, 0, 0, - 0, 570, 635, 651, 619, 588, 551, 643, 585, 589, - 590, 401, 402, 403, 654, 0, 0, 0, 542, 416, - 417, 0, 372, 371, 432, 323, 0, 0, 409, 400, - 469, 329, 368, 411, 405, 418, 419, 420, 378, 313, - 314, 727, 361, 451, 656, 691, 692, 581, 0, 644, - 582, 591, 353, 616, 628, 627, 447, 541, 0, 639, - 642, 571, 726, 0, 636, 650, 730, 649, 723, 457, - 0, 484, 647, 594, 0, 640, 613, 614, 0, 641, - 609, 645, 0, 583, 0, 552, 555, 584, 669, 670, - 671, 320, 554, 673, 674, 675, 676, 677, 678, 679, - 672, 524, 617, 593, 620, 533, 596, 595, 0, 0, - 631, 550, 632, 633, 441, 442, 443, 444, 382, 657, - 342, 553, 471, 0, 618, 0, 0, 0, 0, 0, - 0, 0, 0, 623, 624, 621, 735, 0, 680, 681, - 0, 0, 547, 548, 377, 0, 566, 385, 341, 456, - 379, 531, 408, 0, 559, 625, 560, 473, 474, 683, - 688, 684, 685, 687, 707, 448, 399, 404, 488, 410, - 424, 476, 530, 454, 481, 339, 520, 490, 429, 610, - 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, + 0, 0, 337, 246, 579, 701, 581, 580, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 497, 526, 0, + 539, 0, 407, 408, 0, 0, 0, 0, 0, 0, + 0, 324, 504, 523, 338, 491, 537, 343, 499, 516, + 333, 457, 488, 0, 0, 326, 521, 498, 439, 325, + 0, 482, 366, 383, 363, 455, 0, 0, 520, 550, + 362, 540, 0, 531, 328, 0, 530, 454, 517, 522, + 440, 433, 0, 327, 519, 438, 432, 413, 373, 566, + 414, 415, 416, 417, 418, 419, 387, 469, 430, 470, + 388, 444, 443, 445, 389, 390, 391, 392, 393, 394, + 395, 396, 397, 398, 0, 0, 0, 0, 0, 561, + 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 694, 0, 756, 698, + 0, 533, 0, 0, 0, 0, 0, 0, 502, 0, + 0, 420, 0, 0, 0, 551, 0, 485, 460, 736, + 0, 0, 483, 428, 518, 471, 524, 505, 532, 477, + 472, 318, 506, 365, 441, 334, 336, 726, 367, 370, + 374, 375, 450, 451, 465, 490, 509, 510, 511, 364, + 348, 484, 349, 384, 350, 319, 356, 354, 357, 492, + 358, 321, 466, 515, 0, 380, 480, 436, 322, 435, + 467, 514, 513, 335, 541, 548, 549, 639, 0, 554, + 737, 738, 739, 563, 0, 473, 331, 330, 0, 0, + 0, 360, 468, 344, 346, 347, 345, 463, 464, 568, + 569, 570, 572, 0, 573, 574, 0, 0, 0, 0, + 575, 640, 656, 624, 593, 556, 648, 590, 594, 595, + 401, 402, 403, 404, 659, 0, 0, 0, 547, 421, + 422, 0, 372, 371, 437, 323, 0, 0, 410, 400, + 474, 329, 368, 412, 406, 423, 424, 425, 378, 313, + 314, 732, 361, 456, 661, 696, 697, 586, 0, 649, + 587, 596, 353, 621, 633, 632, 452, 546, 0, 644, + 647, 576, 731, 0, 641, 655, 735, 654, 728, 462, + 0, 489, 652, 599, 0, 645, 618, 619, 0, 646, + 614, 650, 0, 588, 0, 557, 560, 589, 674, 675, + 676, 320, 559, 678, 679, 680, 681, 682, 683, 684, + 677, 529, 622, 598, 625, 538, 601, 600, 0, 0, + 636, 555, 637, 638, 446, 447, 448, 449, 382, 662, + 342, 558, 476, 0, 623, 0, 0, 0, 0, 0, + 0, 0, 0, 628, 629, 626, 740, 0, 685, 686, + 0, 0, 552, 553, 377, 0, 571, 385, 341, 461, + 379, 536, 409, 0, 564, 630, 565, 478, 479, 688, + 693, 689, 690, 692, 712, 453, 399, 405, 493, 411, + 429, 481, 535, 459, 486, 339, 525, 495, 434, 615, + 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 665, 664, - 663, 662, 661, 660, 659, 658, 0, 0, 607, 507, - 355, 307, 351, 352, 359, 724, 720, 725, 708, 711, - 710, 686, 0, 315, 587, 422, 470, 376, 652, 653, - 0, 706, 259, 260, 261, 262, 263, 264, 265, 266, + 0, 0, 0, 0, 0, 0, 0, 0, 670, 669, + 668, 667, 666, 665, 664, 663, 0, 0, 612, 512, + 355, 307, 351, 352, 359, 729, 725, 730, 713, 716, + 715, 691, 0, 315, 592, 427, 475, 376, 657, 658, + 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, - 280, 281, 282, 283, 284, 285, 655, 276, 277, 286, + 280, 281, 282, 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 298, 299, 0, 0, 0, 0, 309, 712, 713, - 714, 715, 716, 0, 0, 310, 311, 312, 0, 0, - 274, 275, 302, 498, 303, 304, 305, 306, 0, 0, - 537, 538, 539, 562, 0, 540, 522, 586, 386, 316, - 502, 529, 722, 0, 0, 0, 0, 0, 0, 0, - 637, 648, 682, 0, 694, 695, 697, 699, 698, 701, - 495, 496, 709, 0, 0, 703, 704, 705, 702, 426, - 482, 503, 489, 0, 728, 577, 578, 729, 690, 317, - 453, 0, 0, 592, 626, 615, 700, 580, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1069, 369, 0, - 0, 421, 630, 611, 622, 612, 597, 598, 599, 606, - 381, 600, 601, 602, 572, 603, 573, 604, 605, 0, - 629, 579, 491, 437, 0, 646, 0, 0, 0, 0, + 297, 298, 299, 0, 0, 0, 0, 309, 717, 718, + 719, 720, 721, 0, 0, 310, 311, 312, 0, 0, + 274, 275, 302, 503, 303, 304, 305, 306, 0, 0, + 542, 543, 544, 567, 0, 545, 527, 591, 386, 316, + 507, 534, 727, 0, 0, 0, 0, 0, 0, 0, + 642, 653, 687, 0, 699, 700, 702, 704, 703, 706, + 500, 501, 714, 0, 0, 708, 709, 710, 707, 431, + 487, 508, 494, 0, 733, 582, 583, 734, 695, 317, + 458, 0, 0, 597, 631, 620, 705, 585, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, + 0, 426, 635, 616, 627, 617, 602, 603, 604, 611, + 381, 605, 606, 607, 577, 608, 578, 609, 610, 0, + 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, - 0, 0, 0, 337, 246, 574, 696, 576, 575, 0, + 0, 0, 0, 337, 246, 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 492, 521, - 0, 534, 0, 406, 407, 0, 0, 0, 0, 0, - 0, 0, 324, 499, 518, 338, 486, 532, 343, 494, - 511, 333, 452, 483, 0, 0, 326, 516, 493, 434, - 325, 0, 477, 366, 383, 363, 450, 0, 0, 515, - 545, 362, 535, 0, 526, 328, 0, 525, 449, 512, - 517, 435, 428, 0, 327, 514, 433, 427, 412, 373, - 561, 413, 414, 387, 464, 425, 465, 388, 439, 438, - 440, 389, 390, 391, 392, 393, 394, 395, 396, 397, - 398, 0, 0, 0, 0, 0, 556, 557, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 689, 0, 0, 693, 0, 528, 0, - 0, 0, 0, 0, 0, 497, 0, 0, 415, 0, - 0, 0, 546, 0, 480, 455, 731, 0, 0, 478, - 423, 513, 466, 519, 500, 527, 472, 467, 318, 501, - 365, 436, 334, 336, 721, 367, 370, 374, 375, 445, - 446, 460, 485, 504, 505, 506, 364, 348, 479, 349, - 384, 350, 319, 356, 354, 357, 487, 358, 321, 461, - 510, 0, 380, 475, 431, 322, 430, 462, 509, 508, - 335, 536, 543, 544, 634, 0, 549, 732, 733, 734, - 558, 0, 468, 331, 330, 0, 0, 0, 360, 463, - 344, 346, 347, 345, 458, 459, 563, 564, 565, 567, - 0, 568, 569, 0, 0, 0, 0, 570, 635, 651, - 619, 588, 551, 643, 585, 589, 590, 401, 402, 403, - 654, 0, 0, 0, 542, 416, 417, 0, 372, 371, - 432, 323, 0, 0, 409, 400, 469, 329, 368, 411, - 405, 418, 419, 420, 378, 313, 314, 727, 361, 451, - 656, 691, 692, 581, 0, 644, 582, 591, 353, 616, - 628, 627, 447, 541, 0, 639, 642, 571, 726, 0, - 636, 650, 730, 649, 723, 457, 0, 484, 647, 594, - 0, 640, 613, 614, 0, 641, 609, 645, 0, 583, - 0, 552, 555, 584, 669, 670, 671, 320, 554, 673, - 674, 675, 676, 677, 678, 679, 672, 524, 617, 593, - 620, 533, 596, 595, 0, 0, 631, 550, 632, 633, - 441, 442, 443, 444, 382, 657, 342, 553, 471, 0, - 618, 0, 0, 0, 0, 0, 0, 0, 0, 623, - 624, 621, 735, 0, 680, 681, 0, 0, 547, 548, - 377, 0, 566, 385, 341, 456, 379, 531, 408, 0, - 559, 625, 560, 473, 474, 683, 688, 684, 685, 687, - 707, 448, 399, 404, 488, 410, 424, 476, 530, 454, - 481, 339, 520, 490, 429, 610, 638, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 665, 664, 663, 662, 661, 660, - 659, 658, 0, 0, 607, 507, 355, 307, 351, 352, - 359, 724, 720, 725, 708, 711, 710, 686, 0, 315, - 587, 422, 470, 376, 652, 653, 0, 706, 259, 260, - 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, - 270, 271, 272, 273, 278, 279, 280, 281, 282, 283, - 284, 285, 655, 276, 277, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, - 0, 0, 0, 309, 712, 713, 714, 715, 716, 0, - 0, 310, 311, 312, 0, 0, 274, 275, 302, 498, - 303, 304, 305, 306, 0, 0, 537, 538, 539, 562, - 0, 540, 522, 586, 386, 316, 502, 529, 722, 0, - 0, 0, 0, 0, 0, 0, 637, 648, 682, 0, - 694, 695, 697, 699, 698, 701, 495, 496, 709, 0, - 0, 703, 704, 705, 702, 426, 482, 503, 489, 0, - 728, 577, 578, 729, 690, 317, 453, 0, 0, 592, - 626, 615, 700, 580, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 369, 0, 0, 421, 630, 611, - 622, 612, 597, 598, 599, 606, 381, 600, 601, 602, - 572, 603, 573, 604, 605, 0, 629, 579, 491, 437, - 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 497, 526, + 0, 539, 0, 407, 408, 0, 0, 0, 0, 0, + 0, 0, 324, 504, 523, 338, 491, 537, 343, 499, + 516, 333, 457, 488, 0, 0, 326, 521, 498, 439, + 325, 0, 482, 366, 383, 363, 455, 0, 0, 520, + 550, 362, 540, 0, 531, 328, 0, 530, 454, 517, + 522, 440, 433, 0, 327, 519, 438, 432, 413, 373, + 566, 414, 415, 416, 417, 418, 419, 387, 469, 430, + 470, 388, 444, 443, 445, 389, 390, 391, 392, 393, + 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, + 561, 562, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 694, 0, 0, + 698, 0, 533, 0, 0, 0, 0, 0, 0, 502, + 0, 0, 420, 0, 0, 0, 551, 0, 485, 460, + 736, 0, 0, 483, 428, 518, 471, 524, 505, 532, + 477, 472, 318, 506, 365, 441, 334, 336, 726, 367, + 370, 374, 375, 450, 451, 465, 490, 509, 510, 511, + 364, 348, 484, 349, 384, 350, 319, 356, 354, 357, + 492, 358, 321, 466, 515, 0, 380, 480, 436, 322, + 435, 467, 514, 513, 335, 541, 548, 549, 639, 0, + 554, 737, 738, 739, 563, 0, 473, 331, 330, 0, + 0, 0, 360, 468, 344, 346, 347, 345, 463, 464, + 568, 569, 570, 572, 0, 573, 574, 0, 0, 0, + 0, 575, 640, 656, 624, 593, 556, 648, 590, 594, + 595, 401, 402, 403, 404, 659, 0, 0, 0, 547, + 421, 422, 0, 372, 371, 437, 323, 0, 0, 410, + 400, 474, 329, 368, 412, 406, 423, 424, 425, 378, + 313, 314, 732, 361, 456, 661, 696, 697, 586, 0, + 649, 587, 596, 353, 621, 633, 632, 452, 546, 0, + 644, 647, 576, 731, 0, 641, 655, 735, 654, 728, + 462, 0, 489, 652, 599, 0, 645, 618, 619, 0, + 646, 614, 650, 0, 588, 0, 557, 560, 589, 674, + 675, 676, 320, 559, 678, 679, 680, 681, 682, 683, + 684, 677, 529, 622, 598, 625, 538, 601, 600, 0, + 0, 636, 555, 637, 638, 446, 447, 448, 449, 382, + 662, 342, 558, 476, 0, 623, 0, 0, 0, 0, + 0, 0, 0, 0, 628, 629, 626, 740, 0, 685, + 686, 0, 0, 552, 553, 377, 0, 571, 385, 341, + 461, 379, 536, 409, 0, 564, 630, 565, 478, 479, + 688, 693, 689, 690, 692, 712, 453, 399, 405, 493, + 411, 429, 481, 535, 459, 486, 339, 525, 495, 434, + 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 670, + 669, 668, 667, 666, 665, 664, 663, 1078, 0, 612, + 512, 355, 307, 351, 352, 359, 729, 725, 730, 713, + 716, 715, 691, 0, 315, 592, 427, 475, 376, 657, + 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, + 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, + 279, 280, 281, 282, 283, 284, 285, 660, 276, 277, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 298, 299, 0, 0, 0, 0, 309, 717, + 718, 719, 720, 721, 0, 0, 310, 311, 312, 0, + 0, 274, 275, 302, 503, 303, 304, 305, 306, 0, + 0, 542, 543, 544, 567, 0, 545, 527, 591, 386, + 316, 507, 534, 727, 0, 0, 0, 0, 0, 0, + 0, 642, 653, 687, 0, 699, 700, 702, 704, 703, + 706, 500, 501, 714, 0, 0, 708, 709, 710, 707, + 431, 487, 508, 494, 0, 733, 582, 583, 734, 695, + 317, 458, 0, 0, 597, 631, 620, 705, 585, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, + 0, 0, 426, 635, 616, 627, 617, 602, 603, 604, + 611, 381, 605, 606, 607, 577, 608, 578, 609, 610, + 0, 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 245, 0, 0, 0, 0, 0, 0, 337, - 246, 574, 696, 576, 575, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, + 0, 0, 0, 0, 337, 246, 579, 701, 581, 580, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 492, 521, 0, 534, 0, 406, - 407, 0, 0, 0, 0, 0, 0, 0, 324, 499, - 518, 338, 486, 532, 343, 494, 511, 333, 452, 483, - 0, 0, 326, 516, 493, 434, 325, 0, 477, 366, - 383, 363, 450, 0, 0, 515, 545, 362, 535, 0, - 526, 328, 0, 525, 449, 512, 517, 435, 428, 0, - 327, 514, 433, 427, 412, 373, 561, 413, 414, 387, - 464, 425, 465, 388, 439, 438, 440, 389, 390, 391, - 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, - 0, 0, 556, 557, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, - 0, 751, 693, 0, 528, 0, 0, 0, 0, 0, - 0, 497, 0, 0, 415, 0, 0, 0, 546, 0, - 480, 455, 731, 0, 0, 478, 423, 513, 466, 519, - 500, 527, 472, 467, 318, 501, 365, 436, 334, 336, - 721, 367, 370, 374, 375, 445, 446, 460, 485, 504, - 505, 506, 364, 348, 479, 349, 384, 350, 319, 356, - 354, 357, 487, 358, 321, 461, 510, 0, 380, 475, - 431, 322, 430, 462, 509, 508, 335, 536, 543, 544, - 634, 0, 549, 732, 733, 734, 558, 0, 468, 331, - 330, 0, 0, 0, 360, 463, 344, 346, 347, 345, - 458, 459, 563, 564, 565, 567, 0, 568, 569, 0, - 0, 0, 0, 570, 635, 651, 619, 588, 551, 643, - 585, 589, 590, 401, 402, 403, 654, 0, 0, 0, - 542, 416, 417, 0, 372, 371, 432, 323, 0, 0, - 409, 400, 469, 329, 368, 411, 405, 418, 419, 420, - 378, 313, 314, 727, 361, 451, 656, 691, 692, 581, - 0, 644, 582, 591, 353, 616, 628, 627, 447, 541, - 0, 639, 642, 571, 726, 0, 636, 650, 730, 649, - 723, 457, 0, 484, 647, 594, 0, 640, 613, 614, - 0, 641, 609, 645, 0, 583, 0, 552, 555, 584, - 669, 670, 671, 320, 554, 673, 674, 675, 676, 677, - 678, 679, 672, 524, 617, 593, 620, 533, 596, 595, - 0, 0, 631, 550, 632, 633, 441, 442, 443, 444, - 382, 657, 342, 553, 471, 0, 618, 0, 0, 0, - 0, 0, 0, 0, 0, 623, 624, 621, 735, 0, - 680, 681, 0, 0, 547, 548, 377, 0, 566, 385, - 341, 456, 379, 531, 408, 0, 559, 625, 560, 473, - 474, 683, 688, 684, 685, 687, 707, 448, 399, 404, - 488, 410, 424, 476, 530, 454, 481, 339, 520, 490, - 429, 610, 638, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, + 526, 0, 539, 0, 407, 408, 0, 0, 0, 0, + 0, 0, 0, 324, 504, 523, 338, 491, 537, 343, + 499, 516, 333, 457, 488, 0, 0, 326, 521, 498, + 439, 325, 0, 482, 366, 383, 363, 455, 0, 0, + 520, 550, 362, 540, 0, 531, 328, 0, 530, 454, + 517, 522, 440, 433, 0, 327, 519, 438, 432, 413, + 373, 566, 414, 415, 416, 417, 418, 419, 387, 469, + 430, 470, 388, 444, 443, 445, 389, 390, 391, 392, + 393, 394, 395, 396, 397, 398, 0, 0, 0, 0, + 0, 561, 562, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 694, 0, + 0, 698, 0, 533, 0, 0, 0, 0, 0, 0, + 502, 0, 0, 420, 0, 0, 0, 551, 0, 485, + 460, 736, 0, 0, 483, 428, 518, 471, 524, 505, + 532, 477, 472, 318, 506, 365, 441, 334, 336, 726, + 367, 370, 374, 375, 450, 451, 465, 490, 509, 510, + 511, 364, 348, 484, 349, 384, 350, 319, 356, 354, + 357, 492, 358, 321, 466, 515, 0, 380, 480, 436, + 322, 435, 467, 514, 513, 335, 541, 548, 549, 639, + 0, 554, 737, 738, 739, 563, 0, 473, 331, 330, + 0, 0, 0, 360, 468, 344, 346, 347, 345, 463, + 464, 568, 569, 570, 572, 0, 573, 574, 0, 0, + 0, 0, 575, 640, 656, 624, 593, 556, 648, 590, + 594, 595, 401, 402, 403, 404, 659, 0, 0, 0, + 547, 421, 422, 0, 372, 371, 437, 323, 0, 0, + 410, 400, 474, 329, 368, 412, 406, 423, 424, 425, + 378, 313, 314, 732, 361, 456, 661, 696, 697, 586, + 0, 649, 587, 596, 353, 621, 633, 632, 452, 546, + 0, 644, 647, 576, 731, 0, 641, 655, 735, 654, + 728, 462, 0, 489, 652, 599, 0, 645, 618, 619, + 0, 646, 614, 650, 0, 588, 0, 557, 560, 589, + 674, 675, 676, 320, 559, 678, 679, 680, 681, 682, + 683, 684, 677, 529, 622, 598, 625, 538, 601, 600, + 0, 0, 636, 555, 637, 638, 446, 447, 448, 449, + 382, 662, 342, 558, 476, 0, 623, 0, 0, 0, + 0, 0, 0, 0, 0, 628, 629, 626, 740, 0, + 685, 686, 0, 0, 552, 553, 377, 0, 571, 385, + 341, 461, 379, 536, 409, 0, 564, 630, 565, 478, + 479, 688, 693, 689, 690, 692, 712, 453, 399, 405, + 493, 411, 429, 481, 535, 459, 486, 339, 525, 495, + 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 665, 664, 663, 662, 661, 660, 659, 658, 0, 0, - 607, 507, 355, 307, 351, 352, 359, 724, 720, 725, - 708, 711, 710, 686, 0, 315, 587, 422, 470, 376, - 652, 653, 0, 706, 259, 260, 261, 262, 263, 264, + 670, 669, 668, 667, 666, 665, 664, 663, 0, 0, + 612, 512, 355, 307, 351, 352, 359, 729, 725, 730, + 713, 716, 715, 691, 0, 315, 592, 427, 475, 376, + 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, - 278, 279, 280, 281, 282, 283, 284, 285, 655, 276, + 278, 279, 280, 281, 282, 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, 0, 309, - 712, 713, 714, 715, 716, 0, 0, 310, 311, 312, - 0, 0, 274, 275, 302, 498, 303, 304, 305, 306, - 0, 0, 537, 538, 539, 562, 0, 540, 522, 586, - 386, 316, 502, 529, 722, 0, 0, 0, 0, 0, - 0, 0, 637, 648, 682, 0, 694, 695, 697, 699, - 698, 701, 495, 496, 709, 0, 0, 703, 704, 705, - 702, 426, 482, 503, 489, 0, 728, 577, 578, 729, - 690, 317, 453, 0, 0, 592, 626, 615, 700, 580, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 369, 0, 0, 421, 630, 611, 622, 612, 597, 598, - 599, 606, 381, 600, 601, 602, 572, 603, 573, 604, - 605, 0, 629, 579, 491, 437, 0, 646, 0, 0, + 717, 718, 719, 720, 721, 0, 0, 310, 311, 312, + 0, 0, 274, 275, 302, 503, 303, 304, 305, 306, + 0, 0, 542, 543, 544, 567, 0, 545, 527, 591, + 386, 316, 507, 534, 727, 0, 0, 0, 0, 0, + 0, 0, 642, 653, 687, 0, 699, 700, 702, 704, + 703, 706, 500, 501, 714, 0, 0, 708, 709, 710, + 707, 431, 487, 508, 494, 0, 733, 582, 583, 734, + 695, 317, 458, 0, 0, 597, 631, 620, 705, 585, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 369, 0, 0, 426, 635, 616, 627, 617, 602, 603, + 604, 611, 381, 605, 606, 607, 577, 608, 578, 609, + 610, 0, 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 0, 0, 0, 0, 337, 246, 574, 696, 576, - 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 337, 246, 579, 701, 581, + 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 492, 521, 0, 534, 0, 406, 407, 0, 0, 0, - 0, 0, 0, 0, 324, 499, 518, 338, 486, 532, - 343, 494, 511, 333, 452, 483, 0, 0, 326, 516, - 493, 434, 325, 0, 477, 366, 383, 363, 450, 0, - 0, 515, 545, 362, 535, 0, 526, 328, 0, 525, - 449, 512, 517, 435, 428, 0, 327, 514, 433, 427, - 412, 373, 561, 413, 414, 387, 464, 425, 465, 388, - 439, 438, 440, 389, 390, 391, 392, 393, 394, 395, - 396, 397, 398, 0, 0, 0, 0, 0, 556, 557, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 689, 0, 0, 693, 0, - 528, 0, 0, 0, 0, 0, 0, 497, 0, 0, - 415, 0, 0, 0, 546, 0, 480, 455, 731, 0, - 0, 478, 423, 513, 466, 519, 500, 527, 472, 467, - 318, 501, 365, 436, 334, 336, 721, 367, 370, 374, - 375, 445, 446, 460, 485, 504, 505, 506, 364, 348, - 479, 349, 384, 350, 319, 356, 354, 357, 487, 358, - 321, 461, 510, 0, 380, 475, 431, 322, 430, 462, - 509, 508, 335, 536, 543, 544, 634, 0, 549, 732, - 733, 734, 558, 0, 468, 331, 330, 0, 0, 0, - 360, 463, 344, 346, 347, 345, 458, 459, 563, 564, - 565, 567, 0, 568, 569, 0, 0, 0, 0, 570, - 635, 651, 619, 588, 551, 643, 585, 589, 590, 401, - 402, 403, 654, 0, 0, 0, 542, 416, 417, 0, - 372, 371, 432, 323, 0, 0, 409, 400, 469, 329, - 368, 411, 405, 418, 419, 420, 378, 313, 314, 727, - 361, 451, 656, 691, 692, 581, 0, 644, 582, 591, - 353, 616, 628, 627, 447, 541, 0, 639, 642, 571, - 726, 0, 636, 650, 730, 649, 723, 457, 0, 484, - 647, 594, 0, 640, 613, 614, 0, 641, 609, 645, - 0, 583, 0, 552, 555, 584, 669, 670, 671, 320, - 554, 673, 674, 675, 676, 677, 678, 679, 672, 524, - 617, 593, 620, 533, 596, 595, 0, 0, 631, 550, - 632, 633, 441, 442, 443, 444, 382, 657, 342, 553, - 471, 0, 618, 0, 0, 0, 0, 0, 0, 0, - 0, 623, 624, 621, 735, 0, 680, 681, 0, 0, - 547, 548, 377, 0, 566, 385, 341, 456, 379, 531, - 408, 0, 559, 625, 560, 473, 474, 683, 688, 684, - 685, 687, 707, 448, 399, 404, 488, 410, 424, 476, - 530, 454, 481, 339, 520, 490, 429, 610, 638, 0, + 497, 526, 0, 539, 0, 407, 408, 0, 0, 0, + 0, 0, 0, 0, 324, 504, 523, 338, 491, 537, + 343, 499, 516, 333, 457, 488, 0, 0, 326, 521, + 498, 439, 325, 0, 482, 366, 383, 363, 455, 0, + 0, 520, 550, 362, 540, 0, 531, 328, 0, 530, + 454, 517, 522, 440, 433, 0, 327, 519, 438, 432, + 413, 373, 566, 414, 415, 416, 417, 418, 419, 387, + 469, 430, 470, 388, 444, 443, 445, 389, 390, 391, + 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, + 0, 0, 561, 562, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, + 0, 0, 698, 0, 533, 0, 0, 0, 0, 0, + 0, 502, 0, 0, 420, 0, 0, 0, 551, 0, + 485, 460, 736, 0, 0, 483, 428, 518, 471, 524, + 505, 532, 477, 472, 318, 506, 365, 441, 334, 336, + 726, 367, 370, 374, 375, 450, 451, 465, 490, 509, + 510, 511, 364, 348, 484, 349, 384, 350, 319, 356, + 354, 357, 492, 358, 321, 466, 515, 0, 380, 3564, + 436, 322, 435, 467, 514, 513, 335, 541, 548, 549, + 639, 0, 554, 737, 738, 739, 563, 0, 473, 331, + 330, 0, 0, 0, 360, 468, 344, 346, 347, 345, + 463, 464, 568, 569, 570, 572, 0, 573, 574, 0, + 0, 0, 0, 575, 640, 656, 624, 593, 556, 648, + 590, 594, 595, 401, 402, 403, 404, 659, 0, 0, + 0, 547, 421, 422, 0, 372, 371, 437, 323, 0, + 0, 410, 400, 474, 329, 368, 412, 406, 423, 424, + 425, 378, 313, 314, 732, 361, 456, 661, 696, 697, + 586, 0, 649, 587, 596, 353, 621, 633, 632, 452, + 546, 0, 644, 647, 576, 731, 0, 641, 655, 735, + 654, 728, 462, 0, 489, 652, 599, 0, 645, 618, + 619, 0, 646, 614, 650, 0, 588, 0, 557, 560, + 589, 674, 675, 676, 320, 559, 678, 679, 680, 681, + 682, 683, 684, 677, 529, 622, 598, 625, 538, 601, + 600, 0, 0, 636, 555, 637, 638, 446, 447, 448, + 449, 382, 662, 342, 558, 476, 0, 623, 0, 0, + 0, 0, 0, 0, 0, 0, 628, 629, 626, 740, + 0, 685, 686, 0, 0, 552, 553, 377, 0, 571, + 385, 341, 461, 379, 536, 409, 0, 564, 630, 565, + 478, 479, 688, 693, 689, 690, 692, 712, 453, 399, + 405, 493, 411, 429, 481, 535, 459, 486, 339, 525, + 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 665, 664, 663, 662, - 661, 660, 659, 658, 1072, 0, 607, 507, 355, 307, - 351, 352, 359, 724, 720, 725, 708, 711, 710, 686, - 0, 315, 587, 422, 470, 376, 652, 653, 0, 706, - 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, - 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, - 282, 283, 284, 285, 655, 276, 277, 286, 287, 288, - 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, - 299, 0, 0, 0, 0, 309, 712, 713, 714, 715, - 716, 0, 0, 310, 311, 312, 0, 0, 274, 275, - 302, 498, 303, 304, 305, 306, 0, 0, 537, 538, - 539, 562, 0, 540, 522, 586, 386, 316, 502, 529, - 722, 0, 0, 0, 0, 0, 0, 0, 637, 648, - 682, 0, 694, 695, 697, 699, 698, 701, 495, 496, - 709, 0, 0, 703, 704, 705, 702, 426, 482, 503, - 489, 0, 728, 577, 578, 729, 690, 317, 453, 0, - 0, 592, 626, 615, 700, 580, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 369, 0, 0, 421, - 630, 611, 622, 612, 597, 598, 599, 606, 381, 600, - 601, 602, 572, 603, 573, 604, 605, 0, 629, 579, - 491, 437, 0, 646, 0, 0, 0, 0, 0, 0, + 0, 670, 669, 668, 667, 666, 665, 664, 663, 0, + 0, 612, 512, 355, 307, 351, 352, 359, 729, 725, + 730, 713, 716, 715, 691, 0, 315, 592, 427, 475, + 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, + 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, + 273, 278, 279, 280, 281, 282, 283, 284, 285, 660, + 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 298, 299, 0, 0, 0, 0, + 309, 717, 718, 719, 720, 721, 0, 0, 310, 311, + 312, 0, 0, 274, 275, 302, 503, 303, 304, 305, + 306, 0, 0, 542, 543, 544, 567, 0, 545, 527, + 591, 386, 316, 507, 534, 727, 0, 0, 0, 0, + 0, 0, 0, 642, 653, 687, 0, 699, 700, 702, + 704, 703, 706, 500, 501, 714, 0, 0, 708, 709, + 710, 707, 431, 487, 508, 494, 0, 733, 582, 583, + 734, 695, 317, 458, 0, 0, 597, 631, 620, 705, + 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 369, 0, 0, 426, 635, 616, 627, 617, 602, + 603, 604, 611, 381, 605, 606, 607, 577, 608, 578, + 609, 610, 0, 634, 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, - 0, 337, 246, 574, 696, 576, 575, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, + 0, 0, 0, 0, 0, 0, 337, 246, 579, 701, + 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 492, 521, 0, 534, - 0, 406, 407, 0, 0, 0, 0, 0, 0, 0, - 324, 499, 518, 338, 486, 532, 343, 494, 511, 333, - 452, 483, 0, 0, 326, 516, 493, 434, 325, 0, - 477, 366, 383, 363, 450, 0, 0, 515, 545, 362, - 535, 0, 526, 328, 0, 525, 449, 512, 517, 435, - 428, 0, 327, 514, 433, 427, 412, 373, 561, 413, - 414, 387, 464, 425, 465, 388, 439, 438, 440, 389, - 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, - 0, 0, 0, 0, 556, 557, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 689, 0, 0, 693, 0, 528, 0, 0, 0, - 0, 0, 0, 497, 0, 0, 415, 0, 0, 0, - 546, 0, 480, 455, 731, 0, 0, 478, 423, 513, - 466, 519, 500, 527, 472, 467, 318, 501, 365, 436, - 334, 336, 721, 367, 370, 374, 375, 445, 446, 460, - 485, 504, 505, 506, 364, 348, 479, 349, 384, 350, - 319, 356, 354, 357, 487, 358, 321, 461, 510, 0, - 380, 475, 431, 322, 430, 462, 509, 508, 335, 536, - 543, 544, 634, 0, 549, 732, 733, 734, 558, 0, - 468, 331, 330, 0, 0, 0, 360, 463, 344, 346, - 347, 345, 458, 459, 563, 564, 565, 567, 0, 568, - 569, 0, 0, 0, 0, 570, 635, 651, 619, 588, - 551, 643, 585, 589, 590, 401, 402, 403, 654, 0, - 0, 0, 542, 416, 417, 0, 372, 371, 432, 323, - 0, 0, 409, 400, 469, 329, 368, 411, 405, 418, - 419, 420, 378, 313, 314, 727, 361, 451, 656, 691, - 692, 581, 0, 644, 582, 591, 353, 616, 628, 627, - 447, 541, 0, 639, 642, 571, 726, 0, 636, 650, - 730, 649, 723, 457, 0, 484, 647, 594, 0, 640, - 613, 614, 0, 641, 609, 645, 0, 583, 0, 552, - 555, 584, 669, 670, 671, 320, 554, 673, 674, 675, - 676, 677, 678, 679, 672, 524, 617, 593, 620, 533, - 596, 595, 0, 0, 631, 550, 632, 633, 441, 442, - 443, 444, 382, 657, 342, 553, 471, 0, 618, 0, - 0, 0, 0, 0, 0, 0, 0, 623, 624, 621, - 735, 0, 680, 681, 0, 0, 547, 548, 377, 0, - 566, 385, 341, 456, 379, 531, 408, 0, 559, 625, - 560, 473, 474, 683, 688, 684, 685, 687, 707, 448, - 399, 404, 488, 410, 424, 476, 530, 454, 481, 339, - 520, 490, 429, 610, 638, 0, 0, 0, 0, 0, + 0, 497, 526, 0, 539, 0, 407, 408, 0, 0, + 0, 0, 0, 0, 0, 324, 504, 523, 338, 491, + 537, 343, 499, 2124, 333, 457, 488, 0, 0, 326, + 521, 498, 439, 325, 0, 482, 366, 383, 363, 455, + 0, 0, 520, 550, 362, 540, 0, 531, 328, 0, + 530, 454, 517, 522, 440, 433, 0, 327, 519, 438, + 432, 413, 373, 566, 414, 415, 416, 417, 418, 419, + 387, 469, 430, 470, 388, 444, 443, 445, 389, 390, + 391, 392, 393, 394, 395, 396, 397, 398, 0, 0, + 0, 0, 0, 561, 562, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 694, 0, 0, 698, 0, 533, 0, 0, 0, 0, + 0, 0, 502, 0, 0, 420, 0, 0, 0, 551, + 0, 485, 460, 736, 0, 0, 483, 428, 518, 471, + 524, 505, 532, 477, 472, 318, 506, 365, 441, 334, + 336, 726, 367, 370, 374, 375, 450, 451, 465, 490, + 509, 510, 511, 364, 348, 484, 349, 384, 350, 319, + 356, 354, 357, 492, 358, 321, 466, 515, 0, 380, + 480, 436, 322, 435, 467, 514, 513, 335, 541, 548, + 549, 639, 0, 554, 737, 738, 739, 563, 0, 473, + 331, 330, 0, 0, 0, 360, 468, 344, 346, 347, + 345, 463, 464, 568, 569, 570, 572, 0, 573, 574, + 0, 0, 0, 0, 575, 640, 656, 624, 593, 556, + 648, 590, 594, 595, 401, 402, 403, 404, 659, 0, + 0, 0, 547, 421, 422, 0, 372, 371, 437, 323, + 0, 0, 410, 400, 474, 329, 368, 412, 406, 423, + 424, 425, 378, 313, 314, 732, 361, 456, 661, 696, + 697, 586, 0, 649, 587, 596, 353, 621, 633, 632, + 452, 546, 0, 644, 647, 576, 731, 0, 641, 655, + 735, 654, 728, 462, 0, 489, 652, 599, 0, 645, + 618, 619, 0, 646, 614, 650, 0, 588, 0, 557, + 560, 589, 674, 675, 676, 320, 559, 678, 679, 680, + 681, 682, 683, 684, 677, 529, 622, 598, 625, 538, + 601, 600, 0, 0, 636, 555, 637, 638, 446, 447, + 448, 449, 382, 662, 342, 558, 476, 0, 623, 0, + 0, 0, 0, 0, 0, 0, 0, 628, 629, 626, + 740, 0, 685, 686, 0, 0, 552, 553, 377, 0, + 571, 385, 341, 461, 379, 536, 409, 0, 564, 630, + 565, 478, 479, 688, 693, 689, 690, 692, 712, 453, + 399, 405, 493, 411, 429, 481, 535, 459, 486, 339, + 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 665, 664, 663, 662, 661, 660, 659, 658, - 0, 0, 607, 507, 355, 307, 351, 352, 359, 724, - 720, 725, 708, 711, 710, 686, 0, 315, 587, 422, - 470, 376, 652, 653, 0, 706, 259, 260, 261, 262, + 0, 0, 670, 669, 668, 667, 666, 665, 664, 663, + 0, 0, 612, 512, 355, 307, 351, 352, 359, 729, + 725, 730, 713, 716, 715, 691, 0, 315, 592, 427, + 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, 283, 284, 285, - 655, 276, 277, 286, 287, 288, 289, 290, 291, 292, + 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, - 0, 309, 712, 713, 714, 715, 716, 0, 0, 310, - 311, 312, 0, 0, 274, 275, 302, 498, 303, 304, - 305, 306, 0, 0, 537, 538, 539, 562, 0, 540, - 522, 586, 386, 316, 502, 529, 722, 0, 0, 0, - 0, 0, 0, 0, 637, 648, 682, 0, 694, 695, - 697, 699, 698, 701, 495, 496, 709, 0, 0, 703, - 704, 705, 702, 426, 482, 503, 489, 0, 728, 577, - 578, 729, 690, 317, 453, 0, 0, 592, 626, 615, - 700, 580, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 369, 0, 0, 421, 630, 611, 622, 612, - 597, 598, 599, 606, 381, 600, 601, 602, 572, 603, - 573, 604, 605, 0, 629, 579, 491, 437, 0, 646, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 0, 0, 0, 0, 337, 246, 574, - 696, 576, 575, 0, 0, 0, 0, 0, 0, 0, + 0, 309, 717, 718, 719, 720, 721, 0, 0, 310, + 311, 312, 0, 0, 274, 275, 302, 503, 303, 304, + 305, 306, 0, 0, 542, 543, 544, 567, 0, 545, + 527, 591, 386, 316, 507, 534, 727, 0, 0, 0, + 0, 0, 0, 0, 642, 653, 687, 0, 699, 700, + 702, 704, 703, 706, 500, 501, 714, 0, 0, 708, + 709, 710, 707, 431, 487, 508, 494, 0, 733, 582, + 583, 734, 695, 317, 458, 0, 0, 597, 631, 620, + 705, 585, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 369, 0, 0, 426, 635, 616, 627, 617, + 602, 603, 604, 611, 381, 605, 606, 607, 577, 608, + 578, 609, 610, 0, 634, 584, 496, 442, 0, 651, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 245, 0, 0, 0, 0, 0, 0, 337, 246, 579, + 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 492, 521, 0, 534, 0, 406, 407, 0, - 0, 0, 0, 0, 0, 0, 324, 499, 518, 338, - 486, 532, 343, 494, 511, 333, 452, 483, 0, 0, - 326, 516, 493, 434, 325, 0, 477, 366, 383, 363, - 450, 0, 0, 515, 545, 362, 535, 0, 526, 328, - 0, 525, 449, 512, 517, 435, 428, 0, 327, 514, - 433, 427, 412, 373, 561, 413, 414, 387, 464, 425, - 465, 388, 439, 438, 440, 389, 390, 391, 392, 393, - 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, - 556, 557, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 689, 0, 0, - 693, 0, 528, 0, 0, 0, 0, 0, 0, 497, - 0, 0, 415, 0, 0, 0, 546, 0, 480, 455, - 731, 0, 0, 478, 423, 513, 466, 519, 500, 527, - 472, 467, 318, 501, 365, 436, 334, 336, 721, 367, - 370, 374, 375, 445, 446, 460, 485, 504, 505, 506, - 364, 348, 479, 349, 384, 350, 319, 356, 354, 357, - 487, 358, 321, 461, 510, 0, 380, 3540, 431, 322, - 430, 462, 509, 508, 335, 536, 543, 544, 634, 0, - 549, 732, 733, 734, 558, 0, 468, 331, 330, 0, - 0, 0, 360, 463, 344, 346, 347, 345, 458, 459, - 563, 564, 565, 567, 0, 568, 569, 0, 0, 0, - 0, 570, 635, 651, 619, 588, 551, 643, 585, 589, - 590, 401, 402, 403, 654, 0, 0, 0, 542, 416, - 417, 0, 372, 371, 432, 323, 0, 0, 409, 400, - 469, 329, 368, 411, 405, 418, 419, 420, 378, 313, - 314, 727, 361, 451, 656, 691, 692, 581, 0, 644, - 582, 591, 353, 616, 628, 627, 447, 541, 0, 639, - 642, 571, 726, 0, 636, 650, 730, 649, 723, 457, - 0, 484, 647, 594, 0, 640, 613, 614, 0, 641, - 609, 645, 0, 583, 0, 552, 555, 584, 669, 670, - 671, 320, 554, 673, 674, 675, 676, 677, 678, 679, - 672, 524, 617, 593, 620, 533, 596, 595, 0, 0, - 631, 550, 632, 633, 441, 442, 443, 444, 382, 657, - 342, 553, 471, 0, 618, 0, 0, 0, 0, 0, - 0, 0, 0, 623, 624, 621, 735, 0, 680, 681, - 0, 0, 547, 548, 377, 0, 566, 385, 341, 456, - 379, 531, 408, 0, 559, 625, 560, 473, 474, 683, - 688, 684, 685, 687, 707, 448, 399, 404, 488, 410, - 424, 476, 530, 454, 481, 339, 520, 490, 429, 610, - 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 665, 664, - 663, 662, 661, 660, 659, 658, 0, 0, 607, 507, - 355, 307, 351, 352, 359, 724, 720, 725, 708, 711, - 710, 686, 0, 315, 587, 422, 470, 376, 652, 653, - 0, 706, 259, 260, 261, 262, 263, 264, 265, 266, - 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, - 280, 281, 282, 283, 284, 285, 655, 276, 277, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 298, 299, 0, 0, 0, 0, 309, 712, 713, - 714, 715, 716, 0, 0, 310, 311, 312, 0, 0, - 274, 275, 302, 498, 303, 304, 305, 306, 0, 0, - 537, 538, 539, 562, 0, 540, 522, 586, 386, 316, - 502, 529, 722, 0, 0, 0, 0, 0, 0, 0, - 637, 648, 682, 0, 694, 695, 697, 699, 698, 701, - 495, 496, 709, 0, 0, 703, 704, 705, 702, 426, - 482, 503, 489, 0, 728, 577, 578, 729, 690, 317, - 453, 0, 0, 592, 626, 615, 700, 580, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, - 0, 421, 630, 611, 622, 612, 597, 598, 599, 606, - 381, 600, 601, 602, 572, 603, 573, 604, 605, 0, - 629, 579, 491, 437, 0, 646, 0, 0, 0, 0, + 0, 0, 497, 526, 0, 539, 0, 407, 408, 0, + 0, 0, 0, 0, 0, 0, 324, 504, 1713, 338, + 491, 537, 343, 499, 516, 333, 457, 488, 0, 0, + 326, 521, 498, 439, 325, 0, 482, 366, 383, 363, + 455, 0, 0, 520, 550, 362, 540, 0, 531, 328, + 0, 530, 454, 517, 522, 440, 433, 0, 327, 519, + 438, 432, 413, 373, 566, 414, 415, 416, 417, 418, + 419, 387, 469, 430, 470, 388, 444, 443, 445, 389, + 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, + 0, 0, 0, 0, 561, 562, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 694, 0, 0, 698, 0, 533, 0, 0, 0, + 0, 0, 0, 502, 0, 0, 420, 0, 0, 0, + 551, 0, 485, 460, 736, 0, 0, 483, 428, 518, + 471, 524, 505, 532, 477, 472, 318, 506, 365, 441, + 334, 336, 726, 367, 370, 374, 375, 450, 451, 465, + 490, 509, 510, 511, 364, 348, 484, 349, 384, 350, + 319, 356, 354, 357, 492, 358, 321, 466, 515, 0, + 380, 480, 436, 322, 435, 467, 514, 513, 335, 541, + 548, 549, 639, 0, 554, 737, 738, 739, 563, 0, + 473, 331, 330, 0, 0, 0, 360, 468, 344, 346, + 347, 345, 463, 464, 568, 569, 570, 572, 0, 573, + 574, 0, 0, 0, 0, 575, 640, 656, 624, 593, + 556, 648, 590, 594, 595, 401, 402, 403, 404, 659, + 0, 0, 0, 547, 421, 422, 0, 372, 371, 437, + 323, 0, 0, 410, 400, 474, 329, 368, 412, 406, + 423, 424, 425, 378, 313, 314, 732, 361, 456, 661, + 696, 697, 586, 0, 649, 587, 596, 353, 621, 633, + 632, 452, 546, 0, 644, 647, 576, 731, 0, 641, + 655, 735, 654, 728, 462, 0, 489, 652, 599, 0, + 645, 618, 619, 0, 646, 614, 650, 0, 588, 0, + 557, 560, 589, 674, 675, 676, 320, 559, 678, 679, + 680, 681, 682, 683, 684, 677, 529, 622, 598, 625, + 538, 601, 600, 0, 0, 636, 555, 637, 638, 446, + 447, 448, 449, 382, 662, 342, 558, 476, 0, 623, + 0, 0, 0, 0, 0, 0, 0, 0, 628, 629, + 626, 740, 0, 685, 686, 0, 0, 552, 553, 377, + 0, 571, 385, 341, 461, 379, 536, 409, 0, 564, + 630, 565, 478, 479, 688, 693, 689, 690, 692, 712, + 453, 399, 405, 493, 411, 429, 481, 535, 459, 486, + 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, - 0, 0, 0, 337, 246, 574, 696, 576, 575, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, + 0, 0, 0, 670, 669, 668, 667, 666, 665, 664, + 663, 0, 0, 612, 512, 355, 307, 351, 352, 359, + 729, 725, 730, 713, 716, 715, 691, 0, 315, 592, + 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, + 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, + 271, 272, 273, 278, 279, 280, 281, 282, 283, 284, + 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 298, 299, 0, 0, + 0, 0, 309, 717, 718, 719, 720, 721, 0, 0, + 310, 311, 312, 0, 0, 274, 275, 302, 503, 303, + 304, 305, 306, 0, 0, 542, 543, 544, 567, 0, + 545, 527, 591, 386, 316, 507, 534, 727, 0, 0, + 0, 0, 0, 0, 0, 642, 653, 687, 0, 699, + 700, 702, 704, 703, 706, 500, 501, 714, 0, 0, + 708, 709, 710, 707, 431, 487, 508, 494, 0, 733, + 582, 583, 734, 695, 317, 458, 0, 0, 597, 631, + 620, 705, 585, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 369, 0, 0, 426, 635, 616, 627, + 617, 602, 603, 604, 611, 381, 605, 606, 607, 577, + 608, 578, 609, 610, 0, 634, 584, 496, 442, 0, + 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 245, 0, 0, 0, 0, 0, 0, 337, 246, + 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 492, 521, - 0, 534, 0, 406, 407, 0, 0, 0, 0, 0, - 0, 0, 324, 499, 518, 338, 486, 532, 343, 494, - 2108, 333, 452, 483, 0, 0, 326, 516, 493, 434, - 325, 0, 477, 366, 383, 363, 450, 0, 0, 515, - 545, 362, 535, 0, 526, 328, 0, 525, 449, 512, - 517, 435, 428, 0, 327, 514, 433, 427, 412, 373, - 561, 413, 414, 387, 464, 425, 465, 388, 439, 438, - 440, 389, 390, 391, 392, 393, 394, 395, 396, 397, - 398, 0, 0, 0, 0, 0, 556, 557, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 689, 0, 0, 693, 0, 528, 0, - 0, 0, 0, 0, 0, 497, 0, 0, 415, 0, - 0, 0, 546, 0, 480, 455, 731, 0, 0, 478, - 423, 513, 466, 519, 500, 527, 472, 467, 318, 501, - 365, 436, 334, 336, 721, 367, 370, 374, 375, 445, - 446, 460, 485, 504, 505, 506, 364, 348, 479, 349, - 384, 350, 319, 356, 354, 357, 487, 358, 321, 461, - 510, 0, 380, 475, 431, 322, 430, 462, 509, 508, - 335, 536, 543, 544, 634, 0, 549, 732, 733, 734, - 558, 0, 468, 331, 330, 0, 0, 0, 360, 463, - 344, 346, 347, 345, 458, 459, 563, 564, 565, 567, - 0, 568, 569, 0, 0, 0, 0, 570, 635, 651, - 619, 588, 551, 643, 585, 589, 590, 401, 402, 403, - 654, 0, 0, 0, 542, 416, 417, 0, 372, 371, - 432, 323, 0, 0, 409, 400, 469, 329, 368, 411, - 405, 418, 419, 420, 378, 313, 314, 727, 361, 451, - 656, 691, 692, 581, 0, 644, 582, 591, 353, 616, - 628, 627, 447, 541, 0, 639, 642, 571, 726, 0, - 636, 650, 730, 649, 723, 457, 0, 484, 647, 594, - 0, 640, 613, 614, 0, 641, 609, 645, 0, 583, - 0, 552, 555, 584, 669, 670, 671, 320, 554, 673, - 674, 675, 676, 677, 678, 679, 672, 524, 617, 593, - 620, 533, 596, 595, 0, 0, 631, 550, 632, 633, - 441, 442, 443, 444, 382, 657, 342, 553, 471, 0, - 618, 0, 0, 0, 0, 0, 0, 0, 0, 623, - 624, 621, 735, 0, 680, 681, 0, 0, 547, 548, - 377, 0, 566, 385, 341, 456, 379, 531, 408, 0, - 559, 625, 560, 473, 474, 683, 688, 684, 685, 687, - 707, 448, 399, 404, 488, 410, 424, 476, 530, 454, - 481, 339, 520, 490, 429, 610, 638, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 497, 526, 0, 539, 0, 407, 408, + 0, 0, 0, 0, 0, 0, 0, 324, 504, 1711, + 338, 491, 537, 343, 499, 516, 333, 457, 488, 0, + 0, 326, 521, 498, 439, 325, 0, 482, 366, 383, + 363, 455, 0, 0, 520, 550, 362, 540, 0, 531, + 328, 0, 530, 454, 517, 522, 440, 433, 0, 327, + 519, 438, 432, 413, 373, 566, 414, 415, 416, 417, + 418, 419, 387, 469, 430, 470, 388, 444, 443, 445, + 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, + 0, 0, 0, 0, 0, 561, 562, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 694, 0, 0, 698, 0, 533, 0, 0, + 0, 0, 0, 0, 502, 0, 0, 420, 0, 0, + 0, 551, 0, 485, 460, 736, 0, 0, 483, 428, + 518, 471, 524, 505, 532, 477, 472, 318, 506, 365, + 441, 334, 336, 726, 367, 370, 374, 375, 450, 451, + 465, 490, 509, 510, 511, 364, 348, 484, 349, 384, + 350, 319, 356, 354, 357, 492, 358, 321, 466, 515, + 0, 380, 480, 436, 322, 435, 467, 514, 513, 335, + 541, 548, 549, 639, 0, 554, 737, 738, 739, 563, + 0, 473, 331, 330, 0, 0, 0, 360, 468, 344, + 346, 347, 345, 463, 464, 568, 569, 570, 572, 0, + 573, 574, 0, 0, 0, 0, 575, 640, 656, 624, + 593, 556, 648, 590, 594, 595, 401, 402, 403, 404, + 659, 0, 0, 0, 547, 421, 422, 0, 372, 371, + 437, 323, 0, 0, 410, 400, 474, 329, 368, 412, + 406, 423, 424, 425, 378, 313, 314, 732, 361, 456, + 661, 696, 697, 586, 0, 649, 587, 596, 353, 621, + 633, 632, 452, 546, 0, 644, 647, 576, 731, 0, + 641, 655, 735, 654, 728, 462, 0, 489, 652, 599, + 0, 645, 618, 619, 0, 646, 614, 650, 0, 588, + 0, 557, 560, 589, 674, 675, 676, 320, 559, 678, + 679, 680, 681, 682, 683, 684, 677, 529, 622, 598, + 625, 538, 601, 600, 0, 0, 636, 555, 637, 638, + 446, 447, 448, 449, 382, 662, 342, 558, 476, 0, + 623, 0, 0, 0, 0, 0, 0, 0, 0, 628, + 629, 626, 740, 0, 685, 686, 0, 0, 552, 553, + 377, 0, 571, 385, 341, 461, 379, 536, 409, 0, + 564, 630, 565, 478, 479, 688, 693, 689, 690, 692, + 712, 453, 399, 405, 493, 411, 429, 481, 535, 459, + 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 665, 664, 663, 662, 661, 660, - 659, 658, 0, 0, 607, 507, 355, 307, 351, 352, - 359, 724, 720, 725, 708, 711, 710, 686, 0, 315, - 587, 422, 470, 376, 652, 653, 0, 706, 259, 260, + 0, 0, 0, 0, 670, 669, 668, 667, 666, 665, + 664, 663, 0, 0, 612, 512, 355, 307, 351, 352, + 359, 729, 725, 730, 713, 716, 715, 691, 0, 315, + 592, 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, 283, - 284, 285, 655, 276, 277, 286, 287, 288, 289, 290, + 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, - 0, 0, 0, 309, 712, 713, 714, 715, 716, 0, - 0, 310, 311, 312, 0, 0, 274, 275, 302, 498, - 303, 304, 305, 306, 0, 0, 537, 538, 539, 562, - 0, 540, 522, 586, 386, 316, 502, 529, 722, 0, - 0, 0, 0, 0, 0, 0, 637, 648, 682, 0, - 694, 695, 697, 699, 698, 701, 495, 496, 709, 0, - 0, 703, 704, 705, 702, 426, 482, 503, 489, 0, - 728, 577, 578, 729, 690, 317, 453, 0, 0, 592, - 626, 615, 700, 580, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 369, 0, 0, 421, 630, 611, - 622, 612, 597, 598, 599, 606, 381, 600, 601, 602, - 572, 603, 573, 604, 605, 0, 629, 579, 491, 437, - 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 309, 717, 718, 719, 720, 721, 0, + 0, 310, 311, 312, 0, 0, 274, 275, 302, 503, + 303, 304, 305, 306, 0, 0, 542, 543, 544, 567, + 0, 545, 527, 591, 386, 316, 507, 534, 727, 0, + 0, 0, 0, 0, 0, 0, 642, 653, 687, 0, + 699, 700, 702, 704, 703, 706, 500, 501, 714, 0, + 0, 708, 709, 710, 707, 431, 487, 508, 494, 0, + 733, 582, 583, 734, 695, 317, 458, 0, 0, 597, + 631, 620, 705, 585, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 369, 0, 0, 426, 635, 616, + 627, 617, 602, 603, 604, 611, 381, 605, 606, 607, + 577, 608, 578, 609, 610, 0, 634, 584, 496, 442, + 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, 337, - 246, 574, 696, 576, 575, 0, 0, 0, 0, 0, + 246, 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 492, 521, 0, 534, 0, 406, - 407, 0, 0, 0, 0, 0, 0, 0, 324, 499, - 1702, 338, 486, 532, 343, 494, 511, 333, 452, 483, - 0, 0, 326, 516, 493, 434, 325, 0, 477, 366, - 383, 363, 450, 0, 0, 515, 545, 362, 535, 0, - 526, 328, 0, 525, 449, 512, 517, 435, 428, 0, - 327, 514, 433, 427, 412, 373, 561, 413, 414, 387, - 464, 425, 465, 388, 439, 438, 440, 389, 390, 391, - 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, - 0, 0, 556, 557, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, - 0, 0, 693, 0, 528, 0, 0, 0, 0, 0, - 0, 497, 0, 0, 415, 0, 0, 0, 546, 0, - 480, 455, 731, 0, 0, 478, 423, 513, 466, 519, - 500, 527, 472, 467, 318, 501, 365, 436, 334, 336, - 721, 367, 370, 374, 375, 445, 446, 460, 485, 504, - 505, 506, 364, 348, 479, 349, 384, 350, 319, 356, - 354, 357, 487, 358, 321, 461, 510, 0, 380, 475, - 431, 322, 430, 462, 509, 508, 335, 536, 543, 544, - 634, 0, 549, 732, 733, 734, 558, 0, 468, 331, - 330, 0, 0, 0, 360, 463, 344, 346, 347, 345, - 458, 459, 563, 564, 565, 567, 0, 568, 569, 0, - 0, 0, 0, 570, 635, 651, 619, 588, 551, 643, - 585, 589, 590, 401, 402, 403, 654, 0, 0, 0, - 542, 416, 417, 0, 372, 371, 432, 323, 0, 0, - 409, 400, 469, 329, 368, 411, 405, 418, 419, 420, - 378, 313, 314, 727, 361, 451, 656, 691, 692, 581, - 0, 644, 582, 591, 353, 616, 628, 627, 447, 541, - 0, 639, 642, 571, 726, 0, 636, 650, 730, 649, - 723, 457, 0, 484, 647, 594, 0, 640, 613, 614, - 0, 641, 609, 645, 0, 583, 0, 552, 555, 584, - 669, 670, 671, 320, 554, 673, 674, 675, 676, 677, - 678, 679, 672, 524, 617, 593, 620, 533, 596, 595, - 0, 0, 631, 550, 632, 633, 441, 442, 443, 444, - 382, 657, 342, 553, 471, 0, 618, 0, 0, 0, - 0, 0, 0, 0, 0, 623, 624, 621, 735, 0, - 680, 681, 0, 0, 547, 548, 377, 0, 566, 385, - 341, 456, 379, 531, 408, 0, 559, 625, 560, 473, - 474, 683, 688, 684, 685, 687, 707, 448, 399, 404, - 488, 410, 424, 476, 530, 454, 481, 339, 520, 490, - 429, 610, 638, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 665, 664, 663, 662, 661, 660, 659, 658, 0, 0, - 607, 507, 355, 307, 351, 352, 359, 724, 720, 725, - 708, 711, 710, 686, 0, 315, 587, 422, 470, 376, - 652, 653, 0, 706, 259, 260, 261, 262, 263, 264, - 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, - 278, 279, 280, 281, 282, 283, 284, 285, 655, 276, - 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 298, 299, 0, 0, 0, 0, 309, - 712, 713, 714, 715, 716, 0, 0, 310, 311, 312, - 0, 0, 274, 275, 302, 498, 303, 304, 305, 306, - 0, 0, 537, 538, 539, 562, 0, 540, 522, 586, - 386, 316, 502, 529, 722, 0, 0, 0, 0, 0, - 0, 0, 637, 648, 682, 0, 694, 695, 697, 699, - 698, 701, 495, 496, 709, 0, 0, 703, 704, 705, - 702, 426, 482, 503, 489, 0, 728, 577, 578, 729, - 690, 317, 453, 0, 0, 592, 626, 615, 700, 580, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 369, 0, 0, 421, 630, 611, 622, 612, 597, 598, - 599, 606, 381, 600, 601, 602, 572, 603, 573, 604, - 605, 0, 629, 579, 491, 437, 0, 646, 0, 0, + 0, 0, 0, 0, 497, 526, 0, 539, 0, 407, + 408, 0, 0, 0, 0, 0, 0, 0, 324, 504, + 523, 338, 491, 537, 343, 499, 1575, 333, 457, 488, + 0, 0, 326, 521, 498, 439, 325, 0, 482, 366, + 383, 363, 455, 0, 0, 520, 550, 362, 540, 0, + 531, 328, 0, 530, 454, 517, 522, 440, 433, 0, + 327, 519, 438, 432, 413, 373, 566, 414, 415, 416, + 417, 418, 419, 387, 469, 430, 470, 388, 444, 443, + 445, 389, 390, 391, 392, 393, 394, 395, 396, 397, + 398, 0, 0, 0, 0, 0, 561, 562, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 694, 0, 0, 698, 0, 533, 0, + 0, 0, 0, 0, 0, 502, 0, 0, 420, 0, + 0, 0, 551, 0, 485, 460, 736, 0, 0, 483, + 428, 518, 471, 524, 505, 532, 477, 472, 318, 506, + 365, 441, 334, 336, 726, 367, 370, 374, 375, 450, + 451, 465, 490, 509, 510, 511, 364, 348, 484, 349, + 384, 350, 319, 356, 354, 357, 492, 358, 321, 466, + 515, 0, 380, 480, 436, 322, 435, 467, 514, 513, + 335, 541, 548, 549, 639, 0, 554, 737, 738, 739, + 563, 0, 473, 331, 330, 0, 0, 0, 360, 468, + 344, 346, 347, 345, 463, 464, 568, 569, 570, 572, + 0, 573, 574, 0, 0, 0, 0, 575, 640, 656, + 624, 593, 556, 648, 590, 594, 595, 401, 402, 403, + 404, 659, 0, 0, 0, 547, 421, 422, 0, 372, + 371, 437, 323, 0, 0, 410, 400, 474, 329, 368, + 412, 406, 423, 424, 425, 378, 313, 314, 732, 361, + 456, 661, 696, 697, 586, 0, 649, 587, 596, 353, + 621, 633, 632, 452, 546, 0, 644, 647, 576, 731, + 0, 641, 655, 735, 654, 728, 462, 0, 489, 652, + 599, 0, 645, 618, 619, 0, 646, 614, 650, 0, + 588, 0, 557, 560, 589, 674, 675, 676, 320, 559, + 678, 679, 680, 681, 682, 683, 684, 677, 529, 622, + 598, 625, 538, 601, 600, 0, 0, 636, 555, 637, + 638, 446, 447, 448, 449, 382, 662, 342, 558, 476, + 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, + 628, 629, 626, 740, 0, 685, 686, 0, 0, 552, + 553, 377, 0, 571, 385, 341, 461, 379, 536, 409, + 0, 564, 630, 565, 478, 479, 688, 693, 689, 690, + 692, 712, 453, 399, 405, 493, 411, 429, 481, 535, + 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, + 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 670, 669, 668, 667, 666, + 665, 664, 663, 0, 0, 612, 512, 355, 307, 351, + 352, 359, 729, 725, 730, 713, 716, 715, 691, 0, + 315, 592, 427, 475, 376, 657, 658, 0, 711, 259, + 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, + 269, 270, 271, 272, 273, 278, 279, 280, 281, 282, + 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, + 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, + 0, 0, 0, 0, 309, 717, 718, 719, 720, 721, + 0, 0, 310, 311, 312, 0, 0, 274, 275, 302, + 503, 303, 304, 305, 306, 0, 0, 542, 543, 544, + 567, 0, 545, 527, 591, 386, 316, 507, 534, 727, + 0, 0, 0, 0, 0, 0, 0, 642, 653, 687, + 0, 699, 700, 702, 704, 703, 706, 500, 501, 714, + 0, 0, 708, 709, 710, 707, 431, 487, 508, 494, + 0, 733, 582, 583, 734, 695, 317, 458, 0, 0, + 597, 631, 620, 705, 585, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 369, 0, 0, 426, 635, + 616, 627, 617, 602, 603, 604, 611, 381, 605, 606, + 607, 577, 608, 578, 609, 610, 0, 634, 584, 496, + 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 0, 0, 0, 0, 337, 246, 574, 696, 576, - 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, + 337, 246, 579, 701, 581, 580, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 492, 521, 0, 534, 0, 406, 407, 0, 0, 0, - 0, 0, 0, 0, 324, 499, 1700, 338, 486, 532, - 343, 494, 511, 333, 452, 483, 0, 0, 326, 516, - 493, 434, 325, 0, 477, 366, 383, 363, 450, 0, - 0, 515, 545, 362, 535, 0, 526, 328, 0, 525, - 449, 512, 517, 435, 428, 0, 327, 514, 433, 427, - 412, 373, 561, 413, 414, 387, 464, 425, 465, 388, - 439, 438, 440, 389, 390, 391, 392, 393, 394, 395, - 396, 397, 398, 0, 0, 0, 0, 0, 556, 557, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 689, 0, 0, 693, 0, - 528, 0, 0, 0, 0, 0, 0, 497, 0, 0, - 415, 0, 0, 0, 546, 0, 480, 455, 731, 0, - 0, 478, 423, 513, 466, 519, 500, 527, 472, 467, - 318, 501, 365, 436, 334, 336, 721, 367, 370, 374, - 375, 445, 446, 460, 485, 504, 505, 506, 364, 348, - 479, 349, 384, 350, 319, 356, 354, 357, 487, 358, - 321, 461, 510, 0, 380, 475, 431, 322, 430, 462, - 509, 508, 335, 536, 543, 544, 634, 0, 549, 732, - 733, 734, 558, 0, 468, 331, 330, 0, 0, 0, - 360, 463, 344, 346, 347, 345, 458, 459, 563, 564, - 565, 567, 0, 568, 569, 0, 0, 0, 0, 570, - 635, 651, 619, 588, 551, 643, 585, 589, 590, 401, - 402, 403, 654, 0, 0, 0, 542, 416, 417, 0, - 372, 371, 432, 323, 0, 0, 409, 400, 469, 329, - 368, 411, 405, 418, 419, 420, 378, 313, 314, 727, - 361, 451, 656, 691, 692, 581, 0, 644, 582, 591, - 353, 616, 628, 627, 447, 541, 0, 639, 642, 571, - 726, 0, 636, 650, 730, 649, 723, 457, 0, 484, - 647, 594, 0, 640, 613, 614, 0, 641, 609, 645, - 0, 583, 0, 552, 555, 584, 669, 670, 671, 320, - 554, 673, 674, 675, 676, 677, 678, 679, 672, 524, - 617, 593, 620, 533, 596, 595, 0, 0, 631, 550, - 632, 633, 441, 442, 443, 444, 382, 657, 342, 553, - 471, 0, 618, 0, 0, 0, 0, 0, 0, 0, - 0, 623, 624, 621, 735, 0, 680, 681, 0, 0, - 547, 548, 377, 0, 566, 385, 341, 456, 379, 531, - 408, 0, 559, 625, 560, 473, 474, 683, 688, 684, - 685, 687, 707, 448, 399, 404, 488, 410, 424, 476, - 530, 454, 481, 339, 520, 490, 429, 610, 638, 0, + 0, 0, 0, 0, 0, 497, 526, 0, 539, 0, + 407, 408, 0, 0, 0, 0, 0, 0, 0, 324, + 504, 523, 338, 491, 537, 343, 499, 516, 333, 457, + 488, 0, 0, 326, 521, 498, 439, 325, 0, 482, + 366, 383, 363, 455, 0, 0, 520, 550, 362, 540, + 0, 531, 328, 0, 530, 454, 517, 522, 440, 433, + 0, 327, 519, 438, 432, 413, 373, 566, 414, 415, + 416, 417, 418, 419, 387, 469, 430, 470, 388, 444, + 443, 445, 389, 390, 391, 392, 393, 394, 395, 396, + 397, 398, 0, 0, 0, 0, 0, 561, 562, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 694, 0, 0, 698, 0, 533, + 0, 0, 0, 0, 0, 0, 502, 0, 0, 420, + 0, 0, 0, 551, 0, 485, 460, 736, 0, 0, + 483, 428, 518, 471, 524, 505, 532, 477, 472, 318, + 506, 365, 441, 334, 336, 831, 367, 370, 374, 375, + 450, 451, 465, 490, 509, 510, 511, 364, 348, 484, + 349, 384, 350, 319, 356, 354, 357, 492, 358, 321, + 466, 515, 0, 380, 480, 436, 322, 435, 467, 514, + 513, 335, 541, 548, 549, 639, 0, 554, 737, 738, + 739, 563, 0, 473, 331, 330, 0, 0, 0, 360, + 468, 344, 346, 347, 345, 463, 464, 568, 569, 570, + 572, 0, 573, 574, 0, 0, 0, 0, 575, 640, + 656, 624, 593, 556, 648, 590, 594, 595, 401, 402, + 403, 404, 659, 0, 0, 0, 547, 421, 422, 0, + 372, 371, 437, 323, 0, 0, 410, 400, 474, 329, + 368, 412, 406, 423, 424, 425, 378, 313, 314, 732, + 361, 456, 661, 696, 697, 586, 0, 649, 587, 596, + 353, 621, 633, 632, 452, 546, 0, 644, 647, 576, + 731, 0, 641, 655, 735, 654, 728, 462, 0, 489, + 652, 599, 0, 645, 618, 619, 0, 646, 614, 650, + 0, 588, 0, 557, 560, 589, 674, 675, 676, 320, + 559, 678, 679, 680, 681, 682, 683, 684, 677, 529, + 622, 598, 625, 538, 601, 600, 0, 0, 636, 555, + 637, 638, 446, 447, 448, 449, 382, 662, 342, 558, + 476, 0, 623, 0, 0, 0, 0, 0, 0, 0, + 0, 628, 629, 626, 740, 0, 685, 686, 0, 0, + 552, 553, 377, 0, 571, 385, 341, 461, 379, 536, + 409, 0, 564, 630, 565, 478, 479, 688, 693, 689, + 690, 692, 712, 453, 399, 405, 493, 411, 429, 481, + 535, 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 665, 664, 663, 662, - 661, 660, 659, 658, 0, 0, 607, 507, 355, 307, - 351, 352, 359, 724, 720, 725, 708, 711, 710, 686, - 0, 315, 587, 422, 470, 376, 652, 653, 0, 706, + 0, 0, 0, 0, 0, 0, 670, 669, 668, 667, + 666, 665, 664, 663, 0, 0, 612, 512, 355, 307, + 351, 352, 359, 729, 725, 730, 713, 716, 715, 691, + 0, 315, 592, 427, 475, 376, 657, 658, 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, 281, - 282, 283, 284, 285, 655, 276, 277, 286, 287, 288, + 282, 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, - 299, 0, 0, 0, 0, 309, 712, 713, 714, 715, - 716, 0, 0, 310, 311, 312, 0, 0, 274, 275, - 302, 498, 303, 304, 305, 306, 0, 0, 537, 538, - 539, 562, 0, 540, 522, 586, 386, 316, 502, 529, - 722, 0, 0, 0, 0, 0, 0, 0, 637, 648, - 682, 0, 694, 695, 697, 699, 698, 701, 495, 496, - 709, 0, 0, 703, 704, 705, 702, 426, 482, 503, - 489, 0, 728, 577, 578, 729, 690, 317, 453, 0, - 0, 592, 626, 615, 700, 580, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 369, 0, 0, 421, - 630, 611, 622, 612, 597, 598, 599, 606, 381, 600, - 601, 602, 572, 603, 573, 604, 605, 0, 629, 579, - 491, 437, 0, 646, 0, 0, 0, 0, 0, 0, + 299, 0, 0, 0, 0, 309, 717, 718, 719, 720, + 721, 0, 0, 310, 311, 312, 0, 0, 274, 275, + 302, 503, 303, 304, 305, 306, 0, 0, 542, 543, + 544, 567, 0, 545, 527, 591, 386, 316, 507, 534, + 727, 0, 0, 0, 0, 0, 0, 0, 642, 653, + 687, 0, 699, 700, 702, 704, 703, 706, 500, 501, + 714, 0, 0, 708, 709, 710, 707, 431, 487, 508, + 494, 0, 733, 582, 583, 734, 695, 317, 458, 0, + 0, 597, 631, 620, 705, 585, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 369, 0, 0, 426, + 635, 616, 627, 617, 602, 603, 604, 611, 381, 605, + 606, 607, 577, 608, 578, 609, 610, 0, 634, 584, + 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, - 0, 337, 246, 574, 696, 576, 575, 0, 0, 0, + 0, 337, 246, 579, 701, 581, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 492, 521, 0, 534, - 0, 406, 407, 0, 0, 0, 0, 0, 0, 0, - 324, 499, 518, 338, 486, 532, 343, 494, 1564, 333, - 452, 483, 0, 0, 326, 516, 493, 434, 325, 0, - 477, 366, 383, 363, 450, 0, 0, 515, 545, 362, - 535, 0, 526, 328, 0, 525, 449, 512, 517, 435, - 428, 0, 327, 514, 433, 427, 412, 373, 561, 413, - 414, 387, 464, 425, 465, 388, 439, 438, 440, 389, - 390, 391, 392, 393, 394, 395, 396, 397, 398, 0, - 0, 0, 0, 0, 556, 557, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 689, 0, 0, 693, 0, 528, 0, 0, 0, - 0, 0, 0, 497, 0, 0, 415, 0, 0, 0, - 546, 0, 480, 455, 731, 0, 0, 478, 423, 513, - 466, 519, 500, 527, 472, 467, 318, 501, 365, 436, - 334, 336, 721, 367, 370, 374, 375, 445, 446, 460, - 485, 504, 505, 506, 364, 348, 479, 349, 384, 350, - 319, 356, 354, 357, 487, 358, 321, 461, 510, 0, - 380, 475, 431, 322, 430, 462, 509, 508, 335, 536, - 543, 544, 634, 0, 549, 732, 733, 734, 558, 0, - 468, 331, 330, 0, 0, 0, 360, 463, 344, 346, - 347, 345, 458, 459, 563, 564, 565, 567, 0, 568, - 569, 0, 0, 0, 0, 570, 635, 651, 619, 588, - 551, 643, 585, 589, 590, 401, 402, 403, 654, 0, - 0, 0, 542, 416, 417, 0, 372, 371, 432, 323, - 0, 0, 409, 400, 469, 329, 368, 411, 405, 418, - 419, 420, 378, 313, 314, 727, 361, 451, 656, 691, - 692, 581, 0, 644, 582, 591, 353, 616, 628, 627, - 447, 541, 0, 639, 642, 571, 726, 0, 636, 650, - 730, 649, 723, 457, 0, 484, 647, 594, 0, 640, - 613, 614, 0, 641, 609, 645, 0, 583, 0, 552, - 555, 584, 669, 670, 671, 320, 554, 673, 674, 675, - 676, 677, 678, 679, 672, 524, 617, 593, 620, 533, - 596, 595, 0, 0, 631, 550, 632, 633, 441, 442, - 443, 444, 382, 657, 342, 553, 471, 0, 618, 0, - 0, 0, 0, 0, 0, 0, 0, 623, 624, 621, - 735, 0, 680, 681, 0, 0, 547, 548, 377, 0, - 566, 385, 341, 456, 379, 531, 408, 0, 559, 625, - 560, 473, 474, 683, 688, 684, 685, 687, 707, 448, - 399, 404, 488, 410, 424, 476, 530, 454, 481, 339, - 520, 490, 429, 610, 638, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 665, 664, 663, 662, 661, 660, 659, 658, - 0, 0, 607, 507, 355, 307, 351, 352, 359, 724, - 720, 725, 708, 711, 710, 686, 0, 315, 587, 422, - 470, 376, 652, 653, 0, 706, 259, 260, 261, 262, - 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, - 272, 273, 278, 279, 280, 281, 282, 283, 284, 285, - 655, 276, 277, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 298, 299, 0, 0, 0, - 0, 309, 712, 713, 714, 715, 716, 0, 0, 310, - 311, 312, 0, 0, 274, 275, 302, 498, 303, 304, - 305, 306, 0, 0, 537, 538, 539, 562, 0, 540, - 522, 586, 386, 316, 502, 529, 722, 0, 0, 0, - 0, 0, 0, 0, 637, 648, 682, 0, 694, 695, - 697, 699, 698, 701, 495, 496, 709, 0, 0, 703, - 704, 705, 702, 426, 482, 503, 489, 0, 728, 577, - 578, 729, 690, 317, 453, 0, 0, 592, 626, 615, - 700, 580, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 369, 0, 0, 421, 630, 611, 622, 612, - 597, 598, 599, 606, 381, 600, 601, 602, 572, 603, - 573, 604, 605, 0, 629, 579, 491, 437, 0, 646, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 0, 0, 0, 0, 337, 246, 574, - 696, 576, 575, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 497, 526, 0, 539, + 0, 407, 408, 0, 0, 0, 0, 0, 0, 0, + 324, 504, 523, 338, 491, 537, 343, 499, 516, 333, + 457, 488, 0, 0, 326, 521, 498, 439, 325, 0, + 482, 366, 383, 363, 455, 0, 0, 520, 550, 362, + 540, 0, 531, 328, 0, 530, 454, 517, 522, 440, + 433, 0, 327, 519, 438, 432, 413, 373, 566, 414, + 415, 416, 417, 418, 419, 387, 469, 430, 470, 388, + 444, 443, 445, 389, 390, 391, 392, 393, 394, 395, + 396, 397, 398, 0, 0, 0, 0, 0, 561, 562, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 694, 0, 0, 698, 0, + 533, 0, 0, 0, 0, 0, 0, 502, 0, 0, + 420, 0, 0, 0, 551, 0, 485, 460, 736, 0, + 0, 483, 428, 518, 471, 524, 505, 532, 783, 472, + 318, 506, 365, 441, 334, 336, 726, 367, 370, 374, + 375, 450, 451, 465, 490, 509, 510, 511, 364, 348, + 484, 349, 384, 350, 319, 356, 354, 357, 492, 358, + 321, 466, 515, 0, 380, 480, 436, 322, 435, 467, + 514, 513, 335, 541, 548, 549, 639, 0, 554, 737, + 738, 739, 563, 0, 473, 331, 330, 0, 0, 0, + 360, 468, 344, 346, 347, 345, 463, 464, 568, 569, + 570, 572, 0, 573, 574, 0, 0, 0, 0, 575, + 640, 656, 624, 593, 556, 648, 590, 594, 595, 401, + 402, 403, 404, 659, 0, 0, 0, 547, 421, 422, + 0, 372, 371, 437, 323, 0, 0, 410, 400, 474, + 329, 368, 412, 406, 423, 424, 425, 378, 313, 314, + 732, 361, 456, 661, 696, 697, 586, 0, 649, 587, + 596, 353, 621, 633, 632, 452, 546, 0, 644, 647, + 576, 731, 0, 641, 655, 735, 654, 728, 462, 0, + 489, 652, 599, 0, 645, 618, 619, 0, 646, 614, + 650, 0, 588, 0, 557, 560, 589, 674, 675, 676, + 320, 559, 678, 679, 680, 681, 682, 683, 784, 677, + 529, 622, 598, 625, 538, 601, 600, 0, 0, 636, + 555, 637, 638, 446, 447, 448, 449, 382, 662, 342, + 558, 476, 0, 623, 0, 0, 0, 0, 0, 0, + 0, 0, 628, 629, 626, 740, 0, 685, 686, 0, + 0, 552, 553, 377, 0, 571, 385, 341, 461, 379, + 536, 409, 0, 564, 630, 565, 478, 479, 688, 693, + 689, 690, 692, 712, 453, 399, 405, 493, 411, 429, + 481, 535, 459, 486, 339, 525, 495, 434, 615, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 670, 669, 668, + 667, 666, 665, 664, 663, 0, 0, 612, 512, 355, + 307, 351, 352, 359, 729, 725, 730, 713, 716, 715, + 691, 0, 315, 592, 427, 475, 376, 657, 658, 0, + 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, + 267, 268, 269, 270, 271, 272, 273, 278, 279, 280, + 281, 282, 283, 284, 285, 660, 276, 277, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 298, 299, 0, 0, 0, 0, 309, 717, 718, 719, + 720, 721, 0, 0, 310, 311, 312, 0, 0, 274, + 275, 302, 503, 303, 304, 305, 306, 0, 0, 542, + 543, 544, 567, 0, 545, 527, 591, 386, 316, 507, + 534, 727, 0, 0, 0, 0, 0, 0, 0, 642, + 653, 687, 0, 699, 700, 702, 704, 703, 706, 500, + 501, 714, 0, 0, 708, 709, 710, 707, 431, 487, + 508, 494, 0, 733, 582, 583, 734, 695, 317, 458, + 0, 0, 597, 631, 620, 705, 585, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, + 426, 635, 616, 627, 617, 602, 603, 604, 611, 381, + 605, 606, 607, 577, 608, 578, 609, 610, 0, 634, + 584, 496, 442, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 492, 521, 0, 534, 0, 406, 407, 0, - 0, 0, 0, 0, 0, 0, 324, 499, 518, 338, - 486, 532, 343, 494, 511, 333, 452, 483, 0, 0, - 326, 516, 493, 434, 325, 0, 477, 366, 383, 363, - 450, 0, 0, 515, 545, 362, 535, 0, 526, 328, - 0, 525, 449, 512, 517, 435, 428, 0, 327, 514, - 433, 427, 412, 373, 561, 413, 414, 387, 464, 425, - 465, 388, 439, 438, 440, 389, 390, 391, 392, 393, - 394, 395, 396, 397, 398, 0, 0, 0, 0, 0, - 556, 557, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 689, 0, 0, - 693, 0, 528, 0, 0, 0, 0, 0, 0, 497, - 0, 0, 415, 0, 0, 0, 546, 0, 480, 455, - 731, 0, 0, 478, 423, 513, 466, 519, 500, 527, - 472, 467, 318, 501, 365, 436, 334, 336, 826, 367, - 370, 374, 375, 445, 446, 460, 485, 504, 505, 506, - 364, 348, 479, 349, 384, 350, 319, 356, 354, 357, - 487, 358, 321, 461, 510, 0, 380, 475, 431, 322, - 430, 462, 509, 508, 335, 536, 543, 544, 634, 0, - 549, 732, 733, 734, 558, 0, 468, 331, 330, 0, - 0, 0, 360, 463, 344, 346, 347, 345, 458, 459, - 563, 564, 565, 567, 0, 568, 569, 0, 0, 0, - 0, 570, 635, 651, 619, 588, 551, 643, 585, 589, - 590, 401, 402, 403, 654, 0, 0, 0, 542, 416, - 417, 0, 372, 371, 432, 323, 0, 0, 409, 400, - 469, 329, 368, 411, 405, 418, 419, 420, 378, 313, - 314, 727, 361, 451, 656, 691, 692, 581, 0, 644, - 582, 591, 353, 616, 628, 627, 447, 541, 0, 639, - 642, 571, 726, 0, 636, 650, 730, 649, 723, 457, - 0, 484, 647, 594, 0, 640, 613, 614, 0, 641, - 609, 645, 0, 583, 0, 552, 555, 584, 669, 670, - 671, 320, 554, 673, 674, 675, 676, 677, 678, 679, - 672, 524, 617, 593, 620, 533, 596, 595, 0, 0, - 631, 550, 632, 633, 441, 442, 443, 444, 382, 657, - 342, 553, 471, 0, 618, 0, 0, 0, 0, 0, - 0, 0, 0, 623, 624, 621, 735, 0, 680, 681, - 0, 0, 547, 548, 377, 0, 566, 385, 341, 456, - 379, 531, 408, 0, 559, 625, 560, 473, 474, 683, - 688, 684, 685, 687, 707, 448, 399, 404, 488, 410, - 424, 476, 530, 454, 481, 339, 520, 490, 429, 610, - 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, + 0, 0, 337, 246, 579, 701, 581, 580, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 497, 526, 0, + 539, 0, 407, 408, 0, 0, 0, 0, 0, 0, + 0, 324, 504, 523, 338, 491, 537, 343, 499, 516, + 333, 457, 488, 0, 0, 326, 521, 498, 439, 325, + 0, 482, 366, 383, 363, 455, 0, 0, 520, 550, + 362, 540, 0, 531, 328, 0, 530, 454, 517, 522, + 440, 433, 0, 327, 519, 438, 432, 413, 373, 566, + 414, 415, 416, 417, 418, 419, 387, 469, 430, 470, + 388, 444, 443, 445, 389, 390, 391, 392, 393, 394, + 395, 396, 397, 398, 0, 0, 0, 0, 0, 561, + 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 694, 0, 0, 698, + 0, 533, 0, 0, 0, 0, 0, 0, 502, 0, + 0, 420, 0, 0, 0, 551, 0, 485, 460, 736, + 0, 0, 483, 428, 518, 471, 524, 505, 532, 477, + 472, 318, 506, 365, 441, 334, 336, 726, 367, 370, + 374, 375, 450, 451, 465, 490, 509, 510, 511, 364, + 348, 484, 349, 384, 350, 319, 356, 354, 357, 492, + 358, 321, 466, 515, 0, 380, 480, 436, 322, 435, + 467, 514, 513, 335, 541, 548, 549, 639, 0, 554, + 737, 738, 739, 563, 0, 473, 331, 330, 0, 0, + 0, 360, 468, 344, 346, 347, 345, 463, 464, 568, + 569, 570, 572, 0, 573, 574, 0, 0, 0, 0, + 575, 640, 656, 624, 593, 556, 648, 590, 594, 595, + 401, 402, 403, 404, 659, 0, 0, 0, 547, 421, + 422, 0, 372, 371, 437, 323, 0, 0, 410, 400, + 474, 329, 368, 412, 406, 423, 424, 425, 378, 313, + 314, 732, 361, 456, 661, 696, 697, 586, 0, 649, + 587, 596, 353, 621, 633, 632, 452, 546, 0, 644, + 647, 576, 731, 0, 641, 655, 735, 654, 728, 462, + 0, 489, 652, 599, 0, 645, 618, 619, 0, 646, + 614, 650, 0, 588, 0, 557, 560, 589, 674, 675, + 676, 320, 559, 678, 679, 680, 681, 682, 683, 684, + 677, 529, 622, 598, 625, 538, 601, 600, 0, 0, + 636, 555, 637, 638, 446, 447, 448, 449, 382, 662, + 342, 558, 476, 0, 623, 0, 0, 0, 0, 0, + 0, 0, 0, 628, 629, 626, 740, 0, 685, 686, + 0, 0, 552, 553, 377, 0, 571, 385, 341, 461, + 379, 536, 409, 0, 564, 630, 565, 478, 479, 688, + 693, 689, 690, 692, 712, 453, 399, 405, 493, 411, + 429, 481, 535, 459, 486, 339, 525, 495, 434, 615, + 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 665, 664, - 663, 662, 661, 660, 659, 658, 0, 0, 607, 507, - 355, 307, 351, 352, 359, 724, 720, 725, 708, 711, - 710, 686, 0, 315, 587, 422, 470, 376, 652, 653, - 0, 706, 259, 260, 261, 262, 263, 264, 265, 266, + 0, 0, 0, 0, 0, 0, 0, 0, 670, 669, + 668, 667, 666, 665, 664, 663, 0, 0, 612, 512, + 355, 307, 351, 352, 359, 729, 725, 730, 713, 716, + 715, 779, 0, 315, 592, 427, 475, 376, 657, 658, + 0, 711, 259, 260, 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, 278, 279, - 280, 281, 282, 283, 284, 285, 655, 276, 277, 286, + 280, 281, 282, 283, 284, 285, 660, 276, 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 298, 299, 0, 0, 0, 0, 309, 712, 713, - 714, 715, 716, 0, 0, 310, 311, 312, 0, 0, - 274, 275, 302, 498, 303, 304, 305, 306, 0, 0, - 537, 538, 539, 562, 0, 540, 522, 586, 386, 316, - 502, 529, 722, 0, 0, 0, 0, 0, 0, 0, - 637, 648, 682, 0, 694, 695, 697, 699, 698, 701, - 495, 496, 709, 0, 0, 703, 704, 705, 702, 426, - 482, 503, 489, 0, 728, 577, 578, 729, 690, 317, - 453, 0, 0, 592, 626, 615, 700, 580, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, - 0, 421, 630, 611, 622, 612, 597, 598, 599, 606, - 381, 600, 601, 602, 572, 603, 573, 604, 605, 0, - 629, 579, 491, 437, 0, 646, 0, 0, 0, 0, + 297, 298, 299, 0, 0, 0, 0, 309, 717, 718, + 719, 720, 721, 0, 0, 310, 311, 312, 0, 0, + 274, 275, 302, 503, 303, 304, 305, 306, 0, 0, + 542, 543, 544, 567, 0, 545, 527, 591, 386, 316, + 507, 534, 727, 0, 0, 0, 0, 0, 0, 0, + 642, 653, 687, 0, 699, 700, 702, 704, 703, 706, + 500, 501, 714, 0, 0, 708, 709, 710, 707, 431, + 487, 508, 494, 0, 733, 582, 583, 734, 695, 317, + 2270, 0, 0, 0, 0, 2231, 0, 0, 2278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, - 0, 0, 0, 337, 246, 574, 696, 576, 575, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2272, 2240, + 0, 0, 0, 0, 0, 0, 0, 0, 2273, 2274, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 2270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 2239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 2247, 0, 0, 0, 0, 2272, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 492, 521, - 0, 534, 0, 406, 407, 0, 0, 0, 0, 0, - 0, 0, 324, 499, 518, 338, 486, 532, 343, 494, - 511, 333, 452, 483, 0, 0, 326, 516, 493, 434, - 325, 0, 477, 366, 383, 363, 450, 0, 0, 515, - 545, 362, 535, 0, 526, 328, 0, 525, 449, 512, - 517, 435, 428, 0, 327, 514, 433, 427, 412, 373, - 561, 413, 414, 387, 464, 425, 465, 388, 439, 438, - 440, 389, 390, 391, 392, 393, 394, 395, 396, 397, - 398, 0, 0, 0, 0, 0, 556, 557, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 689, 0, 0, 693, 0, 528, 0, - 0, 0, 0, 0, 0, 497, 0, 0, 415, 0, - 0, 0, 546, 0, 480, 455, 731, 0, 0, 478, - 423, 513, 466, 519, 500, 527, 778, 467, 318, 501, - 365, 436, 334, 336, 721, 367, 370, 374, 375, 445, - 446, 460, 485, 504, 505, 506, 364, 348, 479, 349, - 384, 350, 319, 356, 354, 357, 487, 358, 321, 461, - 510, 0, 380, 475, 431, 322, 430, 462, 509, 508, - 335, 536, 543, 544, 634, 0, 549, 732, 733, 734, - 558, 0, 468, 331, 330, 0, 0, 0, 360, 463, - 344, 346, 347, 345, 458, 459, 563, 564, 565, 567, - 0, 568, 569, 0, 0, 0, 0, 570, 635, 651, - 619, 588, 551, 643, 585, 589, 590, 401, 402, 403, - 654, 0, 0, 0, 542, 416, 417, 0, 372, 371, - 432, 323, 0, 0, 409, 400, 469, 329, 368, 411, - 405, 418, 419, 420, 378, 313, 314, 727, 361, 451, - 656, 691, 692, 581, 0, 644, 582, 591, 353, 616, - 628, 627, 447, 541, 0, 639, 642, 571, 726, 0, - 636, 650, 730, 649, 723, 457, 0, 484, 647, 594, - 0, 640, 613, 614, 0, 641, 609, 645, 0, 583, - 0, 552, 555, 584, 669, 670, 671, 320, 554, 673, - 674, 675, 676, 677, 678, 779, 672, 524, 617, 593, - 620, 533, 596, 595, 0, 0, 631, 550, 632, 633, - 441, 442, 443, 444, 382, 657, 342, 553, 471, 0, - 618, 0, 0, 0, 0, 0, 0, 0, 0, 623, - 624, 621, 735, 0, 680, 681, 0, 0, 547, 548, - 377, 0, 566, 385, 341, 456, 379, 531, 408, 0, - 559, 625, 560, 473, 474, 683, 688, 684, 685, 687, - 707, 448, 399, 404, 488, 410, 424, 476, 530, 454, - 481, 339, 520, 490, 429, 610, 638, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 665, 664, 663, 662, 661, 660, - 659, 658, 0, 0, 607, 507, 355, 307, 351, 352, - 359, 724, 720, 725, 708, 711, 710, 686, 0, 315, - 587, 422, 470, 376, 652, 653, 0, 706, 259, 260, - 261, 262, 263, 264, 265, 266, 308, 267, 268, 269, - 270, 271, 272, 273, 278, 279, 280, 281, 282, 283, - 284, 285, 655, 276, 277, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 298, 299, 0, - 0, 0, 0, 309, 712, 713, 714, 715, 716, 0, - 0, 310, 311, 312, 0, 0, 274, 275, 302, 498, - 303, 304, 305, 306, 0, 0, 537, 538, 539, 562, - 0, 540, 522, 586, 386, 316, 502, 529, 722, 0, - 0, 0, 0, 0, 0, 0, 637, 648, 682, 0, - 694, 695, 697, 699, 698, 701, 495, 496, 709, 0, - 0, 703, 704, 705, 702, 426, 482, 503, 489, 0, - 728, 577, 578, 729, 690, 317, 453, 0, 0, 592, - 626, 615, 700, 580, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 369, 0, 0, 421, 630, 611, - 622, 612, 597, 598, 599, 606, 381, 600, 601, 602, - 572, 603, 573, 604, 605, 0, 629, 579, 491, 437, - 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 245, 0, 0, 0, 0, 0, 0, 337, - 246, 574, 696, 576, 575, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 2272, 0, 0, 0, + 0, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 2263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 492, 521, 0, 534, 0, 406, - 407, 0, 0, 0, 0, 0, 0, 0, 324, 499, - 518, 338, 486, 532, 343, 494, 511, 333, 452, 483, - 0, 0, 326, 516, 493, 434, 325, 0, 477, 366, - 383, 363, 450, 0, 0, 515, 545, 362, 535, 0, - 526, 328, 0, 525, 449, 512, 517, 435, 428, 0, - 327, 514, 433, 427, 412, 373, 561, 413, 414, 387, - 464, 425, 465, 388, 439, 438, 440, 389, 390, 391, - 392, 393, 394, 395, 396, 397, 398, 0, 0, 0, - 0, 0, 556, 557, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, - 0, 0, 693, 0, 528, 0, 0, 0, 0, 0, - 0, 497, 0, 0, 415, 0, 0, 0, 546, 0, - 480, 455, 731, 0, 0, 478, 423, 513, 466, 519, - 500, 527, 472, 467, 318, 501, 365, 436, 334, 336, - 721, 367, 370, 374, 375, 445, 446, 460, 485, 504, - 505, 506, 364, 348, 479, 349, 384, 350, 319, 356, - 354, 357, 487, 358, 321, 461, 510, 0, 380, 475, - 431, 322, 430, 462, 509, 508, 335, 536, 543, 544, - 634, 0, 549, 732, 733, 734, 558, 0, 468, 331, - 330, 0, 0, 0, 360, 463, 344, 346, 347, 345, - 458, 459, 563, 564, 565, 567, 0, 568, 569, 0, - 0, 0, 0, 570, 635, 651, 619, 588, 551, 643, - 585, 589, 590, 401, 402, 403, 654, 0, 0, 0, - 542, 416, 417, 0, 372, 371, 432, 323, 0, 0, - 409, 400, 469, 329, 368, 411, 405, 418, 419, 420, - 378, 313, 314, 727, 361, 451, 656, 691, 692, 581, - 0, 644, 582, 591, 353, 616, 628, 627, 447, 541, - 0, 639, 642, 571, 726, 0, 636, 650, 730, 649, - 723, 457, 0, 484, 647, 594, 0, 640, 613, 614, - 0, 641, 609, 645, 0, 583, 0, 552, 555, 584, - 669, 670, 671, 320, 554, 673, 674, 675, 676, 677, - 678, 679, 672, 524, 617, 593, 620, 533, 596, 595, - 0, 0, 631, 550, 632, 633, 441, 442, 443, 444, - 382, 657, 342, 553, 471, 0, 618, 0, 0, 0, - 0, 0, 0, 0, 0, 623, 624, 621, 735, 0, - 680, 681, 0, 0, 547, 548, 377, 0, 566, 385, - 341, 456, 379, 531, 408, 0, 559, 625, 560, 473, - 474, 683, 688, 684, 685, 687, 707, 448, 399, 404, - 488, 410, 424, 476, 530, 454, 481, 339, 520, 490, - 429, 610, 638, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 300, 301, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 4372, 0, 0, + 0, 2263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 665, 664, 663, 662, 661, 660, 659, 658, 0, 0, - 607, 507, 355, 307, 351, 352, 359, 724, 720, 725, - 708, 711, 710, 774, 0, 315, 587, 422, 470, 376, - 652, 653, 0, 706, 259, 260, 261, 262, 263, 264, - 265, 266, 308, 267, 268, 269, 270, 271, 272, 273, - 278, 279, 280, 281, 282, 283, 284, 285, 655, 276, - 277, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 298, 299, 0, 0, 0, 0, 309, - 712, 713, 714, 715, 716, 0, 0, 310, 311, 312, - 0, 0, 274, 275, 302, 498, 303, 304, 305, 306, - 0, 0, 537, 538, 539, 562, 0, 540, 522, 586, - 386, 316, 502, 529, 722, 0, 0, 0, 0, 0, - 0, 0, 637, 648, 682, 0, 694, 695, 697, 699, - 698, 701, 495, 496, 709, 0, 0, 703, 704, 705, - 702, 426, 482, 503, 489, 0, 728, 577, 578, 729, - 690, 317, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 2230, + 3231, 2229, 0, 0, 0, 3230, 0, 0, 0, 0, + 2251, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2263, 2257, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 2245, 2279, 0, 0, 2246, 2248, 2250, 0, 2252, + 2253, 2254, 2258, 2259, 2260, 2262, 2265, 2266, 2267, 0, + 0, 0, 0, 0, 0, 0, 2255, 2264, 2256, 2251, + 0, 0, 0, 0, 0, 0, 0, 0, 2234, 0, + 2257, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2245, 2279, 0, 0, 2246, 2248, 2250, 0, 2252, 2253, + 2254, 2258, 2259, 2260, 2262, 2265, 2266, 2267, 2251, 0, + 2271, 0, 0, 0, 0, 2255, 2264, 2256, 0, 2257, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 2245, + 2279, 0, 0, 2246, 2248, 2250, 0, 2252, 2253, 2254, + 2258, 2259, 2260, 2262, 2265, 2266, 2267, 2227, 2228, 0, + 0, 0, 0, 0, 2255, 2264, 2256, 0, 0, 2271, + 0, 0, 0, 0, 0, 2268, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 2244, 0, 0, 0, 2243, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2271, 0, + 0, 2261, 0, 0, 0, 0, 0, 0, 0, 0, + 2249, 0, 0, 0, 2268, 0, 0, 0, 0, 0, + 0, 0, 0, 2276, 2275, 0, 0, 0, 0, 0, + 0, 0, 2244, 0, 0, 0, 2243, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2261, 0, 0, 2268, 0, 0, 0, 0, 0, 2249, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2236, 2244, 0, 0, 0, 2243, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 2261, + 0, 0, 0, 0, 0, 0, 0, 0, 2249, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2277, } var yyPact = [...]int{ - 4916, -1000, -1000, -1000, -399, 18716, -1000, -1000, -1000, -1000, + 4765, -1000, -1000, -1000, -404, 18559, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 61886, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 61743, + -1000, -1000, -1000, -1000, -1000, -1000, 566, 61886, -401, -1000, + 3443, 1234, -1000, -1000, -1000, 446, 60444, 20744, 61886, 772, + 770, 67654, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, 457, 61743, -396, -1000, - 3747, 1147, -1000, -1000, -1000, 339, 60311, 20886, 61743, 647, - 645, 67471, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, 1207, -1000, 66933, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 1037, + 5820, 66212, 14210, -268, -1000, 2036, -66, 3273, 618, 5, + 3, 757, 1455, 1476, 1537, 1402, 61886, 1395, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, 1084, -1000, 66755, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 981, - 5628, 66039, 14397, -273, -1000, 1759, -65, 3182, 715, -3, - -4, 622, 1330, 1338, 1512, 1461, 61743, 1290, -1000, -1000, + -1000, 4795, 35918, 61165, 1272, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, 5180, 35955, 61027, 1156, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, 5208, 510, 1203, 1272, 26534, 215, 214, 2036, 3719, + -77, 455, -1000, 2287, 4921, 213, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, 14210, 14210, 18559, + -439, 18559, 14210, 61886, 61886, -1000, -1000, -1000, -1000, -401, + 60444, 1037, 5820, 14210, 3273, 618, 5, 3, 757, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, 5237, 344, 1082, 1156, 26636, 23, 16, 1759, 3463, - -144, 609, -1000, 1931, 5130, 214, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, 14397, 14397, 18716, - -435, 18716, 14397, 61743, 61743, -1000, -1000, -1000, -1000, -396, - 60311, 981, 5628, 14397, 3182, 715, -3, -4, 622, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -77, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -144, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, @@ -8931,8 +9038,8 @@ var yyPact = [...]int{ -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 214, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 16, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, @@ -8951,480 +9058,482 @@ var yyPact = [...]int{ -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, 456, -1000, 1959, + -1000, -1000, 353, -1000, 2108, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, 2866, 3734, 1955, 3178, - -1000, -1000, -1000, -1000, 1759, 4132, 59595, -1000, -1000, 4113, - -1000, 61743, 148, 61743, 158, 2374, -1000, 849, 766, 745, - 828, 375, 1954, -1000, -1000, -1000, -1000, -1000, -1000, 848, - 4112, -1000, 61743, 61743, 61743, 3745, 61743, -1000, 354, 878, - -1000, 5878, 3942, 1828, 1103, 3763, -1000, -1000, 3732, -1000, - 384, 883, 368, 677, 451, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 330, -1000, 4018, -1000, -1000, 359, -1000, -1000, - 374, -1000, -1000, -1000, 14, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -85, -1000, -1000, 1419, - 2572, 14397, 2601, -1000, 4143, 2077, -1000, -1000, -1000, 9364, - 17987, 17987, 17987, 17987, 61743, -1000, -1000, 3580, 14397, 3727, - 3726, 3725, 3723, -1000, -1000, -1000, -1000, -1000, -1000, 3722, - 1945, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - 2488, -1000, -1000, -1000, 17269, -1000, 3721, 3719, 3718, 3716, - 3715, 3714, 3713, 3712, 3709, 3708, 3707, 3706, 3704, 3703, - 3702, 3409, 20159, 3700, 3165, 3164, 3699, 3698, 3697, 3158, - 3696, 3695, 3694, 3409, 3409, 3693, 3690, 3689, 3684, 3683, - 3682, 3681, 3680, 3679, 3672, 3671, 3669, 3668, 3666, 3665, - 3662, 3655, 3641, 3635, 3634, 3632, 3624, 3622, 3620, 3619, - 3617, 3616, 3615, 3614, 3612, 3608, 3602, 3601, 3600, 3596, - 3594, 3593, 3592, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, 3019, 3871, 2101, 3270, -1000, -1000, -1000, -1000, 2036, + 4268, 59723, -1000, -1000, 4250, -1000, 61886, 146, 61886, 358, + 2464, -1000, 774, 758, 732, 1209, 484, 2097, -1000, -1000, + -1000, -1000, -1000, -1000, 918, 4247, -1000, 61886, 61886, 61886, + 3884, 61886, -1000, 437, 953, -1000, 5859, 4090, 1786, 1194, + 3908, -1000, -1000, 3870, -1000, 494, 539, 481, 656, 564, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, 457, -1000, 4148, + -1000, -1000, 488, -1000, -1000, 458, -1000, -1000, -1000, 158, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -15, -1000, -1000, 1605, 2627, 14210, 2534, -1000, 4669, + 2189, -1000, -1000, -1000, 9142, 17825, 17825, 17825, 17825, 61886, + -1000, -1000, 3724, 14210, 3866, 3865, 3862, 3861, -1000, -1000, + -1000, -1000, -1000, -1000, 3860, 3858, 2067, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, 2616, -1000, -1000, -1000, + 17102, -1000, 3857, 3856, 3855, 3850, 3849, 3848, 3846, 3845, + 3844, 3843, 3842, 3840, 3839, 3838, 3837, 3574, 20012, 3836, + 3268, 3266, 3835, 3831, 3829, 3265, 3828, 3824, 3823, 3574, + 3574, 3821, 3820, 3819, 3818, 3817, 3816, 3815, 3814, 3813, + 3812, 3811, 3807, 3801, 3799, 3798, 3796, 3795, 3794, 3788, + 3786, 3783, 3782, 3778, 3773, 3772, 3771, 3770, 3769, 3768, + 3767, 3765, 3764, 3755, 3753, 3751, 3749, 3747, 3746, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, 1777, -1000, 3590, 4135, - 3472, -1000, 4003, 4001, 3999, 3992, -332, 3589, 2725, -1000, - -1000, 95, 61743, 61743, 300, 61743, -355, 431, 536, -157, - -160, 527, -162, 1247, -1000, 493, -1000, -1000, 1458, -1000, - 1257, 65323, 1043, -1000, -1000, 61743, 975, 975, 975, 975, - 61743, 189, 1027, 1240, 975, 975, 975, 975, 995, 975, - 4034, 1081, 1078, 1076, 1071, 975, -105, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, 2373, 2368, 3841, 927, 59595, 61743, - -1000, 1802, 61743, -1000, 3535, 1217, -1000, -1000, -1000, -1000, - 431, -1000, -43, -384, 3761, 2131, 2131, 4095, 4095, 4032, - 4031, 894, 889, 887, 2131, 733, -1000, 2259, 2259, 2259, - 2259, 2131, 469, 905, 4038, 4038, 5, 2259, 3, 2131, - 2131, 3, 2131, 2131, 478, -1000, 2355, 529, 233, -343, - -1000, -1000, -1000, -1000, 2259, 2259, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 4011, 4010, 981, 981, 61743, 981, 61743, - 334, 187, 61743, 981, 981, 981, 61743, 987, -383, -63, - 64607, 63891, 2822, 354, 877, 862, 1804, 2309, -1000, 2201, - 61743, 61743, 2201, 2201, 30227, 29511, -1000, 61743, -1000, 4135, - 3472, 3387, 2185, 3385, 3472, -165, 981, 981, 981, 981, - 981, 981, 981, 313, 981, 981, 981, 981, 981, 61743, - 61743, 58879, 981, 516, 981, 981, 981, 12236, 1931, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 18716, 2531, 2562, 212, 2, -374, 277, -1000, - -1000, 61743, 3902, 2042, -1000, -1000, -1000, 3534, 3522, -1000, - 3529, 3529, 3529, 3529, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, 3529, 3529, 3533, 3587, -1000, -1000, - 3528, 3528, 3528, 3522, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 1359, - 3531, 3532, 3532, 3531, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 61743, 4140, -1000, -1000, 14397, 61743, 3926, 4135, - 3911, 4038, 4087, 927, 2535, -1000, -1000, 61743, 320, -1000, - 1944, 2720, 3157, -1000, 375, -1000, 722, 375, -1000, 798, - 798, 2243, -1000, 1510, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, 61743, -85, 712, -1000, -1000, -1000, 3115, 3585, -1000, - 732, 1607, 1628, -1000, 564, 6011, 48133, 354, 48133, 61743, - -1000, -1000, -1000, -1000, -1000, -1000, 8, -1000, -1000, -1000, + -1000, -1000, 1764, -1000, 3745, 4279, 3625, -1000, 4136, 4134, + 4130, 4125, -338, 3739, 2913, -1000, -1000, 91, 61886, 61886, + 301, 61886, -358, 477, 657, -103, -104, 645, -106, 1160, + -1000, 613, -1000, -1000, 1499, -1000, 1363, 65491, 1140, -1000, + -1000, 61886, 1033, 1033, 1033, 1033, 61886, 334, 1152, 1295, + 1033, 1033, 1033, 1033, 1149, 1033, 4163, 1196, 1195, 1182, + 1179, 1033, -38, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + 2463, 2461, 3993, 1005, 59723, 61886, -1000, 1910, 61886, -1000, + 3679, 1332, -1000, -1000, -1000, -1000, 477, -1000, 90, -389, + 3905, 2220, 2220, 4231, 4231, 4161, 4160, 978, 965, 961, + 2220, 860, -1000, 2360, 2360, 2360, 2360, 2220, 610, 958, + 4168, 4168, 291, 2360, 144, 2220, 2220, 144, 2220, 2220, + 616, -1000, 2338, 655, 359, -345, -1000, -1000, -1000, -1000, + 2360, 2360, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 4128, + 4124, 1037, 1037, 61886, 1037, 61886, 336, 254, 61886, 1037, + 1037, 1037, 61886, 1059, -387, 77, 64770, 64049, 2847, 437, + 938, 937, 1919, 2429, -1000, 2262, 61886, 61886, 2262, 2262, + 30150, 29429, -1000, 61886, -1000, 4279, 3625, 3533, 2428, 3525, + 3625, -107, 1037, 1037, 1037, 1037, 1037, 1037, 1037, 443, + 1037, 1037, 1037, 1037, 1037, 61886, 61886, 59002, 1037, 626, + 1037, 1037, 1037, 12034, 2287, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 18559, 2651, + 2589, 212, -63, -376, 275, -1000, -1000, 61886, 4041, 2184, + -1000, -1000, -1000, 3678, 3668, -1000, 3671, 3671, 3671, 3671, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + 3671, 3671, 3671, 3671, 3671, 3671, 3677, 3737, -1000, -1000, + 3670, 3670, 3670, 3668, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 1514, + 3672, 3673, 3673, 3672, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, 382, -1000, 14397, 14397, 14397, 14397, 14397, -1000, 1003, - 16551, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 17987, 17987, - 17987, 17987, 17987, 17987, 17987, 17987, 17987, 17987, 17987, 17987, - 17987, 17987, 3579, 2343, 17987, 17987, 17987, 17987, 350, 32375, - 2185, 3629, 1803, 326, 2077, 2077, 2077, 2077, 14397, -1000, - 2311, 2572, 14397, 14397, 14397, 14397, 39535, 61743, -1000, -1000, - 9364, 4793, 14397, 14397, 5935, 17987, 14397, 3988, 14397, 14397, - 14397, 3384, 7188, 61743, 14397, -1000, 3380, 3373, -1000, -1000, - 2538, 14397, -1000, -1000, 14397, -1000, -1000, 14397, 17987, 14397, - -1000, 14397, 14397, 14397, -1000, -1000, 694, 694, 1075, 3988, - 3988, 3988, 2289, 14397, 14397, 3988, 3988, 3988, 2244, 3988, - 3988, 3988, 3988, 3988, 3988, 3988, 3988, 3988, 3988, 3988, - 3988, 3988, 3367, 3365, 3364, 3363, 14397, 3356, 14397, 14397, - 14397, 14397, 14397, 13679, 4038, -273, -1000, 11518, 3911, 4038, + -1000, -1000, 61886, 4273, -1000, -1000, 14210, 61886, 4083, 4279, + 4074, 4168, 4225, 1005, 2804, -1000, -1000, 61886, 325, -1000, + 2066, 2891, 3264, -1000, 484, -1000, 751, 484, -1000, 812, + 812, 2350, -1000, 1557, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, 61886, -15, 1958, -1000, -1000, -1000, 3226, 3736, -1000, + 859, 1636, 1830, -1000, 521, 5765, 48181, 437, 48181, 61886, + -1000, -1000, -1000, -1000, -1000, -1000, 156, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -334, 3584, 61743, 3153, 3152, -408, -409, 1381, -409, 1941, - -1000, -356, 1307, 292, 61743, -1000, -1000, 61743, 3151, 2718, - 61743, 3150, 2716, 228, 213, 61743, 61743, 61743, -53, 1323, - 1248, 1264, -1000, -1000, 61743, 63175, -1000, 61743, 2404, 61743, - 61743, 61743, 3975, -1000, 61743, 61743, 975, 975, 975, -1000, - 56015, 3145, 48133, 61743, 61743, 354, 61743, 61743, 61743, 975, - 975, 975, 975, 61743, -1000, 3860, 48133, 3845, 3328, 3583, - 927, -1000, 61743, 1802, 3967, 61743, 987, -1000, -1000, -1000, - 4030, -1000, -1000, -1000, 859, 4095, 17987, 17987, -1000, -1000, - 14397, -1000, 224, 58163, 2259, 2131, 2131, -1000, -1000, 61743, - -1000, -1000, -1000, 2259, 61743, 2259, 2259, 4095, 2259, -1000, - -1000, -1000, 2131, 2131, -1000, -1000, 14397, -1000, -1000, 2259, - 2259, -1000, -1000, 4095, 61743, 1, 4095, 4095, -17, -1000, - -1000, 61743, -1000, 2131, 3143, -1000, 61743, 61743, 975, 61743, - -1000, 61743, 61743, -1000, -1000, 61743, 61743, 5939, 61743, 438, - 3938, 1130, 56015, 57447, 4009, -1000, 48133, 61743, 61743, 1797, - -1000, 1036, 43115, -1000, 61743, 1727, -1000, -54, -1000, -58, - -63, 2201, -63, 2201, 1032, -1000, 720, 436, 28079, 641, - 48133, 8635, -1000, -1000, 2201, 2201, 8635, 8635, 2019, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, 1791, -1000, 242, 4038, - -1000, -1000, -1000, -1000, -1000, 2713, 56731, 61743, 61743, 56015, - 48133, 354, 61743, 981, 61743, 61743, 61743, 61743, 61743, -1000, - 3581, 1917, -1000, 3935, 61743, 981, 61743, 61743, 61743, 1545, - -1000, -1000, 24466, 1916, -1000, -1000, 2384, -1000, 14397, 18716, - -308, 14397, 18716, 18716, 14397, 18716, -1000, 14397, 1988, -1000, - -1000, 4582, -1000, -1000, 2712, -1000, 2710, -1000, -1000, -1000, - -1000, -1000, 3140, 3140, -1000, 2709, -1000, -1000, -1000, -1000, - 3531, 2707, -1000, -1000, 2704, -1000, -1000, -1000, -1000, -201, - 3352, 1419, -1000, 3139, 4038, -1000, -279, 4081, 14397, 1822, - 981, -417, 2362, 2358, 2357, 4023, 61743, -1000, 4028, -1000, - -1000, 375, -1000, -1000, -1000, 798, 683, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, 1913, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -145, -146, 1790, - -1000, 61743, -1000, -1000, 564, 48133, 52429, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, 1738, -1000, -1000, 193, -1000, 1031, - 262, 2238, -1000, -1000, 195, 229, 221, 1221, 2572, -1000, - 2395, 2395, 2410, -1000, 788, -1000, -1000, -1000, -1000, 3580, - -1000, -1000, -1000, 3586, 2513, -1000, 2321, 2321, 2025, 2025, - 2025, 2025, 2025, 2305, 2305, 2077, 2077, -1000, -1000, -1000, - 9364, 3579, 17987, 17987, 17987, 17987, 1108, 1108, 5006, 4969, - -1000, -1000, 2009, 2009, -1000, -1000, -1000, -1000, 14397, 178, - 2381, -1000, 14397, 3257, 2184, 2831, 1843, 2231, -1000, 3522, - 14397, 1909, 2101, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, 518, -1000, 14210, 14210, 14210, 14210, 14210, -1000, 1041, + 16379, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 17825, 17825, + 17825, 17825, 17825, 17825, 17825, 17825, 17825, 17825, 17825, 17825, + 17825, 17825, 3717, 2426, 17825, 17825, 17825, 17825, 251, 32313, + 2428, 3775, 1918, 337, 2189, 2189, 2189, 2189, 14210, -1000, + 2493, 2627, 14210, 14210, 14210, 14210, 39523, 39523, 61886, -1000, + -1000, 9142, 5605, 14210, 14210, 6026, 17825, 14210, 4121, 14210, + 14210, 14210, 3523, 6951, 61886, 14210, -1000, 3516, 3512, -1000, + -1000, 2749, 14210, -1000, -1000, 14210, -1000, -1000, 14210, 17825, + 14210, -1000, 14210, 14210, 14210, -1000, -1000, 694, 694, 1348, + 4121, 4121, 4121, 2447, 14210, 14210, 4121, 4121, 4121, 2432, + 4121, 4121, 4121, 4121, 4121, 4121, 4121, 4121, 4121, 4121, + 4121, 4121, 4121, 3511, 3508, 3502, 3499, 14210, 3498, 14210, + 14210, 14210, 14210, 14210, 13487, 4168, -268, -1000, 11311, 4074, + 4168, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -340, 3733, 61886, 3261, 3260, -410, -411, 1385, -411, + 2061, -1000, -362, 1445, 290, 61886, -1000, -1000, 61886, 3258, + 2879, 61886, 3255, 2877, 343, 340, 61886, 61886, 61886, 94, + 1450, 1374, 1377, -1000, -1000, 61886, 63328, -1000, 61886, 2498, + 61886, 61886, 61886, 4103, -1000, 61886, 61886, 1033, 1033, 1033, + -1000, 56118, 3250, 48181, 61886, 61886, 437, 61886, 61886, 61886, + 1033, 1033, 1033, 1033, 61886, -1000, 4013, 48181, 4002, 3681, + 3728, 1005, -1000, 61886, 1910, 4102, 61886, 1059, -1000, -1000, + -1000, 4159, -1000, -1000, -1000, 935, 4231, 17825, 17825, -1000, + -1000, 14210, -1000, 372, 58281, 2360, 2220, 2220, -1000, -1000, + 61886, -1000, -1000, -1000, 2360, 61886, 2360, 2360, 4231, 2360, + -1000, -1000, -1000, 2220, 2220, -1000, -1000, 14210, -1000, -1000, + 2360, 2360, -1000, -1000, 4231, 61886, 148, 4231, 4231, 125, + -1000, -1000, 61886, -1000, 2220, 3245, -1000, 61886, 61886, 1033, + 61886, -1000, 61886, 61886, -1000, -1000, 61886, 61886, 6088, 61886, + 486, 4089, 1217, 56118, 57560, 4120, -1000, 48181, 61886, 61886, + 1908, -1000, 1137, 43128, -1000, 61886, 1808, -1000, 82, -1000, + 92, 77, 2262, 77, 2262, 1136, -1000, 856, 705, 27987, + 773, 48181, 8408, -1000, -1000, 2262, 2262, 8408, 8408, 2135, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, 1907, -1000, 303, + 4168, -1000, -1000, -1000, -1000, -1000, 2876, 56839, 61886, 61886, + 56118, 48181, 437, 61886, 1037, 61886, 61886, 61886, 61886, 61886, + -1000, 3725, 2044, -1000, 4088, 61886, 1037, 61886, 61886, 61886, + 1805, -1000, -1000, 24349, 2041, -1000, -1000, 2486, -1000, 14210, + 18559, -311, 14210, 18559, 18559, 14210, 18559, -1000, 14210, 1815, + -1000, -1000, 473, -1000, -1000, 2875, -1000, 2873, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, 3244, 3244, -1000, + 2872, -1000, -1000, -1000, -1000, 3672, 2871, -1000, -1000, 2870, + -1000, -1000, -1000, -1000, -195, 3497, 1605, -1000, 3243, 4168, + -1000, -283, 4220, 14210, 1780, 1037, -421, 2460, 2450, 2449, + 4153, 61886, -1000, 4158, -1000, -1000, 484, -1000, -1000, -1000, + 812, 614, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 2021, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 3351, 3349, 2799, 4111, 3729, 3347, 14397, - -1000, -1000, 2228, 2227, 2224, -1000, 2605, 12961, -1000, -1000, - -1000, 3339, 1893, 3334, -1000, -1000, -1000, 3333, 2223, 1577, - 3332, 2907, 3330, 3329, 3325, 3313, 1783, 1782, 1781, -1000, - -1000, -1000, -1000, 14397, 14397, 14397, 14397, 3310, 2222, 2220, - 14397, 14397, 14397, 14397, 3309, 14397, 14397, 14397, 14397, 14397, - 14397, 14397, 14397, 14397, 14397, 61743, 14397, 14397, 98, 98, - 98, 98, 3597, 98, 2110, 2085, 3576, 3526, 2169, 1779, - 1778, -1000, -1000, 2219, -1000, 2572, -1000, -1000, 4081, -1000, - 3575, 2702, 1741, -1000, -1000, -392, 3032, 1025, 61743, -357, - 61743, 1025, 61743, 61743, 2356, 1025, 61743, -358, 3138, -1000, - -1000, -1000, 3125, -1000, -1000, 61743, 61743, 61743, 61743, -171, - 3925, 3921, -1000, -1000, 1293, 1233, 1421, -1000, 61743, -1000, - 3122, 3934, 4027, 1038, -151, 61743, 3574, 3571, 61743, 61743, - 61743, 323, -1000, -1000, 61743, 1534, -1000, 262, -95, 649, - 1495, 3744, 963, 4139, 61743, 61743, 61743, 61743, 3966, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, 3760, -276, -1000, - 25920, 61743, 61743, 3328, -1000, 3570, 2209, -1000, 55299, 4044, - 61743, 354, -1000, 2077, 2077, 2572, 61743, 61743, 61743, 3743, - 61743, 61743, 4095, 4095, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, 2259, 4095, 4095, 1827, 2131, 2259, -1000, -1000, 2259, - -417, -1000, 2259, -1000, -1000, -1000, -417, 1891, -417, 61743, - -1000, -1000, -1000, 3964, 3535, 1740, -1000, -1000, -1000, 4085, - 1424, 961, 961, 1295, 714, 4082, 23034, -1000, 2218, 1602, - 1023, 3873, 371, -1000, 2218, -197, 934, 2218, 2218, 2218, - 2218, 2218, 2218, 2218, 839, 832, 2218, 2218, 2218, 2218, - 2218, 2218, 2218, 2218, 2218, 2218, 2218, 1336, 2218, 2218, - 2218, 2218, 2218, -1000, 2218, 3568, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 871, 773, -1000, -1000, 303, 354, 1022, - -28, -29, 287, 4008, 415, -1000, 409, 1534, 724, 4007, - 448, 61743, 61743, 1160, 1708, -1000, -1000, -1000, -1000, -1000, - 33091, 33091, 27363, 33091, -1000, 215, 2201, -63, -70, -1000, - -1000, 1727, 8635, 1727, 8635, 2701, -1000, -1000, 1017, -1000, - -1000, 1495, -1000, 61743, 61743, -1000, -1000, 3567, 2348, -1000, - -1000, 20159, -1000, 8635, 8635, -1000, -1000, 35239, 61743, -1000, - -90, -1000, -69, 4081, -1000, -370, -1000, -1000, 61743, -1000, - 1462, -1000, -1000, 1726, 1495, 3759, 61743, 1462, 1462, 1462, - -1000, -1000, 21602, 61743, 61743, -1000, 3121, -1000, 4110, -370, - 4095, 12236, -1000, 43115, -1000, -1000, 54577, -1000, 53861, 2239, - -1000, 18716, 2482, 204, -1000, 273, -379, 202, 2415, 201, - 2572, -1000, -1000, 3302, 3301, 3298, 2207, -1000, 2200, 3296, - -1000, 2182, 2172, 2698, -1000, -2, 4081, 3120, 3911, -246, - 1724, -1000, 2804, -1000, -276, -1000, 25193, -1000, 61743, 61743, - 3116, -1000, 14397, 53145, 14397, 1198, 1861, 151, -1000, -1000, - -1000, 61743, 3115, 2171, 52429, 1597, -1000, 1001, 1858, 1857, - -1000, 48133, 369, 48133, -1000, 48133, -1000, -1000, 4072, -1000, - 61743, 3916, -1000, -1000, -1000, 3032, 2339, -415, 61743, -1000, - -1000, -1000, -1000, -1000, 2147, -1000, 1108, 1108, 5006, 3049, - -1000, 17987, -1000, 17987, -1000, -1000, -1000, -1000, 3519, -1000, - 2210, -1000, 14397, 2472, 350, 14397, 350, 2012, 31659, 39535, - -175, 3915, 3495, 61743, 14397, -1000, -1000, 14397, 14397, 17987, - -1000, 3488, -1000, -1000, -1000, -1000, 14397, 14397, 2732, -1000, - 61743, -1000, -1000, -1000, -1000, 31659, -1000, 17987, -1000, -1000, - -1000, -1000, 14397, 14397, 14397, 1702, 1702, 3479, 2145, 98, - 98, 98, 3464, 3457, 3448, 2144, 98, 3440, 3417, 3386, - 3382, 3372, 3348, 3341, 3315, 3308, 3293, 2133, 3289, 2149, - -1000, 3565, -1000, -1000, -1000, 98, -1000, 98, 14397, 98, - 14397, 98, 98, 14397, 2470, 15833, 11518, -1000, 3911, 317, - 1720, 2697, 3112, 132, -1000, 2338, -1000, 442, -1000, 61743, - 4109, -1000, 1854, 3097, 51713, -1000, 1312, 61743, -1000, -1000, - 4108, 4107, -1000, -1000, 61743, 61743, 61743, -1000, -1000, -1000, - 1228, -1000, 3095, -1000, 370, 208, 2614, 2295, 3091, 341, - 1578, 21602, 3535, 3563, 3535, 137, 2218, 561, 723, 48133, - 857, -1000, 50997, 2516, 2337, 3757, 1039, 3900, 61743, 50281, - 3561, 1356, 3559, 3552, 3963, 575, 4582, -1000, 3880, 1440, - -1000, 3551, -1000, 2107, 3837, -1000, 1680, -1000, 2332, 2068, - -1000, -1000, 5130, -1000, 61743, 61743, 1536, -1000, 1849, -1000, - 2696, -1000, -1000, -1000, -1000, 61743, -1000, 354, -1000, 2131, - -1000, -1000, 4095, -1000, -1000, 14397, 14397, 4095, 2131, 2131, - -1000, 2259, -1000, 61743, -1000, -417, 575, 4582, 3961, 6299, - 760, 3392, -1000, 61743, -1000, -1000, -1000, 1030, -1000, 1268, - 975, 61743, 2434, 1268, 2433, 3546, -1000, -1000, 61743, 61743, - 61743, 61743, -1000, -1000, 61743, -1000, 61743, 61743, 61743, 61743, - 61743, 49565, -1000, 61743, 61743, -1000, 61743, 2426, 61743, 2423, - 3978, -1000, 2218, 2218, 1168, -1000, -1000, 680, -1000, 49565, - 2694, 2691, 2690, 2680, 3090, 3089, 3088, 2218, 2218, 2679, - 3086, 48849, 3085, 1574, 2678, 2676, 2674, 2649, 3083, 1200, - -1000, 3081, 2594, 2592, 2581, 61743, 3541, 2979, -1000, -1000, - 2614, 3080, 3538, 2673, 3076, 1093, 354, 3063, 3756, 137, - 2218, 413, 61743, 2322, 2320, 723, 703, 703, 644, -96, - 28795, -1000, -1000, -1000, 61743, 43115, 43115, 43115, 43115, 43115, - 43115, -1000, 3796, 3779, 3536, -1000, 3782, 3781, 3780, 643, - 3795, 3750, 61743, 43115, 3535, -1000, 48849, -1000, -1000, -1000, - 2185, 2061, 1412, 1234, 14397, 8635, -1000, -1000, -61, -64, - -1000, -1000, -1000, -1000, 48133, 3060, 641, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, 3911, -1000, -1000, 61743, 61743, 993, - 3294, 1710, -1000, -1000, -1000, 4582, 3534, 3529, 3529, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 3529, 3529, - 3533, -1000, -1000, 3528, 3528, 3528, 3522, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, 1359, 3531, 3532, 3532, - 3531, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -78, -84, 1906, -1000, 61886, -1000, -1000, 521, + 48181, 52507, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 1809, + -1000, -1000, 195, -1000, 1133, 384, 2340, -1000, -1000, 259, + 224, 362, 1310, 2627, -1000, 2556, 2556, 2509, -1000, 906, + -1000, -1000, -1000, -1000, 3724, -1000, -1000, -1000, 5048, 2933, + -1000, 2385, 2385, 2147, 2147, 2147, 2147, 2147, 2504, 2504, + 2189, 2189, -1000, -1000, -1000, 9142, 3717, 17825, 17825, 17825, + 17825, 1167, 1167, 3071, 5014, -1000, -1000, 2171, 2171, -1000, + -1000, -1000, -1000, 14210, 181, 2475, -1000, 14210, 3513, 2201, + 3286, 2005, 2329, -1000, 3668, 14210, 2328, 2008, 4174, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 61743, -1000, 4091, -1000, 1691, -1000, -1000, 1848, - -1000, 2388, -404, 18716, 2242, 2070, -1000, 14397, 18716, 14397, - -318, 393, -321, -1000, -1000, -1000, -1000, 3042, -1000, -1000, - -1000, 2671, -1000, 2668, -1000, 117, 216, 3911, 211, -1000, - 4136, 14397, 3865, -1000, -1000, 1440, 2057, 3836, 1680, 4135, - -1000, 174, -425, -426, 165, 3036, 61743, 2661, -1000, -1000, - -1000, 4106, 48133, 354, 2039, 47417, -1000, 357, -1000, 1709, - 664, 3035, -1000, 1070, 131, 3033, 3032, -1000, -1000, -1000, - -1000, 17987, 2077, -1000, -1000, -1000, 2572, 14397, 3282, 2644, - 3280, 3275, -1000, 3529, 3529, -1000, 3522, 3528, 3522, 2009, - 2009, 3274, -1000, 3515, -1000, 3915, -1000, 2050, 2738, 3239, - 4910, -1000, 3235, 3180, 14397, -1000, 3270, 4865, 1831, 1674, - 3161, -108, -230, 98, 98, -1000, -1000, -1000, -1000, 98, - 98, 98, 98, -1000, 98, 98, 98, 98, 98, 98, - 98, 98, 98, 98, 98, 98, 14397, 931, -1000, -1000, - 1989, -1000, 1895, -1000, -1000, 3146, -113, -349, -114, -350, - -1000, -1000, 3261, 1677, -1000, -1000, -1000, -1000, -1000, 5935, - 1675, 663, 663, 3032, 3027, 61743, 3026, -361, 61743, -1000, - -430, -431, -362, 61743, 3025, 61743, 61743, -13, 2226, 2471, - -1000, 3023, -1000, -1000, 46701, 61743, 61743, 62459, 762, 61743, - 61743, 3022, -1000, -202, 3514, -153, 3021, 3259, 1667, -1000, - -1000, 61743, -1000, -1000, -1000, 3258, 3959, 22318, 3958, 2748, - -1000, -1000, -1000, 34523, 61743, 703, -1000, -1000, -1000, 794, - 351, 2655, 662, -1000, 61743, 578, 440, 3851, 2318, 3020, - 61743, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, 3900, -1000, 1520, -417, 61743, 560, 41683, 19443, -1000, - 3291, 61743, -1000, 61743, 45979, 22318, 22318, 3291, 563, 2274, - -1000, 2422, -276, 11518, 3429, 61743, -276, 61743, 11518, -1000, - 61743, 3256, -1000, 927, 1664, 142, 43115, 61743, -1000, 43831, - -1000, -1000, 1495, 4095, -1000, 2572, 2572, -417, 4095, 4095, - 2131, -1000, -1000, 563, -1000, 3291, -1000, 1544, 23750, 676, - 547, 434, -1000, 761, -1000, -1000, 926, 3874, 4582, -1000, - 61743, -1000, 61743, -1000, 61743, 61743, 975, 14397, 3874, 61743, - 998, -1000, 1364, 510, 530, 972, 972, 1610, -1000, 3915, - -1000, -1000, 1604, -1000, -1000, -1000, -1000, 61743, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, 31659, 31659, 4006, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, 3019, 3017, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 3495, + 3489, 3296, 4246, 4941, 3487, 14210, -1000, -1000, 2323, 2318, + 2315, -1000, 2711, 12764, -1000, -1000, -1000, 3486, 2003, 3484, + -1000, -1000, -1000, 3483, 2311, 1642, 3482, 2888, 3479, 3478, + 3472, 3471, 1887, 1879, 1866, -1000, -1000, -1000, -1000, 14210, + 14210, 14210, 14210, 3460, 2307, 2304, 14210, 14210, 14210, 14210, + 3458, 14210, 14210, 14210, 14210, 14210, 14210, 14210, 14210, 14210, + 14210, 61886, 14210, 14210, 258, 258, 258, 258, 3762, 258, + 2093, 2073, 3730, 3692, 2169, 1863, 1856, -1000, -1000, 2300, + -1000, 2627, -1000, -1000, 4220, -1000, 3715, 2863, 1836, -1000, + -1000, -398, 3172, 1129, 61886, -365, 61886, 1129, 61886, 61886, + 2444, 1129, 61886, -366, 3240, -1000, -1000, -1000, 3239, -1000, + -1000, 61886, 61886, 61886, 61886, -113, 4082, 4081, -1000, -1000, + 1420, 1357, 1477, -1000, 61886, -1000, 3238, 4087, 4157, 1162, + -95, 61886, 3714, 3710, 61886, 61886, 61886, 431, -1000, -1000, + 61886, 1674, -1000, 384, -28, 787, 1635, 3883, 1106, 4272, + 61886, 61886, 61886, 61886, 4101, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, 3904, -278, -1000, 25813, 61886, 61886, 3681, + -1000, 3708, 2297, -1000, 55397, 4172, 61886, 437, -1000, 2189, + 2189, 2627, 61886, 61886, 61886, 3881, 61886, 61886, 4231, 4231, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, 2360, 4231, 4231, + 1851, 2220, 2360, -1000, -1000, 2360, -421, -1000, 2360, -1000, + -1000, -1000, -421, 1971, -421, 61886, -1000, -1000, -1000, 4100, + 3679, 1810, -1000, -1000, -1000, 4224, 1750, 1029, 1029, 1352, + 893, 4222, 22907, -1000, 2231, 1556, 1117, 4033, 490, -1000, + 2231, -192, 1008, 2231, 2231, 2231, 2231, 2231, 2231, 2231, + 916, 911, 2231, 2231, 2231, 2231, 2231, 2231, 2231, 2231, + 2231, 2231, 2231, 1461, 2231, 2231, 2231, 2231, 2231, -1000, + 2231, 3707, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 925, + 873, -1000, -1000, 310, 437, 1108, 106, 102, 429, 4118, + 538, -1000, 534, 1674, 844, 4115, 563, 61886, 61886, 1946, + 1696, -1000, -1000, -1000, -1000, -1000, 33034, 33034, 27266, 33034, + -1000, 216, 2262, 77, 66, -1000, -1000, 1808, 8408, 1808, + 8408, 2861, -1000, -1000, 1099, -1000, -1000, 1635, -1000, 61886, + 61886, -1000, -1000, 3702, 2420, -1000, -1000, 20012, -1000, 8408, + 8408, -1000, -1000, 35197, 61886, -1000, -23, -1000, 54, 4220, + -1000, -372, -1000, -1000, 61886, -1000, 1628, -1000, -1000, 1804, + 1635, 3902, 61886, 1628, 1628, 1628, -1000, -1000, 21465, 61886, + 61886, -1000, 3236, -1000, 4245, -372, 4231, 12034, -1000, 43128, + -1000, -1000, 54670, -1000, 53949, 2383, -1000, 18559, 2569, 206, + -1000, 266, -379, 204, 2459, 203, 2627, -1000, -1000, 3457, + 3456, 3454, 2271, -1000, 2268, 3440, -1000, 2253, 2240, 2844, + -1000, 136, 4220, 3233, 4074, -241, 1787, -1000, 2731, -1000, + -278, -1000, 25081, -1000, 61886, 61886, 3231, -1000, 14210, 53228, + 14210, 1315, 1970, 357, -1000, -1000, -1000, 61886, 3226, 2225, + 52507, 1669, -1000, 1098, 1961, 1960, -1000, 48181, 474, 48181, + -1000, 48181, -1000, -1000, 4181, -1000, 61886, 4079, -1000, -1000, + -1000, 3172, 2412, -418, 61886, -1000, -1000, -1000, -1000, -1000, + 2223, -1000, 1167, 1167, 3071, 4775, -1000, 17825, -1000, 17825, + -1000, -1000, -1000, -1000, 3675, -1000, 2363, -1000, 14210, 2565, + 251, 14210, 251, 1539, 31592, 39523, -155, 4060, 3622, -157, + 61886, 14210, -1000, -1000, 14210, 14210, 17825, -1000, 3610, -1000, + -1000, -1000, -1000, 14210, 14210, 2843, -1000, 61886, -1000, -1000, + -1000, -1000, 31592, -1000, 17825, -1000, -1000, -1000, -1000, 14210, + 14210, 14210, 1761, 1761, 3584, 2216, 258, 258, 258, 3558, + 3553, 3532, 2202, 258, 3527, 3518, 3509, 3501, 3477, 3464, + 3423, 3418, 3407, 3399, 2200, 3351, 2119, -1000, 3701, -1000, + -1000, -1000, 258, -1000, 258, 14210, 258, 14210, 258, 258, + 14210, 2633, 15656, 11311, -1000, 4074, 309, 1773, 2842, 3220, + 134, -1000, 2404, -1000, 557, -1000, 61886, 4244, -1000, 1947, + 3217, 51786, -1000, 1418, 61886, -1000, -1000, 4243, 4242, -1000, + -1000, 61886, 61886, 61886, -1000, -1000, -1000, 1342, -1000, 3215, + -1000, 465, 387, 2756, 2492, 3214, 448, 1587, 21465, 3679, + 3699, 3679, 265, 2231, 710, 857, 48181, 927, -1000, 51065, + 2619, 2401, 3901, 2358, 4040, 61886, 50344, 3697, 1231, 3696, + 3691, 4099, 741, 473, -1000, 4056, 1624, -1000, 3690, -1000, + 2194, 3986, -1000, 1737, -1000, 2399, 2180, -1000, -1000, 4921, + -1000, 61886, 61886, 1803, -1000, 1938, -1000, 2841, -1000, -1000, + -1000, -1000, 61886, -1000, 437, -1000, 2220, -1000, -1000, 4231, + -1000, -1000, 14210, 14210, 4231, 2220, 2220, -1000, 2360, -1000, + 61886, -1000, -421, 741, 473, 4098, 68394, 837, 3535, -1000, + 61886, -1000, -1000, -1000, 1124, -1000, 1243, 1033, 61886, 2546, + 1243, 2536, 3688, -1000, -1000, 61886, 61886, 61886, 61886, -1000, + -1000, 61886, -1000, 61886, 61886, 61886, 61886, 61886, 49623, -1000, + 61886, 61886, -1000, 61886, 2529, 61886, 2526, 4066, -1000, 2231, + 2231, 1289, -1000, -1000, 824, -1000, 49623, 2840, 2837, 2835, + 2833, 3212, 3209, 3206, 2231, 2231, 2830, 3195, 48902, 3194, + 1614, 2829, 2823, 2822, 2790, 3193, 1299, -1000, 3192, 2788, + 2774, 2747, 61886, 3685, 3108, -1000, -1000, 2756, 3188, 3684, + 2821, 3187, 1174, 437, 3184, 3900, 265, 2231, 529, 61886, + 2397, 2395, 857, 800, 800, 775, -29, 28708, -1000, -1000, + -1000, 61886, 43128, 43128, 43128, 43128, 43128, 43128, -1000, 3959, + 3928, 3683, -1000, 3952, 3946, 3943, 778, 3954, 3912, 61886, + 43128, 3679, -1000, 48902, -1000, -1000, -1000, 2428, 2179, 1465, + 1343, 14210, 8408, -1000, -1000, 71, 69, -1000, -1000, -1000, + -1000, 48181, 3183, 773, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, 4074, -1000, -1000, 61886, 61886, 1077, 3439, 1772, -1000, + -1000, -1000, 473, 3678, 3671, 3671, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, 3671, 3671, 3671, 3671, 3671, + 3671, 3677, -1000, -1000, 3670, 3670, 3670, 3668, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, 1514, 3672, 3673, + 3673, 3672, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, 61886, -1000, 4229, -1000, 1747, -1000, -1000, + 1934, -1000, 2491, -405, 18559, 2470, 2342, -1000, 14210, 18559, + 14210, -312, 507, -315, -1000, -1000, -1000, -1000, 3179, -1000, + -1000, -1000, 2818, -1000, 2817, -1000, 326, 333, 4074, 344, + -1000, 4271, 14210, 4026, -1000, -1000, 1624, 2161, 3983, 1737, + 4279, -1000, 177, -431, -432, 173, 3178, 61886, 2816, -1000, + -1000, -1000, 4241, 48181, 437, 2155, 47460, -1000, 479, -1000, + 1693, 803, 3175, -1000, 1177, 133, 3174, 3172, -1000, -1000, + -1000, -1000, 17825, 2189, -1000, -1000, -1000, 2627, 14210, 3438, + 2839, 3430, 3421, -1000, 3671, 3671, -1000, 3668, 3670, 3668, + 2171, 2171, 3420, -1000, 3667, -1000, 4060, 3666, -1000, 2152, + 2669, 3338, 4737, -1000, 3329, 3317, 14210, -1000, 3419, 4687, + 2110, 2102, 3257, -42, -225, 258, 258, -1000, -1000, -1000, + -1000, 258, 258, 258, 258, -1000, 258, 258, 258, 258, + 258, 258, 258, 258, 258, 258, 258, 258, 14210, 1007, + -1000, -1000, 2039, -1000, 2000, -1000, -1000, 3204, -114, -351, + -115, -352, -1000, -1000, 3417, 1728, -1000, -1000, -1000, -1000, + -1000, 6026, 1700, 797, 797, 3172, 3171, 61886, 3170, -367, + 61886, -1000, -433, -434, -368, 61886, 3168, 61886, 61886, 126, + 2465, 2576, -1000, 3167, -1000, -1000, 46739, 61886, 61886, 62607, + 872, 61886, 61886, 3161, -1000, -196, 3664, -98, 3157, 3415, + 1689, -1000, -1000, 61886, -1000, -1000, -1000, 3414, 4096, 22186, + 4095, 2922, -1000, -1000, -1000, 34476, 61886, 800, -1000, -1000, + -1000, 912, 508, 2809, 792, -1000, 61886, 706, 556, 3999, + 2392, 3156, 61886, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, 4040, -1000, 1126, -421, 61886, 699, + 41686, 19291, -1000, 3394, 61886, -1000, 61886, 46012, 22186, 22186, + 3394, 727, 2349, -1000, 2523, -278, 11311, 3594, 61886, -278, + 61886, 11311, -1000, 61886, 3408, -1000, 1005, 1622, 140, 43128, + 61886, -1000, 43849, -1000, -1000, 1635, 4231, -1000, 2627, 2627, + -421, 4231, 4231, 2220, -1000, -1000, 727, -1000, 3394, -1000, + 2051, 23628, 820, 569, 567, -1000, 883, -1000, -1000, 1002, + 4017, 473, -1000, 61886, -1000, 61886, -1000, 61886, 61886, 1033, + 14210, 4017, 61886, 1093, -1000, 1460, 766, 793, 1080, 1080, + 1685, -1000, 4060, -1000, -1000, 1681, -1000, -1000, -1000, -1000, + 61886, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 31592, 31592, + 4112, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, 3153, 3149, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 61743, 2040, -1000, 2315, 3016, -153, 7917, -1000, - -1000, 997, -1000, 3753, 1068, 2748, 34523, 2299, 2201, 3015, - 3008, 703, -1000, 3005, 3000, -1000, 2516, 2297, 1065, 61743, - -1000, 1472, 61743, 61743, -1000, 1671, -1000, 2296, 3740, 3752, - 3740, -1000, 3740, -1000, -1000, -1000, -1000, 3792, 2999, -1000, - 3791, -1000, 3784, -1000, 3783, -1000, -1000, -1000, -1000, 1716, - -1000, -1000, -1000, -1000, -1000, 1234, -1000, 4026, 1268, 1268, - 1268, 3255, -1000, -1000, -1000, -1000, 1597, 3252, -1000, -1000, - 4025, -1000, -1000, -1000, -1000, -1000, -1000, 21602, 3893, 556, - 4089, 4080, 45263, -1000, -404, 2080, -1000, 2428, 199, 2403, - 61743, -1000, -1000, -1000, 3249, 3248, -282, 139, 4079, 4078, - 4025, -292, 2997, 355, -1000, -1000, 3879, 1446, -276, 4038, - -1000, -1000, -1000, -1000, -433, -1000, -1000, 354, -1000, 1660, - -1000, -1000, -1000, -1000, -1000, -1000, 240, -1000, 61743, -1000, - 1570, 127, -1000, 2572, -1000, 350, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, 2996, -1000, -1000, -1000, - 14397, -1000, -1000, -1000, -1000, 3124, -1000, -1000, 14397, 14397, - -1000, 3247, 2995, 3243, 2994, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 2989, 4135, -1000, 4077, 98, 14397, 98, - 14397, 98, 2034, 3238, 3237, 2021, 3236, 3233, -1000, 14397, - 3229, 5935, 1186, 2988, 1186, -1000, -1000, -1000, -1000, 61743, - -1000, -1000, -1000, 61743, 4105, 33807, 989, -417, 583, 3511, - -1000, 599, 2226, 1269, 3502, 2986, -1000, 61743, 4104, 61743, - 2614, 758, 2614, 784, 61743, -370, -155, 2654, 7917, -1000, - 2981, -1000, -178, 1578, 4582, 1033, 3291, 3227, 1546, -1000, - -1000, -1000, -1000, 3291, -1000, 2980, 261, -1000, -1000, -1000, - 506, -1000, 2653, -1000, -1000, 2568, 1961, 278, -1000, -1000, - -1000, -1000, -1000, -1000, 2657, 61743, 44547, 2657, 2739, 2293, - -418, -1000, 3500, -1000, 2218, 2218, 2218, 989, 554, 61743, - 2014, -1000, 2218, 2218, 3222, -1000, -1000, 3855, 61743, 3221, - 3215, 4134, 936, 2237, 2236, -1000, 2650, 1244, -1000, 3211, - 1497, -276, -1000, -1000, 1440, -1000, -1000, -1000, -1000, 33091, - 43115, 43831, 1524, -1000, 1846, -1000, -1000, -1000, -1000, -1000, - 4095, 936, -1000, 674, 2646, 17987, 3497, 17987, 3491, 711, - 3484, 1999, -1000, 61743, -1000, -1000, 61743, 4386, 3482, -1000, - 3474, 3742, 655, 3473, 3466, 61743, 2950, -1000, 3874, 61743, - 845, 3889, -1000, 443, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 778, -1000, 61743, -1000, 61743, -1000, 2002, -1000, - 31659, -1000, -1000, 1996, -1000, 2979, 2977, -1000, -1000, 3203, - 2572, -1000, 1759, 354, 1056, 61743, -1000, 261, 2976, 8635, - -1000, -1000, -1000, -1000, -1000, 3851, 2971, 2657, 61743, -1000, - 61743, 1472, 1472, 4135, 43115, 61743, 11518, -1000, -1000, 14397, - 3460, -1000, 14397, -1000, -1000, -1000, 3202, -1000, -1000, -1000, - -1000, -1000, -1000, 3456, 3849, -1000, -1000, -1000, -1000, -1000, - -1000, 4119, -1000, 1929, 61743, -1000, 14397, 15115, -1000, 964, - 18716, -325, 391, -1000, -1000, -1000, -284, 2970, -1000, -1000, - 4076, 2954, 2772, -1000, -2, 2953, -1000, 14397, -1000, -1000, - -1000, -276, -1000, 1440, -1000, -1000, 1495, -1000, -1000, 1521, - 841, -1000, 3199, 2211, -1000, 2917, -1000, 2903, 2892, 98, - -1000, 98, -1000, 98, 198, 14397, -1000, 2870, -1000, 2836, - -1000, -1000, 2951, -1000, -1000, -1000, 2947, -1000, -1000, 2819, - -1000, 3198, -1000, 2946, -1000, -1000, 2938, 2935, -365, -1000, - -1000, 439, 989, -1000, 416, 61743, 592, -1000, 42399, 7917, - -420, 552, 61743, 4100, 2934, 2614, 2932, 2614, 61743, 743, - -1000, 3948, 2924, -1000, 3196, -1000, 2922, 2921, -1000, -1000, - 4582, 4130, 4134, 22318, 4130, -1000, -1000, 4056, -1000, 1930, - 424, -1000, -1000, 2536, 713, -1000, -1000, 2910, 675, -1000, - 1472, -1000, -1000, 2291, 2458, 2832, 39535, 31659, 32375, 2909, - -1000, 61743, -1000, -1000, 41683, 1929, 1929, 6376, 989, 4016, - 545, 382, 6658, -1000, 3455, 1350, 2234, -1000, 2643, -1000, - 2641, -1000, 61743, -1000, -1000, 1440, 4095, 1524, 141, -1000, - -1000, 2032, -1000, 1350, 3392, 4075, -1000, 4749, 61743, 4611, - 61743, 3454, 2278, 17987, -1000, 926, 3834, -1000, -1000, 4386, - -1000, -1000, 2441, 17987, -1000, -1000, 2908, 32375, 1206, 2276, - 2273, 1164, 3451, -1000, 782, 4118, 2639, -1000, -1000, -1000, - 1146, 3444, -1000, -297, 3437, 2421, 2408, -1000, 61743, -1000, - 39535, 39535, 1110, 1110, 39535, 39535, 3435, 972, -1000, -1000, - 17987, -1000, -1000, -1000, 2270, 4848, 4848, 4848, 4848, -1000, - -1000, -1000, 2218, 1987, -1000, -1000, -1000, -1000, -1000, 61743, - 1842, -1000, -1000, -1000, 2739, -1000, -1000, 1462, -1000, 4038, - 1524, -1000, -1000, 2572, 61743, 2572, -1000, 40967, -1000, 4074, - 4070, -1000, -1000, -1000, 2572, 1523, 264, 3434, 3433, -1000, - -404, 61743, 61743, -286, 2637, -1000, 2899, 134, -1000, -1000, - 117, -1000, 1419, 1440, -288, -17, 31659, 2258, -1000, 3195, - 364, -185, -1000, -1000, -1000, -1000, -1000, -1000, 3194, -1000, - 1414, -1000, -1000, -1000, 1419, 98, 98, 3193, 3192, -1000, - -1000, -1000, -1000, -1000, 61743, 61743, -1000, 61743, 2896, 2635, - -1000, -1000, 1984, -1000, -1000, -1000, 2409, 2406, 1981, 3129, - 2816, 61743, 544, 61743, -370, 2889, -370, 2888, 739, 2614, - -339, -1000, -1000, -1000, -1000, -179, -1000, -1000, 418, -1000, - -1000, -1000, 684, 2783, 2625, -1000, -1000, 421, -1000, -1000, - -1000, 2657, 2886, -1000, -1000, 126, -1000, 2247, 1939, -1000, - -1000, -1000, 506, -1000, -1000, -1000, 921, -1000, 3291, 6577, - -1000, 1602, -1000, -1000, 61743, -1000, 1521, 921, 38103, 767, - 2316, -1000, 2624, -1000, -1000, 1417, 4135, -1000, 754, -1000, - 707, -1000, 1937, -1000, 1927, 40251, 2623, 4073, -1000, 6490, - 1077, -1000, -1000, 5006, -1000, -1000, -1000, -1000, -1000, -1000, - 2881, 2880, -1000, -1000, -1000, -1000, -1000, 2622, 3431, -89, - -1000, 3991, 2871, 3946, 14397, -1000, -1000, 3426, 1921, 1906, + -1000, -1000, -1000, -1000, -1000, 61886, 2141, -1000, 2391, 3148, + -98, 7685, -1000, -1000, 1073, -1000, 3898, 1172, 2922, 34476, + 2390, 2262, 3147, 3146, 800, -1000, 3143, 3142, -1000, 2619, + 2389, 1165, 61886, -1000, 1634, 61886, 61886, -1000, 1692, -1000, + 2386, 3878, 3897, 3878, -1000, 3878, -1000, -1000, -1000, -1000, + 3953, 3139, -1000, 3945, -1000, 3944, -1000, 3936, -1000, -1000, + -1000, -1000, 1763, -1000, -1000, -1000, -1000, -1000, 1343, -1000, + 4156, 1243, 1243, 1243, 3405, -1000, -1000, -1000, -1000, 1669, + 3402, -1000, -1000, 4155, -1000, -1000, -1000, -1000, -1000, -1000, + 21465, 4036, 696, 4227, 4217, 45291, -1000, -405, 2339, -1000, + 2490, 201, 2436, 61886, -1000, -1000, -1000, 3401, 3398, -285, + 347, 4215, 4214, 4155, -295, 3137, 476, -1000, -1000, 4029, + 1646, -278, 4168, -1000, -1000, -1000, -1000, -436, -1000, -1000, + 437, -1000, 1663, -1000, -1000, -1000, -1000, -1000, -1000, 377, + -1000, 61886, -1000, 1665, 129, -1000, 2627, -1000, 251, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 3135, + -1000, 3135, -1000, -1000, 14210, -1000, -1000, -1000, -1000, 3141, + -1000, -1000, 14210, 14210, -1000, 3387, 3118, 3386, 3115, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 1903, 1902, 39535, -1000, -1000, 5006, 4848, 2453, - -1000, 2218, 2218, 2868, 2867, 473, -1000, -1000, 2218, 2218, - 2218, 2218, 2218, 2218, 3425, 2864, 2862, 2218, 2218, 2218, - 2218, -1000, -1000, 2245, 2218, 2218, 31659, 2218, 1836, 61743, - -1000, -1000, -1000, 1820, 1819, -1000, -1000, -1000, -1000, -1000, - -378, 3423, 14397, 14397, -1000, -1000, -1000, 3421, -1000, -1000, - 4068, -282, -290, 2850, 115, 241, -1000, 2849, -1000, -181, - 3828, -188, -1000, -1000, 983, -277, 94, 87, 44, -1000, - -1000, -1000, 14397, -1000, -1000, -1000, -1000, 2848, -1000, -1000, - -1000, -1000, -1000, 61743, 2846, -1000, -1000, 114, -1000, 2229, - -1000, 61743, 539, -1000, -370, -1000, -370, 2614, 2834, -1000, - 61743, 725, -1000, -1000, -1000, -1000, 239, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, 2832, 2830, -1000, -1000, 666, 4067, - -1000, 6658, -1000, 2218, 506, -1000, 666, 1815, -1000, 2218, - 2218, -1000, 572, -1000, 2154, -1000, 2609, -1000, 4038, -1000, - 567, -1000, 668, -1000, -1000, -1000, 1766, -1000, -1000, -1000, - 6490, 687, -1000, 901, 3419, -1000, -1000, 2955, 14397, 3409, - 2218, 2875, 3398, 2815, -167, 39535, 3631, 3588, 3442, 3331, - 1729, -1000, -1000, 2602, 2593, -1000, -1000, 61743, 2582, 2569, - 2567, 2548, 2526, 2519, 61743, -1000, -1000, 2510, 2505, 2502, - 2500, 2448, 2492, 2480, -1000, 31659, 61743, -1000, -1000, -1000, - 38819, -1000, 3397, 1722, 1690, 61743, 2772, -284, -1000, 2817, - -1000, 985, 235, 241, -1000, 4066, 130, 4065, 4064, 1393, - 3818, -1000, -1000, 2396, -1000, 102, 100, 92, -1000, -1000, - -1000, -1000, -1000, 2327, 2327, -370, 2816, 2812, -1000, 61743, - -1000, -1000, 2811, -370, 596, -1000, 353, -1000, -1000, -1000, - 4848, -1000, 4062, 760, -1000, 31659, -1000, -1000, -1000, 38103, - 1929, 1929, -1000, -1000, 2475, -1000, -1000, -1000, -1000, 2473, - -1000, -1000, -1000, 1686, -1000, 61743, 1118, 10800, -1000, 2745, - -1000, 61743, -1000, 14397, -300, 3749, -1000, 297, 1683, 4848, - 1110, 4848, 1110, 4848, 1110, 4848, 1110, 348, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, 1645, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, 3052, 4279, -1000, + 4212, 258, 14210, 258, 14210, 258, 2126, 3382, 3374, 2106, + 3363, 3362, -1000, 14210, 3361, 6026, 1297, 3113, 1297, -1000, + -1000, -1000, -1000, 61886, -1000, -1000, -1000, 61886, 4238, 33755, + 1066, -421, 744, 3655, -1000, 754, 2465, 1401, 3654, 3112, + -1000, 61886, 4237, 61886, 2756, 870, 2756, 910, 61886, -372, + -100, 2808, 7685, -1000, 3111, -1000, -167, 1587, 473, 1154, + 3394, 3360, 1661, -1000, -1000, -1000, -1000, 3394, -1000, 3109, + 381, -1000, -1000, -1000, 638, -1000, 2802, -1000, -1000, 2740, + 1959, 415, -1000, -1000, -1000, -1000, -1000, -1000, 2763, 61886, + 44570, 2763, 2797, 2377, -423, -1000, 3648, -1000, 2231, 2231, + 2231, 1066, 689, 61886, 2070, -1000, 2231, 2231, 3352, -1000, + -1000, 4011, 61886, 3350, 3349, 4269, 1010, 2356, 2336, -1000, + 2799, 1331, -1000, 3337, 1647, -278, -1000, -1000, 1624, -1000, + -1000, -1000, -1000, 33034, 43128, 43849, 1753, -1000, 1933, -1000, + -1000, -1000, -1000, -1000, 4231, 1010, -1000, 817, 2795, 17825, + 3644, 17825, 3642, 833, 3637, 2019, -1000, 61886, -1000, -1000, + 61886, 5295, 3631, -1000, 3630, 3880, 795, 3629, 3626, 61886, + 3048, -1000, 4017, 61886, 974, 4031, -1000, 579, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, 858, -1000, 61886, + -1000, 61886, -1000, 2130, -1000, 31592, -1000, -1000, 2012, -1000, + 3108, 3106, -1000, -1000, 3336, 2627, -1000, 2036, 437, 1110, + 61886, -1000, 381, 3104, 8408, -1000, -1000, -1000, -1000, -1000, + 3999, 3103, 2763, 61886, -1000, 61886, 1634, 1634, 4279, 43128, + 61886, 11311, -1000, -1000, 14210, 3621, -1000, 14210, -1000, -1000, + -1000, 3334, -1000, -1000, -1000, -1000, -1000, -1000, 3620, 3992, + -1000, -1000, -1000, -1000, -1000, -1000, 4256, -1000, 2670, 61886, + -1000, 14210, 14933, -1000, 1032, 18559, -325, 504, -1000, -1000, + -1000, -287, 3100, -1000, -1000, 4211, 3092, 2936, -1000, 136, + 3089, -1000, 14210, -1000, -1000, -1000, -278, -1000, 1624, -1000, + -1000, 1635, -1000, -1000, 1434, 913, -1000, 3332, 2353, -1000, + 3331, 3032, -1000, 3013, 2975, 258, -1000, 258, -1000, 258, + 396, 14210, -1000, 2854, -1000, 2834, -1000, -1000, 3086, -1000, + -1000, -1000, 3084, -1000, -1000, 2813, -1000, 3330, -1000, 3083, + -1000, -1000, 3082, 3079, -369, -1000, -1000, 554, 1066, -1000, + 491, 61886, 768, -1000, 42407, 7685, -424, 687, 61886, 4236, + 3078, 2756, 3065, 2756, 61886, 865, -1000, 4094, 3064, -1000, + 3328, -1000, 3062, 3061, -1000, -1000, 473, 4266, 4269, 22186, + 4266, -1000, -1000, 4179, -1000, 1752, 549, -1000, -1000, 2689, + 835, -1000, -1000, 3059, 804, -1000, 1634, -1000, -1000, 2375, + 2592, 2980, 39523, 31592, 32313, 3058, -1000, 61886, -1000, -1000, + 41686, 2670, 2670, 6411, 1066, 4137, 683, 518, 68502, -1000, + 3619, 1492, 2312, -1000, 2794, -1000, 2783, -1000, 61886, -1000, + -1000, 1624, 4231, 1753, 138, -1000, -1000, 2146, -1000, 1492, + 3535, 4207, -1000, 4607, 61886, 4442, 61886, 3616, 2374, 17825, + -1000, 1002, 3982, -1000, -1000, 5295, -1000, -1000, 2563, 17825, + -1000, -1000, 3051, 32313, 1176, 2361, 2324, 1222, 3609, -1000, + 876, 4255, 2782, -1000, -1000, -1000, 1287, 3605, -1000, -303, + 3603, 2520, 2519, -1000, 61886, -1000, 39523, 39523, 1807, 1807, + 39523, 39523, 3600, 1080, -1000, -1000, 17825, -1000, -1000, -1000, + 2321, 1843, 1843, 1843, 1843, 1843, -1000, -1000, -1000, 2231, + 2010, -1000, -1000, -1000, -1000, -1000, 61886, 1932, -1000, -1000, + -1000, 2797, -1000, -1000, 1628, -1000, 4168, 1753, -1000, -1000, + 2627, 61886, 2627, -1000, 40965, -1000, 4204, 4188, -1000, -1000, + -1000, 2627, 1660, 267, 3598, 3596, -1000, -405, 61886, 61886, + -289, 2780, -1000, 3050, 342, -1000, -1000, 326, -1000, 1605, + 1624, -291, 125, 31592, 2298, -1000, 3321, 375, -183, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, 3318, -1000, 960, -1000, + -1000, -1000, 1605, 258, 258, 3312, 3309, -1000, -1000, -1000, + -1000, -1000, 61886, 61886, -1000, 61886, 3045, 2778, -1000, -1000, + 1998, -1000, -1000, -1000, 2507, 2502, 1997, 3307, 2973, 61886, + 668, 61886, -372, 3044, -372, 3041, 864, 2756, -342, -1000, + -1000, -1000, -1000, -174, -1000, -1000, 480, -1000, -1000, -1000, + 881, 2953, 2766, -1000, -1000, 544, -1000, -1000, -1000, 2763, + 3039, -1000, -1000, 128, -1000, 2281, 1966, -1000, -1000, -1000, + 638, -1000, -1000, -1000, 999, -1000, 3394, 68453, -1000, 1556, + -1000, -1000, 61886, -1000, 1434, 999, 38081, 909, 2380, -1000, + 2764, -1000, -1000, 1591, 4279, -1000, 896, -1000, 828, -1000, + 1922, -1000, 1914, 40244, 2762, 4070, -1000, 6471, 1161, -1000, + -1000, 3071, -1000, -1000, -1000, -1000, -1000, -1000, 3034, 3033, + -1000, -1000, -1000, -1000, -1000, 2761, 3595, 51, -1000, 4110, + 3030, 4092, 14210, -1000, -1000, 3592, 1882, 1877, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - 1641, 14397, -1000, -1000, 1638, -1000, -1000, -286, -1000, 3395, - 2462, 139, 125, 4060, -1000, 2772, 4046, 2772, 2772, -1000, - 107, 4123, 983, -1000, -1000, -1000, -1000, 2226, -1000, 2226, - -1000, -1000, -1000, -1000, -370, -1000, 2808, -1000, -1000, -1000, - 37387, 676, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 687, - 6658, -1000, 10800, 1615, -1000, 2572, -1000, 972, -1000, 2651, - -1000, -1000, -1000, -1000, 3748, 2998, 4099, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 3394, 2841, - -1000, 61743, -1000, 3980, 30943, 200, -1000, -1000, -1000, 2806, - -1000, 2772, -1000, -1000, 2214, -186, -1000, -1000, -1000, -1000, - -345, -1000, 61743, 674, -1000, 6658, 1538, -1000, 10800, -1000, - -300, -1000, 4101, -1000, 4116, 1182, 1182, 4848, 4848, 4848, - 4848, 14397, -1000, -1000, -1000, 61743, -1000, 1464, -1000, -1000, - -1000, 1835, -1000, -1000, -1000, -1000, 2764, -194, -1000, -1000, - 2736, 1443, 3392, -1000, -1000, -1000, -1000, -1000, -1000, 2512, - 792, -1000, 2659, 1361, -1000, 2204, -1000, 36671, 61743, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 61743, 10082, - -1000, 1823, -1000, -1000, 2572, 61743, -1000, + -1000, 1875, 1854, 39523, -1000, -1000, 3071, 1843, 2582, -1000, + 2231, 2231, 3022, 3020, 622, -1000, -1000, 2231, 2231, 2231, + 2231, 2231, 2231, 3591, 3016, 3010, 2231, 2231, 2231, 2231, + -1000, -1000, -1000, 2272, 2231, 2231, 2231, 31592, 2231, 1931, + 61886, -1000, -1000, -1000, 1841, 1828, -1000, -1000, -1000, -1000, + -1000, -381, 3579, 14210, 14210, -1000, -1000, -1000, 3576, -1000, + -1000, 4187, -285, -293, 3009, 286, 295, -1000, 3005, -1000, + -180, 3970, -186, -1000, -1000, 991, -281, 253, 239, 221, + -1000, -1000, -1000, 14210, -1000, -1000, -1000, -1000, 3003, -1000, + -1000, -1000, -1000, -1000, 61886, 3002, -1000, -1000, 127, -1000, + 2234, -1000, 61886, 667, -1000, -372, -1000, -372, 2756, 2981, + -1000, 61886, 862, -1000, -1000, -1000, -1000, 373, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, 2980, 2978, -1000, -1000, 807, + 4185, -1000, 68502, -1000, 2231, 638, -1000, 807, 1791, -1000, + 2231, 2231, -1000, 730, -1000, 2237, -1000, 2741, -1000, 4168, + -1000, 729, -1000, 811, -1000, -1000, -1000, 1789, -1000, -1000, + -1000, 6471, 818, -1000, 982, 3575, -1000, -1000, 3305, 14210, + 3574, 2231, 3292, 3573, 2800, -110, 39523, 3879, 3874, 3777, + 2983, 1751, -1000, -1000, 2733, 2732, -1000, -1000, 61886, 2727, + 2721, 2713, 2709, 2705, 2695, 61886, -1000, -1000, 2688, 2685, + 2679, 2658, 2567, 2657, 2636, 2617, -1000, 31592, 61886, -1000, + -1000, -1000, 38802, -1000, 3571, 1744, 1739, 61886, 2936, -287, + -1000, 2974, -1000, 1046, 305, 295, -1000, 4182, 327, 4180, + 4178, 1579, 3969, -1000, -1000, 2396, -1000, 246, 237, 229, + -1000, -1000, -1000, -1000, -1000, 2516, 2516, -372, 2973, 2969, + -1000, 61886, -1000, -1000, 2961, -372, 756, -1000, 468, -1000, + -1000, -1000, 1843, -1000, 4177, 837, -1000, 31592, -1000, -1000, + -1000, 38081, 2670, 2670, -1000, -1000, 2611, -1000, -1000, -1000, + -1000, 2600, -1000, -1000, -1000, 1730, -1000, 61886, 1210, 10588, + -1000, 2791, -1000, 61886, -1000, 14210, -306, 3896, -1000, 324, + 1672, 1843, 1807, 1843, 1807, 1843, 1807, 1843, 1807, 456, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 1655, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, 1644, 14210, -1000, -1000, 1626, -1000, -1000, + -289, -1000, 3551, 2595, 347, 302, 4176, -1000, 2936, 4175, + 2936, 2936, -1000, 272, 4240, 991, -1000, -1000, -1000, -1000, + 2465, -1000, 2465, -1000, -1000, -1000, -1000, -372, -1000, 2959, + -1000, -1000, -1000, 37360, 820, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, 818, 68502, -1000, 10588, 1612, -1000, 2627, -1000, + 1080, -1000, 2757, -1000, -1000, -1000, -1000, 3887, 3886, 4235, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, 3550, 3053, -1000, 61886, -1000, 4108, 30871, 284, -1000, + -1000, -1000, 2955, -1000, 2936, -1000, -1000, 2222, -184, -1000, + -1000, -1000, -1000, -349, -1000, 61886, 817, -1000, 68502, 1608, + -1000, 10588, -1000, -306, -1000, 4254, -1000, 4252, 1188, 1188, + 1843, 1843, 1843, 1843, 14210, -1000, -1000, -1000, 61886, -1000, + 1604, -1000, -1000, -1000, 1925, -1000, -1000, -1000, -1000, 2929, + -187, -1000, -1000, 2927, 1602, 3535, -1000, -1000, -1000, -1000, + -1000, -1000, 2587, 884, -1000, 3018, 1561, -1000, 2183, -1000, + 36639, 61886, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, 61886, 9865, -1000, 1615, -1000, -1000, 2627, 61886, -1000, } var yyPgo = [...]int{ - 0, 185, 58, 261, 192, 4924, 98, 273, 320, 3894, - 308, 271, 258, 4923, 4922, 4921, 3893, 3891, 4920, 4919, - 4918, 4917, 4915, 4914, 4913, 4908, 4907, 4906, 4891, 4889, - 4886, 4885, 4884, 4877, 4870, 4868, 4867, 4866, 4863, 4861, - 4860, 4858, 4856, 4855, 4854, 4852, 4851, 4849, 4848, 4847, - 4846, 4845, 4844, 256, 4843, 4842, 4841, 4840, 4839, 4836, - 4835, 4834, 4833, 4795, 4793, 4784, 4778, 4772, 4769, 4768, - 4753, 4749, 4748, 4747, 4746, 4745, 4741, 4739, 4738, 4737, - 4731, 4722, 4721, 4720, 4719, 4718, 4717, 4709, 4708, 4707, - 4706, 4705, 279, 4704, 3890, 4703, 4697, 4696, 4695, 4694, - 4687, 4685, 4682, 4681, 4680, 4679, 4678, 361, 4677, 4676, - 4675, 4674, 4673, 4672, 4671, 4669, 4668, 4666, 4662, 4661, - 4656, 344, 4654, 4653, 4652, 4651, 259, 4650, 229, 4649, - 190, 154, 4648, 4647, 4646, 4644, 4642, 4641, 112, 134, - 4640, 4639, 4638, 4637, 4636, 4630, 4629, 4627, 4626, 4624, - 4620, 4617, 4615, 4613, 253, 166, 80, 4609, 56, 4608, - 270, 224, 4607, 232, 4588, 162, 4583, 156, 4582, 4578, - 4575, 4574, 4573, 4570, 4568, 4567, 4566, 4565, 4564, 4562, - 4561, 4560, 4559, 4558, 4557, 4556, 4554, 4545, 4543, 4541, - 4540, 4539, 4538, 4537, 4536, 4535, 4534, 4532, 62, 4531, - 268, 4530, 85, 4529, 187, 4528, 87, 4527, 4526, 81, - 4525, 36, 32, 4524, 61, 111, 123, 265, 209, 272, - 4523, 205, 4522, 4519, 267, 186, 4518, 4517, 275, 4516, - 181, 237, 174, 99, 149, 4515, 161, 4514, 274, 55, - 64, 257, 221, 144, 4512, 4510, 66, 171, 155, 4508, - 204, 114, 4507, 4504, 4499, 127, 4494, 4488, 120, 4485, - 252, 194, 4483, 124, 4482, 4479, 4478, 25, 4477, 4475, - 220, 207, 4474, 4472, 110, 4470, 4467, 73, 148, 4466, - 88, 141, 177, 138, 4463, 3395, 143, 97, 4462, 137, - 117, 4459, 93, 4458, 4457, 4456, 4455, 198, 4454, 4452, - 160, 4450, 70, 4448, 4447, 4446, 76, 4445, 84, 4444, - 33, 4443, 71, 4442, 4441, 4439, 4438, 4437, 4436, 4435, - 4434, 4433, 4432, 4429, 4428, 41, 4427, 4425, 4420, 4418, - 7, 15, 18, 4415, 30, 4413, 180, 4412, 4411, 188, - 4410, 214, 4404, 4403, 106, 104, 4402, 105, 4401, 179, - 4400, 12, 31, 83, 4399, 4397, 4396, 316, 4395, 4394, - 4392, 302, 4390, 4389, 4388, 173, 4387, 4385, 4384, 561, - 4382, 4380, 4379, 4378, 4377, 4376, 142, 4375, 1, 231, - 28, 4374, 153, 157, 4372, 46, 35, 4371, 57, 133, - 219, 158, 118, 4370, 4368, 4367, 642, 218, 119, 37, - 0, 115, 239, 170, 4366, 4365, 4364, 289, 4363, 243, - 222, 238, 354, 278, 370, 4362, 4361, 69, 4360, 175, - 39, 60, 151, 517, 21, 241, 4359, 2554, 10, 196, - 4358, 227, 4357, 8, 17, 355, 146, 4355, 4353, 42, - 276, 4352, 4351, 4350, 150, 4349, 4347, 208, 86, 4346, - 4345, 4344, 4343, 4341, 45, 4339, 195, 19, 4338, 121, - 4337, 254, 101, 200, 159, 201, 189, 169, 234, 249, - 92, 95, 4336, 2160, 164, 122, 16, 4335, 9, 233, - 4334, 212, 131, 4333, 113, 4331, 255, 277, 228, 4330, - 197, 14, 54, 44, 29, 49, 11, 107, 91, 4328, - 4325, 26, 53, 4324, 63, 4322, 23, 4320, 4318, 51, - 48, 4317, 75, 5, 4316, 4315, 20, 22, 4314, 43, - 226, 182, 136, 109, 67, 4313, 4311, 172, 178, 4310, - 165, 168, 163, 4309, 47, 4307, 4305, 4303, 4302, 3547, - 264, 4301, 4300, 4279, 4278, 4277, 4276, 4275, 4274, 217, - 4273, 89, 52, 4270, 4269, 4268, 4267, 94, 147, 4264, - 4262, 4261, 4259, 34, 90, 4258, 13, 4257, 27, 24, - 40, 4256, 65, 4255, 4254, 4253, 3, 203, 4252, 4251, - 4, 4249, 4248, 2, 4244, 4243, 135, 4241, 108, 38, - 202, 125, 4240, 4236, 102, 199, 145, 4235, 4232, 116, - 248, 4231, 225, 4230, 298, 247, 266, 4214, 230, 4213, - 4211, 4209, 4208, 4207, 1345, 4206, 4205, 246, 74, 96, - 4198, 250, 132, 4197, 4196, 100, 176, 130, 129, 72, - 103, 4194, 128, 223, 4193, 213, 4192, 235, 4191, 4190, - 4189, 4188, 126, 4187, 4186, 4185, 4183, 206, 4179, 4178, - 211, 236, 4177, 4176, 286, 4174, 4173, 4171, 4170, 4169, - 4168, 4167, 4164, 4160, 4156, 263, 260, 4155, 4096, + 0, 199, 59, 261, 197, 4998, 110, 279, 345, 4044, + 327, 272, 269, 4994, 4993, 4992, 4043, 4041, 4991, 4990, + 4989, 4988, 4987, 4986, 4984, 4983, 4982, 4981, 4980, 4978, + 4977, 4976, 4974, 4972, 4971, 4970, 4969, 4968, 4967, 4965, + 4964, 4963, 4961, 4960, 4959, 4958, 4957, 4956, 4954, 4952, + 4951, 4949, 4948, 267, 4947, 4944, 4941, 4940, 4934, 4933, + 4932, 4917, 4915, 4914, 4913, 4912, 4910, 4908, 4907, 4906, + 4905, 4904, 4903, 4902, 4901, 4899, 4898, 4897, 4896, 4879, + 4878, 4877, 4876, 4875, 4874, 4870, 4869, 4868, 4866, 4864, + 4863, 4862, 288, 4861, 4031, 4860, 4859, 4858, 4857, 4855, + 4851, 4848, 4847, 4845, 4828, 4827, 4805, 322, 4804, 4801, + 4800, 4798, 4797, 4796, 4795, 4794, 4793, 4792, 4790, 4788, + 4787, 243, 4786, 4785, 4784, 4778, 263, 4777, 276, 4775, + 192, 149, 4774, 4773, 4772, 4771, 4768, 4767, 114, 133, + 4766, 4765, 4748, 4747, 4746, 4745, 4742, 4741, 4740, 4739, + 4738, 4737, 4736, 4733, 250, 166, 81, 4730, 57, 4729, + 265, 223, 4728, 234, 4727, 164, 4726, 168, 4725, 4724, + 4723, 4722, 4719, 4716, 4714, 4713, 4712, 4711, 4710, 4709, + 4707, 4706, 4705, 4704, 4702, 4701, 4699, 4698, 4695, 4691, + 4688, 4687, 4686, 4681, 4680, 4679, 4678, 4677, 56, 4674, + 282, 4673, 85, 4672, 194, 4671, 86, 4670, 4669, 83, + 4668, 28, 34, 4663, 72, 112, 124, 262, 3065, 273, + 4662, 210, 4657, 4656, 268, 191, 4654, 4651, 275, 4650, + 221, 246, 187, 92, 134, 4649, 163, 4648, 281, 51, + 71, 259, 207, 144, 4647, 4646, 67, 174, 143, 4643, + 202, 109, 4640, 4639, 4637, 129, 4636, 4634, 125, 4633, + 249, 200, 4632, 127, 4628, 4627, 4626, 22, 4625, 4624, + 219, 211, 4623, 4622, 111, 4619, 4618, 89, 146, 4617, + 87, 139, 185, 138, 4616, 3115, 137, 96, 4606, 142, + 120, 4604, 119, 4603, 4601, 4599, 4584, 195, 4582, 4580, + 153, 4579, 69, 4578, 4577, 4576, 75, 4574, 90, 4572, + 33, 4570, 65, 4568, 4567, 4563, 4562, 4561, 4560, 4559, + 4558, 4557, 4556, 4555, 4554, 41, 4551, 4550, 4547, 117, + 7, 13, 15, 4546, 31, 4543, 188, 4538, 4537, 182, + 4536, 215, 4535, 4533, 113, 105, 4532, 106, 4529, 181, + 4527, 11, 32, 84, 4525, 4522, 4519, 220, 4516, 4513, + 4511, 306, 4509, 4507, 4506, 169, 4503, 4502, 4500, 566, + 4499, 4498, 4497, 4496, 4495, 4494, 94, 4493, 1, 232, + 29, 4491, 158, 162, 4490, 45, 35, 4489, 58, 130, + 225, 157, 118, 4488, 4487, 4486, 744, 218, 123, 37, + 0, 115, 254, 190, 4484, 4483, 4482, 280, 4480, 252, + 266, 255, 277, 283, 258, 4478, 4477, 66, 4476, 176, + 39, 61, 159, 98, 24, 222, 4475, 2034, 10, 206, + 4474, 226, 4473, 8, 16, 365, 145, 4471, 4470, 44, + 284, 4469, 4468, 4467, 154, 4466, 4464, 203, 100, 4462, + 4460, 4459, 4457, 4456, 55, 4452, 201, 19, 4451, 126, + 4450, 278, 97, 264, 170, 204, 196, 177, 236, 247, + 93, 73, 4447, 2328, 172, 121, 14, 4446, 9, 239, + 4445, 178, 160, 4443, 103, 4442, 256, 286, 229, 4441, + 205, 18, 53, 43, 36, 52, 12, 325, 80, 4435, + 4434, 25, 54, 4433, 62, 4432, 23, 4431, 4430, 48, + 46, 4428, 74, 5, 4427, 4426, 17, 21, 4425, 42, + 227, 189, 136, 108, 70, 4424, 4423, 173, 180, 4420, + 165, 171, 175, 4419, 47, 4418, 4416, 4415, 4414, 806, + 271, 4410, 4408, 4407, 4406, 4405, 4404, 4403, 4402, 214, + 4401, 91, 49, 4400, 4399, 4398, 4394, 102, 148, 4393, + 4391, 4390, 4389, 38, 88, 4388, 20, 4386, 30, 27, + 40, 4385, 63, 4383, 4382, 4381, 3, 209, 4380, 4379, + 4, 4378, 4377, 2, 4376, 4373, 151, 4369, 107, 26, + 186, 128, 4367, 4366, 101, 208, 141, 4362, 4361, 116, + 253, 4360, 224, 4359, 155, 248, 274, 4358, 231, 4357, + 4356, 4354, 4351, 4349, 1490, 4347, 4346, 238, 76, 104, + 4345, 235, 132, 4344, 4342, 99, 179, 135, 147, 64, + 95, 4341, 131, 230, 4339, 217, 4338, 237, 4337, 4336, + 4334, 4333, 122, 4332, 4331, 4310, 4309, 213, 4308, 4307, + 212, 241, 4306, 4305, 305, 4304, 4303, 4302, 4301, 4300, + 4298, 4297, 4295, 4293, 4292, 257, 260, 4291, 4282, } -//line mysql_sql.y:14585 +//line mysql_sql.y:14692 type yySymType struct { union interface{} id int @@ -10555,153 +10664,154 @@ var yyR1 = [...]int{ 317, 318, 318, 318, 318, 319, 319, 397, 397, 344, 344, 344, 346, 346, 345, 339, 337, 337, 337, 337, 337, 337, 337, 338, 338, 338, 338, 338, 338, 338, - 338, 347, 347, 348, 348, 89, 95, 95, 95, 95, - 624, 624, 90, 90, 90, 635, 635, 539, 539, 419, - 419, 418, 418, 418, 418, 418, 418, 418, 418, 418, - 418, 418, 418, 418, 418, 418, 418, 544, 545, 415, + 338, 338, 347, 347, 348, 348, 89, 95, 95, 95, + 95, 624, 624, 90, 90, 90, 635, 635, 539, 539, + 419, 419, 418, 418, 418, 418, 418, 418, 418, 418, + 418, 418, 418, 418, 418, 418, 418, 418, 544, 545, + 415, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, - 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, - 53, 53, 53, 51, 52, 141, 141, 143, 143, 86, - 87, 88, 66, 60, 63, 64, 193, 196, 196, 196, - 196, 59, 59, 59, 460, 460, 58, 665, 665, 390, - 390, 74, 73, 62, 75, 76, 77, 78, 79, 80, - 57, 72, 72, 72, 72, 72, 72, 72, 72, 83, - 556, 556, 667, 667, 667, 81, 82, 538, 538, 538, - 71, 70, 69, 68, 67, 67, 56, 56, 55, 55, - 61, 178, 180, 65, 179, 179, 181, 181, 412, 412, - 412, 414, 414, 410, 666, 666, 504, 504, 413, 413, - 54, 54, 54, 54, 84, 411, 411, 389, 409, 409, - 409, 13, 13, 11, 18, 18, 18, 18, 18, 18, + 53, 53, 53, 53, 51, 52, 141, 141, 143, 143, + 86, 87, 88, 66, 60, 63, 64, 193, 196, 196, + 196, 196, 59, 59, 59, 460, 460, 58, 665, 665, + 390, 390, 74, 73, 62, 75, 76, 77, 78, 79, + 80, 57, 72, 72, 72, 72, 72, 72, 72, 72, + 83, 556, 556, 667, 667, 667, 81, 82, 538, 538, + 538, 71, 70, 69, 68, 67, 67, 56, 56, 55, + 55, 61, 178, 180, 65, 179, 179, 181, 181, 412, + 412, 412, 414, 414, 410, 666, 666, 504, 504, 413, + 413, 54, 54, 54, 54, 84, 411, 411, 389, 409, + 409, 409, 13, 13, 11, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 18, 18, 18, 49, 27, 28, 30, 468, 468, 465, - 29, 21, 20, 20, 24, 23, 19, 19, 22, 25, - 26, 26, 10, 10, 10, 10, 16, 16, 17, 225, - 225, 286, 286, 618, 618, 614, 614, 615, 615, 615, - 616, 616, 617, 617, 668, 668, 668, 128, 550, 550, - 550, 550, 550, 550, 550, 550, 216, 8, 8, 9, - 9, 251, 251, 549, 549, 549, 549, 549, 549, 472, - 472, 472, 595, 595, 595, 596, 250, 250, 243, 243, - 551, 551, 436, 597, 597, 559, 559, 558, 558, 557, - 557, 248, 248, 249, 249, 228, 228, 155, 155, 573, - 573, 574, 574, 564, 564, 564, 564, 572, 572, 534, - 534, 325, 325, 380, 380, 381, 381, 214, 214, 215, - 215, 215, 215, 215, 215, 654, 654, 655, 656, 657, - 657, 658, 658, 658, 659, 659, 659, 659, 659, 604, - 604, 606, 606, 605, 247, 247, 240, 240, 241, 241, - 241, 242, 242, 239, 239, 238, 237, 237, 236, 234, - 234, 234, 235, 235, 235, 258, 258, 218, 218, 218, - 217, 217, 217, 217, 217, 361, 361, 361, 361, 361, - 361, 361, 361, 361, 361, 361, 361, 219, 222, 222, - 223, 223, 224, 224, 224, 224, 224, 224, 224, 224, - 224, 224, 358, 358, 359, 359, 359, 359, 359, 153, - 153, 543, 543, 357, 357, 220, 220, 221, 221, 221, - 221, 356, 356, 355, 233, 233, 232, 231, 231, 231, - 226, 226, 226, 226, 226, 227, 367, 367, 366, 366, - 365, 365, 365, 365, 365, 365, 368, 131, 152, 152, - 154, 257, 257, 245, 244, 364, 363, 363, 363, 363, - 256, 256, 255, 255, 246, 246, 230, 230, 230, 230, - 362, 229, 360, 644, 644, 643, 643, 642, 640, 640, - 640, 641, 641, 641, 641, 587, 587, 587, 587, 587, - 398, 398, 398, 403, 403, 401, 401, 401, 401, 401, - 407, 7, 7, 7, 7, 7, 7, 7, 7, 7, - 7, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 48, 136, 136, 139, 139, 137, - 137, 138, 138, 142, 142, 140, 140, 40, 269, 270, - 41, 271, 271, 272, 272, 273, 273, 274, 275, 276, - 276, 276, 276, 452, 452, 39, 260, 260, 261, 261, - 262, 262, 263, 264, 264, 264, 268, 265, 266, 266, - 662, 662, 661, 38, 38, 31, 31, 199, 199, 200, - 200, 200, 202, 202, 321, 321, 321, 201, 201, 203, - 203, 203, 619, 621, 621, 623, 622, 622, 622, 625, - 625, 625, 625, 625, 626, 626, 626, 626, 627, 627, - 32, 175, 175, 175, 206, 206, 185, 630, 630, 630, - 629, 629, 510, 510, 631, 631, 632, 632, 384, 384, - 385, 385, 197, 198, 198, 187, 177, 205, 205, 205, - 205, 205, 207, 207, 288, 288, 176, 182, 183, 184, - 186, 188, 190, 190, 192, 620, 628, 628, 628, 469, - 469, 466, 467, 467, 464, 463, 463, 463, 634, 634, - 633, 633, 633, 399, 399, 33, 459, 459, 461, 462, - 462, 462, 462, 462, 462, 462, 462, 453, 453, 453, - 453, 37, 457, 457, 458, 458, 458, 458, 458, 458, + 18, 18, 18, 18, 49, 27, 28, 30, 468, 468, + 465, 29, 21, 20, 20, 24, 23, 19, 19, 22, + 25, 26, 26, 10, 10, 10, 10, 16, 16, 17, + 225, 225, 286, 286, 618, 618, 614, 614, 615, 615, + 615, 616, 616, 617, 617, 668, 668, 668, 128, 550, + 550, 550, 550, 550, 550, 550, 550, 216, 8, 8, + 9, 9, 251, 251, 549, 549, 549, 549, 549, 549, + 472, 472, 472, 595, 595, 595, 596, 250, 250, 243, + 243, 551, 551, 436, 597, 597, 559, 559, 558, 558, + 557, 557, 248, 248, 249, 249, 228, 228, 155, 155, + 573, 573, 574, 574, 564, 564, 564, 564, 572, 572, + 534, 534, 325, 325, 380, 380, 381, 381, 214, 214, + 215, 215, 215, 215, 215, 215, 654, 654, 655, 656, + 657, 657, 658, 658, 658, 659, 659, 659, 659, 659, + 604, 604, 606, 606, 605, 247, 247, 240, 240, 241, + 241, 241, 242, 242, 239, 239, 238, 237, 237, 236, + 234, 234, 234, 235, 235, 235, 258, 258, 218, 218, + 218, 217, 217, 217, 217, 217, 361, 361, 361, 361, + 361, 361, 361, 361, 361, 361, 361, 361, 219, 222, + 222, 223, 223, 224, 224, 224, 224, 224, 224, 224, + 224, 224, 224, 358, 358, 359, 359, 359, 359, 359, + 153, 153, 543, 543, 357, 357, 220, 220, 221, 221, + 221, 221, 356, 356, 355, 233, 233, 232, 231, 231, + 231, 226, 226, 226, 226, 226, 227, 367, 367, 366, + 366, 365, 365, 365, 365, 365, 365, 368, 131, 152, + 152, 154, 257, 257, 245, 244, 364, 363, 363, 363, + 363, 256, 256, 255, 255, 246, 246, 230, 230, 230, + 230, 362, 229, 360, 644, 644, 643, 643, 642, 640, + 640, 640, 641, 641, 641, 641, 587, 587, 587, 587, + 587, 398, 398, 398, 403, 403, 401, 401, 401, 401, + 401, 407, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 48, 136, 136, 139, 139, + 137, 137, 138, 138, 142, 142, 140, 140, 40, 269, + 270, 41, 271, 271, 272, 272, 273, 273, 274, 275, + 276, 276, 276, 276, 452, 452, 39, 260, 260, 261, + 261, 262, 262, 263, 264, 264, 264, 268, 265, 266, + 266, 662, 662, 661, 38, 38, 31, 31, 199, 199, + 200, 200, 200, 202, 202, 321, 321, 321, 201, 201, + 203, 203, 203, 619, 621, 621, 623, 622, 622, 622, + 625, 625, 625, 625, 625, 626, 626, 626, 626, 627, + 627, 32, 175, 175, 175, 206, 206, 185, 630, 630, + 630, 629, 629, 510, 510, 631, 631, 632, 632, 384, + 384, 385, 385, 197, 198, 198, 187, 177, 205, 205, + 205, 205, 205, 207, 207, 288, 288, 176, 182, 183, + 184, 186, 188, 190, 190, 192, 620, 628, 628, 628, + 469, 469, 466, 467, 467, 464, 463, 463, 463, 634, + 634, 633, 633, 633, 399, 399, 33, 459, 459, 461, + 462, 462, 462, 462, 462, 462, 462, 462, 453, 453, + 453, 453, 37, 457, 457, 458, 458, 458, 458, 458, 458, 458, 458, 458, 458, 458, 458, 458, 458, 458, 458, 458, 458, 458, 458, 458, 458, 458, 458, 458, - 454, 454, 456, 456, 451, 451, 451, 451, 451, 451, - 451, 451, 451, 451, 36, 36, 36, 204, 204, 450, - 450, 447, 447, 267, 267, 445, 445, 446, 446, 444, - 444, 444, 448, 448, 44, 85, 45, 46, 47, 43, - 449, 449, 208, 208, 208, 208, 208, 208, 208, 208, - 208, 208, 208, 254, 254, 213, 213, 213, 213, 213, - 213, 211, 211, 211, 211, 212, 212, 209, 209, 210, - 210, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 157, 156, 156, 156, 156, 156, 159, - 159, 383, 383, 382, 382, 158, 322, 322, 42, 299, - 299, 526, 526, 521, 521, 521, 521, 521, 541, 541, - 541, 522, 522, 522, 523, 523, 523, 525, 525, 525, - 524, 524, 524, 524, 524, 540, 540, 542, 542, 542, - 492, 492, 493, 493, 493, 496, 496, 513, 513, 514, - 514, 512, 512, 519, 519, 518, 518, 517, 517, 516, - 516, 515, 515, 515, 515, 507, 507, 506, 506, 494, - 494, 494, 494, 494, 495, 495, 495, 505, 505, 511, - 511, 354, 354, 353, 353, 308, 308, 309, 309, 352, - 352, 306, 306, 307, 307, 307, 351, 351, 351, 351, + 458, 458, 458, 454, 454, 456, 456, 451, 451, 451, + 451, 451, 451, 451, 451, 451, 451, 451, 36, 36, + 36, 204, 204, 450, 450, 447, 447, 267, 267, 445, + 445, 446, 446, 444, 444, 444, 448, 448, 44, 85, + 45, 46, 47, 43, 449, 449, 208, 208, 208, 208, + 208, 208, 208, 208, 208, 208, 208, 254, 254, 213, + 213, 213, 213, 213, 213, 211, 211, 211, 211, 212, + 212, 209, 209, 210, 210, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 157, 156, 156, + 156, 156, 156, 159, 159, 383, 383, 382, 382, 158, + 322, 322, 42, 299, 299, 526, 526, 521, 521, 521, + 521, 521, 541, 541, 541, 522, 522, 522, 523, 523, + 523, 525, 525, 525, 524, 524, 524, 524, 524, 540, + 540, 542, 542, 542, 492, 492, 493, 493, 493, 496, + 496, 513, 513, 514, 514, 512, 512, 519, 519, 518, + 518, 517, 517, 516, 516, 515, 515, 515, 515, 507, + 507, 506, 506, 494, 494, 494, 494, 494, 495, 495, + 495, 505, 505, 511, 511, 354, 354, 353, 353, 308, + 308, 309, 309, 352, 352, 306, 306, 307, 307, 307, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, - 351, 593, 593, 594, 311, 311, 323, 323, 323, 323, - 323, 323, 310, 310, 312, 312, 287, 287, 285, 285, - 277, 277, 277, 277, 277, 277, 278, 278, 279, 279, - 280, 280, 280, 284, 284, 283, 283, 283, 283, 281, - 281, 282, 282, 282, 282, 282, 282, 477, 477, 590, - 590, 591, 591, 586, 586, 586, 589, 589, 589, 589, - 589, 589, 589, 589, 589, 589, 592, 592, 592, 588, - 588, 289, 377, 377, 377, 400, 400, 400, 400, 402, - 376, 376, 376, 305, 305, 304, 304, 302, 302, 302, - 302, 302, 302, 302, 302, 302, 302, 302, 302, 302, + 351, 351, 351, 351, 351, 593, 593, 594, 311, 311, + 323, 323, 323, 323, 323, 323, 310, 310, 312, 312, + 287, 287, 285, 285, 277, 277, 277, 277, 277, 277, + 278, 278, 279, 279, 280, 280, 280, 284, 284, 283, + 283, 283, 283, 281, 281, 282, 282, 282, 282, 282, + 282, 477, 477, 590, 590, 591, 591, 586, 586, 586, + 589, 589, 589, 589, 589, 589, 589, 589, 589, 589, + 589, 592, 592, 592, 588, 588, 289, 377, 377, 377, + 400, 400, 400, 400, 402, 376, 376, 376, 305, 305, + 304, 304, 302, 302, 302, 302, 302, 302, 302, 302, 302, 302, 302, 302, 302, 302, 302, 302, 302, 302, - 302, 302, 478, 478, 478, 476, 476, 416, 416, 417, - 417, 334, 333, 333, 333, 333, 333, 331, 332, 330, - 330, 330, 330, 330, 327, 327, 326, 326, 326, 328, - 328, 328, 328, 328, 455, 455, 324, 324, 314, 314, - 314, 313, 313, 313, 520, 423, 423, 423, 423, 423, + 302, 302, 302, 302, 302, 302, 302, 478, 478, 478, + 476, 476, 416, 416, 417, 417, 334, 333, 333, 333, + 333, 333, 331, 332, 330, 330, 330, 330, 330, 327, + 327, 326, 326, 326, 328, 328, 328, 328, 328, 455, + 455, 324, 324, 314, 314, 314, 313, 313, 313, 520, 423, 423, 423, 423, 423, 423, 423, 423, 423, 423, + 423, 423, 423, 423, 423, 425, 425, 425, 425, 425, 425, 425, 425, 425, 425, 425, 425, 425, 425, 425, 425, 425, 425, 425, 425, 425, 425, 425, 425, 425, - 425, 425, 425, 425, 425, 425, 425, 425, 329, 374, + 425, 425, 425, 425, 329, 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, - 374, 374, 374, 374, 375, 375, 375, 375, 375, 375, - 375, 375, 426, 426, 432, 432, 603, 603, 602, 290, - 290, 290, 291, 291, 291, 291, 291, 291, 291, 291, - 291, 300, 300, 300, 501, 501, 501, 501, 502, 502, - 502, 502, 503, 503, 503, 499, 499, 500, 500, 437, - 438, 438, 547, 547, 548, 548, 497, 497, 498, 373, + 375, 375, 375, 375, 375, 375, 375, 375, 426, 426, + 432, 432, 603, 603, 602, 290, 290, 290, 291, 291, + 291, 291, 291, 291, 291, 291, 291, 300, 300, 300, + 501, 501, 501, 501, 502, 502, 502, 502, 503, 503, + 503, 499, 499, 500, 500, 437, 438, 438, 547, 547, + 548, 548, 497, 497, 498, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, - 373, 373, 373, 373, 555, 555, 555, 370, 370, 370, + 555, 555, 555, 370, 370, 370, 370, 370, 370, 370, 370, 370, 370, 370, 370, 370, 370, 370, 370, 370, - 370, 370, 370, 370, 370, 613, 613, 613, 598, 598, - 598, 599, 599, 599, 599, 599, 599, 599, 599, 599, - 599, 599, 599, 600, 600, 600, 600, 600, 600, 600, + 370, 613, 613, 613, 598, 598, 598, 599, 599, 599, + 599, 599, 599, 599, 599, 599, 599, 599, 599, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, - 601, 601, 601, 601, 372, 372, 372, 372, 372, 371, + 600, 600, 600, 600, 600, 600, 601, 601, 601, 601, + 372, 372, 372, 372, 372, 371, 371, 371, 371, 371, 371, 371, 371, 371, 371, 371, 371, 371, 371, 371, - 371, 371, 371, 371, 371, 371, 371, 439, 439, 440, - 440, 552, 552, 552, 552, 552, 552, 553, 553, 554, - 554, 554, 554, 546, 546, 546, 546, 546, 546, 546, + 371, 371, 371, 439, 439, 440, 440, 552, 552, 552, + 552, 552, 552, 553, 553, 554, 554, 554, 554, 546, 546, 546, 546, 546, 546, 546, 546, 546, 546, 546, 546, 546, 546, 546, 546, 546, 546, 546, 546, 546, - 546, 546, 424, 369, 369, 369, 441, 433, 433, 434, - 434, 435, 435, 427, 427, 427, 427, 427, 427, 428, - 428, 430, 430, 430, 430, 430, 430, 430, 430, 430, - 430, 430, 422, 422, 422, 422, 422, 422, 422, 422, - 422, 422, 422, 429, 429, 431, 431, 443, 443, 443, - 442, 442, 442, 442, 442, 442, 442, 303, 303, 303, - 303, 421, 421, 421, 420, 420, 420, 420, 420, 420, - 420, 420, 420, 420, 420, 420, 292, 292, 292, 292, - 292, 296, 296, 298, 298, 298, 298, 298, 298, 298, - 298, 298, 298, 298, 298, 298, 298, 297, 297, 297, - 297, 297, 297, 295, 295, 295, 295, 295, 293, 293, + 546, 546, 546, 546, 546, 546, 546, 546, 424, 369, + 369, 369, 441, 433, 433, 434, 434, 435, 435, 427, + 427, 427, 427, 427, 427, 428, 428, 430, 430, 430, + 430, 430, 430, 430, 430, 430, 430, 430, 422, 422, + 422, 422, 422, 422, 422, 422, 422, 422, 422, 429, + 429, 431, 431, 443, 443, 443, 442, 442, 442, 442, + 442, 442, 442, 303, 303, 303, 303, 421, 421, 421, + 420, 420, 420, 420, 420, 420, 420, 420, 420, 420, + 420, 420, 292, 292, 292, 292, 292, 296, 296, 298, + 298, 298, 298, 298, 298, 298, 298, 298, 298, 298, + 298, 298, 298, 297, 297, 297, 297, 297, 297, 295, + 295, 295, 295, 295, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 129, 130, 130, 294, 301, 301, 301, 301, 301, 301, 301, 301, 301, @@ -10749,14 +10859,14 @@ var yyR1 = [...]int{ 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, - 395, 395, 395, 395, 395, 395, 395, 395, 394, 394, - 394, 394, 394, 394, 394, 394, 394, 393, 393, 393, - 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, + 395, 395, 395, 395, 395, 395, 395, 395, 395, 395, + 395, 395, 395, 394, 394, 394, 394, 394, 394, 394, + 394, 394, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, - 393, + 393, 393, 393, 393, 393, 393, } var yyR2 = [...]int{ @@ -10815,156 +10925,157 @@ var yyR2 = [...]int{ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 2, 1, 3, 2, 1, 2, 2, 1, 2, - 3, 2, 2, 3, 5, 4, 4, 4, 4, 3, - 3, 1, 1, 3, 3, 7, 7, 7, 8, 8, - 0, 4, 7, 6, 6, 0, 3, 0, 2, 0, - 1, 1, 1, 1, 4, 2, 2, 3, 3, 4, - 5, 3, 4, 4, 2, 2, 2, 3, 0, 1, + 3, 2, 2, 3, 5, 4, 4, 4, 4, 4, + 3, 3, 1, 1, 3, 3, 7, 7, 7, 8, + 8, 0, 4, 7, 6, 6, 0, 3, 0, 2, + 0, 1, 1, 1, 1, 4, 2, 2, 3, 3, + 4, 5, 3, 4, 4, 2, 2, 2, 3, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 2, 5, 0, 2, 0, 2, 3, - 3, 3, 5, 4, 3, 3, 3, 4, 5, 6, - 5, 2, 5, 5, 0, 2, 7, 0, 1, 0, - 1, 5, 5, 3, 3, 2, 4, 4, 4, 4, - 4, 1, 1, 1, 3, 3, 1, 1, 1, 6, - 0, 1, 1, 1, 1, 5, 5, 0, 1, 1, - 3, 3, 3, 4, 7, 7, 5, 4, 7, 8, - 3, 3, 4, 2, 3, 4, 4, 3, 0, 2, - 2, 0, 2, 2, 1, 1, 1, 1, 0, 1, - 5, 5, 6, 4, 3, 1, 3, 1, 1, 3, - 5, 2, 3, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 2, 5, 0, 2, 0, 2, + 3, 3, 3, 5, 4, 3, 3, 3, 4, 5, + 6, 5, 2, 5, 5, 0, 2, 7, 0, 1, + 0, 1, 5, 5, 3, 3, 2, 4, 4, 4, + 4, 4, 1, 1, 1, 3, 3, 1, 1, 1, + 6, 0, 1, 1, 1, 1, 5, 5, 0, 1, + 1, 3, 3, 3, 4, 7, 7, 5, 4, 7, + 8, 3, 3, 4, 2, 3, 4, 4, 3, 0, + 2, 2, 0, 2, 2, 1, 1, 1, 1, 0, + 1, 5, 5, 6, 4, 3, 1, 3, 1, 1, + 3, 5, 2, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 4, 4, 4, 4, 1, 3, 1, - 4, 6, 6, 4, 4, 4, 4, 4, 3, 6, - 3, 5, 1, 1, 2, 2, 11, 8, 9, 1, - 3, 2, 4, 0, 2, 0, 1, 1, 1, 1, - 0, 1, 0, 1, 0, 1, 1, 5, 2, 1, - 4, 1, 5, 4, 4, 2, 4, 1, 2, 5, - 5, 1, 3, 2, 1, 5, 4, 4, 2, 0, - 5, 4, 0, 1, 3, 3, 1, 3, 1, 3, - 1, 3, 4, 0, 1, 0, 1, 1, 3, 1, - 1, 0, 4, 1, 3, 2, 1, 0, 10, 0, - 2, 0, 2, 0, 4, 7, 4, 0, 2, 0, - 2, 0, 2, 0, 4, 1, 3, 1, 1, 7, - 4, 6, 8, 4, 6, 0, 1, 3, 8, 0, - 6, 0, 4, 6, 1, 1, 1, 1, 1, 2, - 3, 1, 3, 6, 0, 3, 0, 1, 2, 4, - 4, 0, 5, 0, 1, 3, 1, 3, 3, 0, - 1, 1, 0, 2, 2, 0, 2, 3, 3, 3, - 1, 3, 3, 3, 3, 1, 2, 2, 1, 2, - 2, 1, 2, 2, 1, 2, 2, 7, 0, 1, - 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 0, 2, 0, 4, 7, 6, 6, 3, - 5, 0, 2, 0, 2, 1, 3, 1, 2, 3, - 5, 0, 1, 2, 1, 3, 1, 1, 1, 1, - 4, 4, 4, 3, 4, 3, 2, 2, 2, 2, - 2, 3, 2, 3, 2, 3, 2, 4, 1, 3, - 4, 0, 2, 1, 3, 1, 1, 2, 2, 3, - 0, 1, 2, 4, 1, 3, 1, 3, 2, 3, - 1, 4, 3, 0, 1, 1, 2, 5, 2, 2, - 2, 0, 2, 3, 3, 0, 1, 3, 1, 3, - 0, 1, 2, 1, 1, 0, 1, 2, 1, 2, + 1, 1, 1, 1, 4, 4, 4, 4, 1, 3, + 1, 4, 6, 6, 4, 4, 4, 4, 4, 3, + 6, 3, 5, 1, 1, 2, 2, 11, 8, 9, + 1, 3, 2, 4, 0, 2, 0, 1, 1, 1, + 1, 0, 1, 0, 1, 0, 1, 1, 5, 2, + 1, 4, 1, 5, 4, 4, 2, 4, 1, 2, + 5, 5, 1, 3, 2, 1, 5, 4, 4, 2, + 0, 5, 4, 0, 1, 3, 3, 1, 3, 1, + 3, 1, 3, 4, 0, 1, 0, 1, 1, 3, + 1, 1, 0, 4, 1, 3, 2, 1, 0, 10, + 0, 2, 0, 2, 0, 4, 7, 4, 0, 2, + 0, 2, 0, 2, 0, 4, 1, 3, 1, 1, + 7, 4, 6, 8, 4, 6, 0, 1, 3, 8, + 0, 6, 0, 4, 6, 1, 1, 1, 1, 1, + 2, 3, 1, 3, 6, 0, 3, 0, 1, 2, + 4, 4, 0, 5, 0, 1, 3, 1, 3, 3, + 0, 1, 1, 0, 2, 2, 0, 2, 3, 3, + 3, 1, 3, 3, 3, 3, 1, 2, 2, 1, + 2, 2, 1, 2, 2, 1, 2, 2, 7, 0, + 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 0, 2, 0, 4, 7, 6, 6, + 3, 5, 0, 2, 0, 2, 1, 3, 1, 2, + 3, 5, 0, 1, 2, 1, 3, 1, 1, 1, + 1, 4, 4, 4, 3, 4, 3, 2, 2, 2, + 2, 2, 3, 2, 3, 2, 3, 2, 4, 1, + 3, 4, 0, 2, 1, 3, 1, 1, 2, 2, + 3, 0, 1, 2, 4, 1, 3, 1, 3, 2, + 3, 1, 4, 3, 0, 1, 1, 2, 5, 2, + 2, 2, 0, 2, 3, 3, 0, 1, 3, 1, + 3, 0, 1, 2, 1, 1, 0, 1, 2, 1, + 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 10, 0, 3, 0, 2, + 0, 4, 1, 1, 0, 2, 0, 2, 7, 1, + 1, 9, 1, 3, 0, 1, 1, 3, 1, 3, + 0, 1, 1, 1, 0, 2, 14, 1, 3, 0, + 1, 1, 3, 1, 1, 2, 4, 1, 1, 1, + 1, 0, 1, 2, 9, 9, 7, 8, 1, 2, + 3, 3, 3, 0, 4, 1, 1, 1, 1, 1, + 0, 1, 1, 1, 1, 1, 4, 1, 1, 1, + 3, 3, 4, 3, 3, 0, 1, 1, 1, 0, + 2, 7, 8, 10, 8, 2, 2, 8, 0, 3, + 3, 0, 3, 0, 3, 0, 3, 0, 5, 1, + 3, 0, 3, 3, 0, 2, 9, 8, 0, 2, + 2, 3, 3, 0, 2, 0, 2, 4, 5, 4, + 4, 4, 6, 4, 8, 5, 1, 0, 2, 2, + 1, 3, 2, 1, 3, 2, 1, 3, 2, 0, + 1, 3, 4, 3, 1, 1, 4, 1, 3, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, + 1, 1, 11, 0, 2, 3, 3, 2, 2, 3, + 1, 1, 3, 3, 3, 3, 3, 3, 4, 2, + 2, 3, 3, 3, 3, 1, 1, 1, 3, 3, + 3, 3, 3, 1, 3, 3, 4, 0, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 6, 8, + 10, 0, 4, 1, 1, 0, 3, 0, 1, 0, + 1, 1, 2, 4, 4, 4, 0, 1, 8, 2, + 4, 4, 4, 9, 0, 2, 8, 9, 5, 5, + 8, 7, 8, 12, 12, 13, 13, 0, 4, 0, + 3, 3, 3, 2, 2, 0, 3, 3, 3, 4, + 4, 0, 3, 0, 2, 11, 9, 11, 8, 6, + 9, 7, 10, 7, 6, 9, 11, 2, 2, 9, + 4, 5, 3, 0, 4, 1, 3, 0, 3, 6, + 0, 2, 10, 0, 2, 0, 2, 0, 3, 2, + 4, 3, 0, 2, 1, 0, 2, 3, 0, 2, + 3, 0, 2, 1, 0, 3, 2, 4, 3, 0, + 1, 0, 1, 1, 0, 6, 0, 3, 5, 0, + 4, 0, 3, 1, 3, 4, 5, 0, 3, 1, + 3, 2, 3, 1, 2, 0, 4, 6, 5, 0, + 2, 0, 2, 4, 5, 4, 5, 1, 5, 6, + 5, 0, 3, 0, 1, 1, 3, 3, 3, 0, + 4, 1, 3, 3, 3, 0, 1, 1, 3, 2, + 3, 3, 3, 4, 4, 3, 3, 3, 3, 4, + 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, + 3, 3, 1, 5, 4, 1, 3, 3, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 10, 0, 3, 0, 2, 0, - 4, 1, 1, 0, 2, 0, 2, 7, 1, 1, - 9, 1, 3, 0, 1, 1, 3, 1, 3, 0, - 1, 1, 1, 0, 2, 14, 1, 3, 0, 1, - 1, 3, 1, 1, 2, 4, 1, 1, 1, 1, - 0, 1, 2, 9, 9, 7, 8, 1, 2, 3, - 3, 3, 0, 4, 1, 1, 1, 1, 1, 0, - 1, 1, 1, 1, 1, 4, 1, 1, 1, 3, - 3, 4, 3, 3, 0, 1, 1, 1, 0, 2, - 7, 8, 10, 8, 2, 2, 8, 0, 3, 3, - 0, 3, 0, 3, 0, 3, 0, 5, 1, 3, - 0, 3, 3, 0, 2, 9, 8, 0, 2, 2, - 3, 3, 0, 2, 0, 2, 4, 5, 4, 4, - 4, 6, 4, 8, 5, 1, 0, 2, 2, 1, - 3, 2, 1, 3, 2, 1, 3, 2, 0, 1, - 3, 4, 3, 1, 1, 4, 1, 3, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, - 1, 11, 0, 2, 3, 3, 2, 2, 3, 1, - 1, 3, 3, 3, 3, 3, 3, 4, 2, 2, - 3, 3, 3, 3, 1, 1, 3, 3, 3, 3, - 1, 3, 3, 4, 0, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 6, 8, 10, 0, 4, 1, - 1, 0, 3, 0, 1, 0, 1, 1, 2, 4, - 4, 4, 0, 1, 8, 2, 4, 4, 4, 9, - 0, 2, 8, 9, 5, 5, 8, 7, 8, 12, - 12, 13, 13, 0, 4, 0, 3, 3, 3, 2, - 2, 0, 3, 3, 3, 4, 4, 0, 3, 0, - 2, 11, 9, 11, 8, 6, 9, 7, 10, 7, - 6, 9, 11, 2, 2, 9, 4, 5, 3, 0, - 4, 1, 3, 0, 3, 6, 0, 2, 10, 0, - 2, 0, 2, 0, 3, 2, 4, 3, 0, 2, - 1, 0, 2, 3, 0, 2, 3, 0, 2, 1, - 0, 3, 2, 4, 3, 0, 1, 0, 1, 1, - 0, 6, 0, 3, 5, 0, 4, 0, 3, 1, - 3, 4, 5, 0, 3, 1, 3, 2, 3, 1, - 2, 0, 4, 6, 5, 0, 2, 0, 2, 4, - 5, 4, 5, 1, 5, 6, 5, 0, 3, 0, - 1, 1, 3, 3, 3, 0, 4, 1, 3, 3, - 3, 0, 1, 1, 3, 2, 3, 3, 3, 4, - 4, 3, 3, 3, 3, 4, 4, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, - 3, 3, 3, 3, 3, 3, 3, 3, 1, 5, - 4, 1, 3, 3, 2, 2, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 3, 2, 4, - 0, 5, 5, 5, 5, 6, 0, 1, 1, 3, - 1, 1, 1, 1, 1, 7, 9, 7, 9, 2, - 1, 7, 9, 7, 9, 8, 5, 0, 1, 0, - 1, 1, 1, 1, 3, 3, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, - 1, 3, 1, 3, 5, 1, 1, 1, 1, 1, - 1, 3, 5, 0, 1, 1, 2, 1, 2, 2, - 1, 1, 2, 2, 2, 3, 3, 2, 2, 1, - 5, 6, 4, 2, 1, 1, 1, 5, 4, 1, - 7, 5, 0, 1, 1, 1, 2, 0, 1, 1, - 2, 5, 0, 1, 1, 2, 2, 3, 3, 1, - 1, 2, 2, 2, 0, 1, 2, 2, 2, 0, - 4, 7, 3, 3, 0, 3, 0, 3, 1, 1, - 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, - 1, 1, 1, 3, 5, 2, 2, 2, 2, 4, - 1, 1, 2, 5, 6, 8, 6, 3, 6, 6, - 1, 1, 1, 1, 1, 1, 3, 9, 1, 4, - 4, 4, 4, 5, 4, 5, 7, 9, 5, 7, - 9, 5, 5, 7, 7, 9, 7, 7, 7, 9, - 7, 7, 0, 2, 0, 1, 1, 2, 4, 1, - 2, 2, 1, 2, 2, 1, 2, 2, 2, 2, - 2, 0, 1, 1, 1, 2, 2, 2, 2, 2, - 2, 2, 1, 1, 1, 2, 5, 0, 1, 3, - 0, 1, 0, 2, 0, 2, 0, 1, 6, 8, - 8, 6, 6, 5, 5, 5, 6, 6, 6, 6, - 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, - 6, 6, 6, 8, 1, 1, 1, 4, 6, 4, - 6, 8, 6, 6, 4, 5, 4, 4, 4, 3, - 4, 6, 6, 7, 4, 1, 1, 1, 1, 1, + 1, 3, 2, 4, 0, 5, 5, 5, 5, 6, + 0, 1, 1, 3, 1, 1, 1, 1, 1, 7, + 9, 7, 9, 2, 1, 7, 9, 7, 9, 8, + 5, 0, 1, 0, 1, 1, 1, 1, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 0, 1, 3, 1, 3, 5, + 1, 1, 1, 1, 1, 1, 3, 5, 0, 1, + 1, 2, 1, 2, 2, 1, 1, 2, 2, 2, + 3, 3, 2, 2, 1, 5, 6, 4, 2, 1, + 1, 1, 5, 4, 1, 7, 5, 0, 1, 1, + 1, 2, 0, 1, 1, 2, 5, 0, 1, 1, + 2, 2, 3, 3, 1, 1, 2, 2, 2, 0, + 1, 2, 2, 2, 0, 4, 7, 3, 3, 0, + 3, 0, 3, 1, 1, 1, 1, 1, 1, 1, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 1, 1, 1, 1, 3, 5, + 2, 2, 2, 2, 4, 1, 1, 2, 5, 6, + 8, 6, 3, 6, 6, 1, 1, 1, 1, 1, + 1, 3, 9, 8, 1, 4, 4, 4, 4, 5, + 4, 5, 7, 9, 5, 7, 9, 5, 5, 7, + 7, 9, 7, 7, 7, 9, 7, 7, 0, 2, + 0, 1, 1, 2, 4, 1, 2, 2, 1, 2, + 2, 1, 2, 2, 2, 2, 2, 0, 1, 1, + 1, 2, 2, 2, 2, 2, 2, 2, 1, 1, + 1, 2, 5, 0, 1, 3, 0, 1, 0, 2, + 0, 2, 0, 1, 6, 8, 8, 6, 6, 5, + 5, 5, 6, 6, 6, 6, 5, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 8, + 1, 1, 1, 4, 6, 4, 6, 8, 6, 6, + 4, 5, 4, 4, 4, 3, 4, 6, 6, 7, + 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 2, 2, 8, 8, 6, 4, - 2, 3, 2, 4, 2, 2, 4, 6, 2, 2, - 4, 6, 4, 2, 4, 4, 4, 0, 1, 2, - 3, 1, 1, 1, 1, 1, 1, 0, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 8, 8, 6, 4, 2, 3, 2, 4, + 2, 2, 4, 6, 2, 2, 4, 6, 4, 2, + 4, 4, 4, 0, 1, 2, 3, 1, 1, 1, + 1, 1, 1, 0, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 3, 0, 1, 1, 3, 0, 1, 1, - 3, 1, 3, 3, 3, 3, 3, 2, 1, 1, - 1, 3, 4, 3, 4, 3, 4, 3, 4, 3, - 4, 1, 3, 4, 4, 5, 4, 5, 3, 4, - 5, 6, 1, 0, 2, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 3, 0, + 1, 1, 3, 0, 1, 1, 3, 1, 3, 3, + 3, 3, 3, 2, 1, 1, 1, 3, 4, 3, + 4, 3, 4, 3, 4, 3, 4, 1, 3, 4, + 4, 5, 4, 5, 3, 4, 5, 6, 1, 0, + 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, + 1, 2, 3, 1, 1, 1, 4, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 2, 1, 1, 1, 2, 3, 1, 1, 1, - 4, 2, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 2, 3, 2, - 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, - 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 2, 2, 4, 4, 1, 2, 3, 5, + 1, 1, 1, 2, 3, 2, 2, 2, 2, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, + 2, 2, 2, 2, 4, 4, 1, 2, 3, 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 0, 1, 0, 3, 0, 3, 3, 0, 3, @@ -11017,482 +11128,485 @@ var yyR2 = [...]int{ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, + 1, 1, 1, 1, 1, 1, } var yyChk = [...]int{ - -1000, -660, -663, -2, -5, 714, -1, -4, -130, -99, + -1000, -660, -663, -2, -5, 719, -1, -4, -130, -99, -7, -15, -132, -133, -8, -128, -10, -11, -188, -13, -106, -123, -125, -127, -126, -53, -12, -122, -92, -93, -108, -116, -119, -120, -121, -134, -129, -131, -214, -135, - -144, -145, -195, -148, -150, -151, -183, -184, -208, 704, + -144, -145, -195, -148, -150, -151, -183, -184, -208, 709, -100, -101, -102, -103, -104, -105, -34, -33, -32, -31, - -175, -185, -189, -191, -146, -48, 621, 710, 517, -9, - -604, 570, -16, -17, -18, 264, 291, -404, -405, -406, + -175, -185, -189, -191, -146, -48, 626, 715, 522, -9, + -604, 575, -16, -17, -18, 268, 295, -404, -405, -406, -408, -664, -54, -55, -56, -67, -68, -69, -70, -71, -81, -82, -83, -57, -58, -59, -62, -60, -74, -73, -75, -76, -77, -78, -79, -80, -61, -65, -178, -179, -180, -181, -84, -63, -85, -64, -193, -196, -147, -86, -87, -88, -66, -51, -52, -90, -89, -95, -91, -96, - -177, -187, -14, -194, -97, -50, -98, 265, -94, 79, - -109, -110, -111, -112, -113, -114, -115, -117, -118, 443, - 449, 504, 703, 64, -215, -218, 734, 735, 738, 606, - 608, 309, 177, 178, 180, 181, 185, 188, -35, -36, + -177, -187, -14, -194, -97, -50, -98, 269, -94, 79, + -109, -110, -111, -112, -113, -114, -115, -117, -118, 448, + 454, 509, 708, 64, -215, -218, 739, 740, 743, 611, + 613, 313, 177, 178, 180, 181, 185, 188, -35, -36, -37, -38, -39, -40, -42, -41, -43, -44, -45, -46, - -47, 260, 16, 14, 18, -19, -22, -20, -23, -21, + -47, 264, 16, 14, 18, -19, -22, -20, -23, -21, -29, -30, -28, -25, -27, -176, -182, -26, -186, -24, - -190, -192, -149, -49, 286, 285, 41, 352, 353, 354, - 447, 284, 261, 263, 17, 34, 45, 422, -217, 88, - 607, 262, -219, 15, 741, -6, -3, -2, -162, -166, - -170, -173, -174, -171, -172, -4, -130, 123, 276, 705, - -400, 439, 706, 708, 707, 91, 99, -393, -395, 517, - 291, 443, 449, 703, 735, 738, 606, 608, 309, 623, - 624, 625, 626, 627, 628, 629, 630, 632, 633, 634, - 635, 636, 637, 638, 681, 682, 648, 649, 639, 640, - 641, 642, 643, 644, 645, 646, 650, 651, 652, 653, - 654, 655, 656, 657, 658, 659, 660, 661, 662, 663, - 573, 574, 683, 685, 686, 687, 688, 602, 631, 668, - 676, 677, 678, 420, 421, 614, 700, 740, 303, 327, - 472, 333, 340, 406, 177, 195, 191, 219, 210, 412, - 359, 358, 607, 186, 307, 345, 308, 98, 180, 556, - 113, 529, 501, 183, 365, 368, 366, 367, 322, 324, - 326, 603, 604, 433, 329, 601, 328, 330, 332, 605, - 363, 423, 206, 200, 321, 305, 198, 310, 413, 43, - 311, 404, 403, 224, 312, 313, 618, 525, 419, 531, - 337, 55, 499, 199, 325, 528, 699, 228, 232, 236, - 237, 238, 239, 240, 241, 242, 243, 244, 245, 547, - 410, 392, 393, 394, 548, 415, 168, 169, 533, 409, - 550, 414, 223, 226, 227, 283, 400, 401, 416, 417, - 418, 46, 616, 295, 551, 230, 730, 222, 217, 559, - 341, 339, 405, 221, 194, 216, 306, 68, 234, 233, - 235, 495, 496, 497, 498, 314, 315, 437, 546, 213, - 201, 424, 187, 25, 554, 290, 530, 450, 369, 370, - 316, 334, 342, 364, 229, 231, 297, 302, 357, 411, - 617, 503, 301, 538, 539, 338, 552, 197, 294, 323, - 289, 555, 731, 188, 452, 317, 181, 331, 549, 733, - 558, 67, 163, 193, 184, 721, 722, 280, 684, 178, - 299, 304, 701, 732, 318, 319, 320, 600, 344, 343, - 335, 185, 214, 296, 220, 204, 192, 215, 179, 298, - 557, 164, 697, 422, 482, 212, 209, 300, 273, 702, - 553, 532, 182, 486, 166, 207, 346, 691, 692, 693, - 696, 438, 399, 347, 348, 205, 287, 523, 524, 351, - 492, 387, 466, 502, 473, 467, 251, 252, 355, 535, - 537, 225, 694, 371, 372, 373, 527, 374, 376, 377, - 382, 442, 59, 61, 100, 103, 102, 736, 737, 66, - 32, 428, 431, 464, 468, 389, 698, 615, 386, 390, - 391, 432, 28, 484, 454, 488, 487, 51, 52, 53, - 56, 57, 58, 60, 62, 63, 54, 599, 447, 461, - 560, 48, 50, 457, 458, 30, 434, 483, 505, 385, - 485, 516, 49, 514, 515, 536, 29, 436, 435, 65, - 47, 491, 493, 494, 349, 383, 445, 711, 561, 440, - 456, 460, 441, 388, 430, 462, 70, 453, 712, 448, - 446, 384, 619, 620, 395, 647, 425, 500, 596, 595, - 594, 593, 592, 591, 590, 589, 352, 353, 354, 469, - 470, 471, 481, 474, 475, 476, 477, 478, 479, 480, - 519, 520, 713, 540, 542, 543, 612, 544, 541, 268, - 739, 426, 427, 271, 715, 716, 101, 717, 719, 718, - 31, 720, 729, 726, 727, 728, 622, 545, 609, 723, - 611, 610, 669, 670, 671, 672, 673, -483, -481, -400, - 607, 309, 703, 449, 606, 608, 443, 422, 735, 738, - 447, 291, 352, 353, 354, 517, 420, -271, -400, 739, - -94, -17, -16, -9, -217, -218, -668, 257, 259, 458, - -285, 270, -400, -409, 26, 499, -107, 500, 265, 266, - 88, 80, -400, -10, -121, -8, -128, -92, -214, 504, - -407, -400, 352, 352, 612, -407, 270, -402, 301, 480, - -400, -539, 276, -487, -459, 302, -486, -461, -489, -462, - 35, 260, 262, 261, 621, 298, 18, 447, 272, 16, - 15, 448, 284, 28, 29, 31, 17, 449, 451, 32, - 452, 455, 456, 457, 45, 461, 462, 291, 91, 99, - 94, 669, 670, 671, 672, 673, 309, -270, -400, -435, - -427, 120, -430, -422, -423, -425, -378, -577, -420, 88, - 149, 150, 157, 121, 742, -424, -520, 39, 123, 627, - 631, 668, 571, -370, -371, -372, -373, -374, -375, 613, - -400, -578, -576, 94, 104, 106, 110, 111, 109, 107, - 171, 202, 108, 95, 172, -218, 91, -598, 637, 643, - -394, 660, 685, 686, 687, 688, 659, 64, -546, -554, - 269, -552, 170, 208, 287, 204, 16, 155, 492, 205, - 676, 677, 678, 634, 656, 573, 574, 683, 638, 648, - 663, 629, 630, 632, 624, 625, 626, 628, 639, 641, - 655, -555, 651, 661, 662, 647, 679, 680, 726, 681, - 682, 664, 665, 666, 675, 674, 667, 669, 670, 671, - 672, 673, 719, 93, 92, 654, 653, 640, 635, 636, - 642, 623, 633, 644, 652, 657, 658, 431, 113, 432, - 433, 563, 423, 83, 434, 276, 499, 73, 435, 436, - 437, 438, 439, 570, 440, 74, 441, 430, 291, 482, - 442, 207, 225, 576, 575, 577, 567, 564, 562, 565, - 566, 568, 569, 645, 646, 650, -152, -154, 689, -654, - -361, -655, 6, 7, 8, 9, -656, 172, -645, 501, - 617, 94, 563, 270, 345, 420, 19, 725, 381, 605, - 725, 381, 605, 359, 182, 179, -473, 182, 119, 188, - 187, 274, 182, -473, -400, 185, 725, 184, 721, 612, - 355, -449, -199, 420, 482, 374, 100, 301, -453, -450, - 603, -540, 349, 345, 321, 271, 116, -200, 281, 280, - 114, 563, 269, 459, 340, 59, 61, -228, 275, 42, - -285, -606, 597, -605, -400, -614, -615, 257, 258, 259, - 725, 544, 612, 730, 539, 433, 102, 103, 721, 722, - 30, 270, 444, 297, 537, 535, 536, 540, 541, 542, - 543, -72, -556, -538, 532, 531, -413, 524, 530, 522, - 534, 525, 421, 377, 374, 621, 376, 381, 260, 715, - 604, 598, -388, 466, 502, 560, 561, 445, 503, 547, - 549, 526, 113, 211, 208, 271, 273, 270, 721, 612, - 301, 420, 563, 482, 100, 374, 270, -614, 730, 179, - 547, 549, 501, 301, 480, 44, -480, 492, -479, -481, - 548, 559, 92, 93, 546, -388, 113, 523, 523, -654, - -361, -215, -218, -131, -604, 605, 725, 612, 271, 420, - 482, 301, 272, 270, 600, 603, 273, 563, 269, 352, - 444, 297, 374, 381, 100, 184, 721, -222, -223, -224, - 253, 254, 255, 72, 258, 256, 69, 35, 36, 37, - -1, 127, 741, -427, -427, -6, 744, -6, -427, -400, - -400, 174, -292, -296, -293, -295, -294, 740, -298, -297, - 208, 209, 170, 212, 218, 214, 215, 216, 217, 219, - 220, 221, 222, 223, 226, 227, 224, 34, 225, 287, - 204, 205, 206, 207, -301, 191, 210, 615, 246, 192, - 247, 193, 248, 194, 249, 168, 169, 250, 195, 198, - 199, 200, 201, 197, 228, 229, 230, 231, 232, 233, - 234, 235, 237, 236, 238, 239, 240, 241, 242, 243, - 244, 245, 173, -259, 94, 35, 88, 173, 94, -654, - -238, -239, 11, -228, 19, -285, -277, 173, 742, -376, - -400, 501, 130, -107, 80, -107, 500, 80, -107, 500, - 265, -607, -608, -609, -611, 265, 500, 499, 266, 336, - -126, 173, 309, 19, -407, -407, -400, 86, -285, -461, - 301, -487, -459, 39, 85, 174, 274, 174, 85, 88, - 445, 420, 482, 446, 563, 270, 459, 273, 301, 460, - 420, 482, 270, 273, 563, 301, 420, 270, 273, 482, - 301, 460, 420, 522, 523, 273, 30, 450, 453, 454, - 523, -560, 559, 174, 119, 116, 117, 118, -427, 137, + -190, -192, -149, -49, 290, 289, 41, 356, 357, 358, + 452, 288, 265, 267, 17, 34, 45, 427, -217, 88, + 612, 266, -219, 15, 746, -6, -3, -2, -162, -166, + -170, -173, -174, -171, -172, -4, -130, 123, 280, 710, + -400, 444, 711, 713, 712, 91, 99, -393, -395, 522, + 295, 448, 454, 708, 740, 743, 611, 613, 313, 628, + 629, 630, 631, 632, 633, 634, 635, 637, 638, 639, + 640, 641, 642, 643, 686, 687, 653, 654, 644, 645, + 646, 647, 648, 649, 650, 651, 655, 656, 657, 658, + 659, 660, 661, 662, 663, 664, 665, 666, 667, 668, + 578, 579, 688, 690, 691, 692, 693, 607, 636, 673, + 681, 682, 683, 425, 426, 619, 705, 745, 307, 331, + 477, 337, 344, 411, 177, 195, 191, 219, 210, 417, + 363, 362, 612, 186, 311, 349, 312, 98, 180, 561, + 113, 534, 506, 183, 369, 372, 370, 371, 326, 328, + 330, 608, 609, 438, 333, 606, 332, 334, 336, 610, + 367, 428, 206, 200, 325, 309, 198, 314, 418, 43, + 315, 409, 408, 224, 316, 317, 623, 530, 424, 536, + 341, 55, 504, 199, 329, 533, 704, 232, 236, 240, + 241, 242, 243, 244, 245, 246, 247, 248, 249, 552, + 415, 396, 397, 398, 399, 553, 420, 168, 169, 538, + 414, 555, 419, 223, 226, 227, 228, 229, 230, 231, + 287, 405, 406, 421, 422, 423, 46, 621, 299, 556, + 234, 735, 222, 217, 564, 345, 343, 410, 221, 194, + 216, 310, 68, 238, 237, 239, 500, 501, 502, 503, + 318, 319, 442, 551, 213, 201, 429, 187, 25, 559, + 294, 535, 455, 373, 374, 320, 338, 346, 368, 233, + 235, 301, 306, 361, 416, 622, 508, 305, 543, 544, + 342, 557, 197, 298, 327, 293, 560, 736, 188, 457, + 321, 181, 335, 554, 738, 563, 67, 163, 193, 184, + 726, 727, 284, 689, 178, 303, 308, 706, 737, 322, + 323, 324, 605, 348, 347, 339, 185, 214, 300, 220, + 204, 192, 215, 179, 302, 562, 164, 702, 427, 487, + 212, 209, 304, 277, 707, 558, 537, 182, 491, 166, + 207, 350, 696, 697, 698, 701, 443, 404, 351, 352, + 205, 291, 528, 529, 355, 497, 391, 471, 507, 478, + 472, 255, 256, 359, 540, 542, 225, 699, 375, 376, + 377, 532, 378, 380, 381, 386, 447, 59, 61, 100, + 103, 102, 741, 742, 66, 32, 433, 436, 469, 473, + 393, 703, 620, 390, 394, 395, 437, 28, 489, 459, + 493, 492, 51, 52, 53, 56, 57, 58, 60, 62, + 63, 54, 604, 452, 466, 565, 48, 50, 462, 463, + 30, 439, 488, 510, 389, 490, 521, 49, 519, 520, + 541, 29, 441, 440, 65, 47, 496, 498, 499, 353, + 387, 450, 716, 566, 445, 461, 465, 446, 392, 435, + 467, 70, 458, 717, 453, 451, 388, 624, 625, 400, + 652, 430, 505, 601, 600, 599, 598, 597, 596, 595, + 594, 356, 357, 358, 474, 475, 476, 486, 479, 480, + 481, 482, 483, 484, 485, 524, 525, 718, 545, 547, + 548, 617, 549, 546, 272, 744, 431, 432, 275, 720, + 721, 101, 722, 724, 723, 31, 725, 734, 731, 732, + 733, 627, 550, 614, 728, 616, 615, 674, 675, 676, + 677, 678, -483, -481, -400, 612, 313, 708, 454, 611, + 613, 448, 427, 740, 743, 452, 295, 356, 357, 358, + 522, 425, -271, -400, 744, -94, -17, -16, -9, -217, + -218, -668, 261, 263, 463, -285, 274, -400, -409, 26, + 504, -107, 505, 269, 270, 88, 80, -400, -10, -121, + -8, -128, -92, -214, 509, -407, -400, 356, 356, 617, + -407, 274, -402, 305, 485, -400, -539, 280, -487, -459, + 306, -486, -461, -489, -462, 35, 264, 266, 265, 626, + 302, 18, 452, 276, 16, 15, 453, 288, 28, 29, + 31, 17, 454, 456, 32, 457, 460, 461, 462, 45, + 466, 467, 295, 91, 99, 94, 674, 675, 676, 677, + 678, 313, -270, -400, -435, -427, 120, -430, -422, -423, + -425, -378, -577, -420, 88, 149, 150, 157, 121, 747, + -424, -520, 39, 123, 632, 636, 673, 576, -370, -371, + -372, -373, -374, -375, 618, 399, -400, -578, -576, 94, + 104, 106, 110, 111, 109, 107, 171, 202, 108, 95, + 172, -218, 91, -598, 642, 648, -394, 665, 690, 691, + 692, 693, 664, 64, -546, -554, 273, -552, 170, 208, + 291, 204, 16, 155, 497, 205, 681, 682, 683, 639, + 661, 578, 579, 688, 643, 653, 668, 634, 635, 637, + 629, 630, 631, 633, 644, 646, 660, -555, 656, 666, + 667, 652, 684, 685, 731, 686, 687, 669, 670, 671, + 680, 679, 672, 674, 675, 676, 677, 678, 724, 93, + 92, 659, 658, 645, 640, 641, 647, 628, 638, 649, + 657, 662, 663, 436, 113, 437, 438, 568, 428, 83, + 439, 280, 504, 73, 440, 441, 442, 443, 444, 575, + 445, 74, 446, 435, 295, 487, 447, 207, 225, 581, + 580, 582, 572, 569, 567, 570, 571, 573, 574, 650, + 651, 655, -152, -154, 694, -654, -361, -655, 6, 7, + 8, 9, -656, 172, -645, 506, 622, 94, 568, 274, + 349, 425, 19, 730, 385, 610, 730, 385, 610, 363, + 182, 179, -473, 182, 119, 188, 187, 278, 182, -473, + -400, 185, 730, 184, 726, 617, 359, -449, -199, 425, + 487, 378, 100, 305, -453, -450, 608, -540, 353, 349, + 325, 275, 116, -200, 285, 284, 114, 568, 273, 464, + 344, 59, 61, -228, 279, 42, -285, -606, 602, -605, + -400, -614, -615, 261, 262, 263, 730, 549, 617, 735, + 544, 438, 102, 103, 726, 727, 30, 274, 449, 301, + 542, 540, 541, 545, 546, 547, 548, -72, -556, -538, + 537, 536, -413, 529, 535, 527, 539, 530, 426, 381, + 378, 626, 380, 385, 264, 720, 609, 603, -388, 471, + 507, 565, 566, 450, 508, 552, 554, 531, 113, 211, + 208, 275, 277, 274, 726, 617, 305, 425, 568, 487, + 100, 378, 274, -614, 735, 179, 552, 554, 506, 305, + 485, 44, -480, 497, -479, -481, 553, 564, 92, 93, + 551, -388, 113, 528, 528, -654, -361, -215, -218, -131, + -604, 610, 730, 617, 275, 425, 487, 305, 276, 274, + 605, 608, 277, 568, 273, 356, 449, 301, 378, 385, + 100, 184, 726, -222, -223, -224, 257, 258, 259, 72, + 262, 260, 69, 35, 36, 37, -1, 127, 746, -427, + -427, -6, 749, -6, -427, -400, -400, 174, -292, -296, + -293, -295, -294, 745, -298, -297, 208, 209, 170, 212, + 218, 214, 215, 216, 217, 219, 220, 221, 222, 223, + 226, 227, 228, 229, 230, 231, 224, 34, 225, 291, + 204, 205, 206, 207, -301, 191, 210, 620, 250, 192, + 251, 193, 252, 194, 253, 168, 169, 254, 195, 198, + 199, 200, 201, 197, 232, 233, 234, 235, 236, 237, + 238, 239, 241, 240, 242, 243, 244, 245, 246, 247, + 248, 249, 173, -259, 94, 35, 88, 173, 94, -654, + -238, -239, 11, -228, 19, -285, -277, 173, 747, -376, + -400, 506, 130, -107, 80, -107, 505, 80, -107, 505, + 269, -607, -608, -609, -611, 269, 505, 504, 270, 340, + -126, 173, 313, 19, -407, -407, -400, 86, -285, -461, + 305, -487, -459, 39, 85, 174, 278, 174, 85, 88, + 450, 425, 487, 451, 568, 274, 464, 277, 305, 465, + 425, 487, 274, 277, 568, 305, 425, 274, 277, 487, + 305, 465, 425, 527, 528, 277, 30, 455, 458, 459, + 528, -560, 564, 174, 119, 116, 117, 118, -427, 137, -442, 130, 131, 132, 133, 134, 135, 136, 144, 143, 156, 149, 150, 151, 152, 153, 154, 155, 145, 146, 147, 148, 140, 120, 138, 142, 139, 122, 161, 160, -218, -427, -435, 64, -425, -425, -425, -425, -400, -520, - -432, -427, 88, 88, 88, 88, 88, 173, 107, 94, - 88, -427, 88, 88, 88, 88, 88, 88, 88, 88, - 88, 88, 88, 88, 88, -553, 88, 88, -439, -440, - 88, 88, -420, -376, 88, 94, 94, 88, 88, 88, - 94, 88, 88, 88, -440, -440, 88, 88, 88, 88, + -432, -427, 88, 88, 88, 88, 88, 88, 173, 107, + 94, 88, -427, 88, 88, 88, 88, 88, 88, 88, + 88, 88, 88, 88, 88, 88, -553, 88, 88, -439, + -440, 88, 88, -420, -376, 88, 94, 94, 88, 88, + 88, 94, 88, 88, 88, -440, -440, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, - 88, 88, 88, 88, -239, 174, -238, 88, -238, -239, - -219, -218, 35, 36, 35, 36, 35, 36, 35, 36, - -657, 712, 88, 104, 736, 251, -252, -400, -253, -400, - -160, 19, 742, -400, 721, -637, 35, 612, 375, 612, - 612, 375, 612, 260, 18, 363, 57, 364, 552, 14, - 186, 187, 188, -400, 185, 274, -400, -447, 276, -447, - -447, -447, -269, -400, 297, 444, 273, 600, 273, -200, - -447, 19, -447, -447, -447, -447, 272, -447, 26, 270, - 270, 270, 270, -447, 570, 130, 130, 62, -248, 293, - -228, -285, 174, -606, -247, 88, -616, 190, -638, -637, - 545, 731, 732, 733, 85, -412, 138, 142, -412, -357, - 20, -357, 26, 26, 299, 299, 299, -412, 339, -665, - -666, 19, 140, -410, -666, -410, -410, -412, -667, 272, - 533, 46, 300, 299, -240, -241, 24, -240, 527, 523, - -504, 528, 529, -414, -666, -413, -412, -412, -413, -412, - -412, 380, -412, 35, 375, 376, 270, 273, 563, 374, - 716, -665, -665, 34, 34, -539, -539, -285, -539, -400, - 276, -462, -539, 598, -389, -400, -539, -539, -539, -340, - -341, -285, -617, 275, 733, -651, -650, 550, -653, 552, - 179, -481, 179, -481, 91, -461, 301, 301, 174, 130, - 26, -482, 130, 141, -481, -481, -482, -482, -310, 44, - -399, 170, -400, 94, -310, 44, -648, -647, -285, -239, - -219, -218, 89, 89, 89, 612, -539, -539, -539, -539, - -539, -539, -539, -540, -539, -539, -539, -539, -539, -407, - -260, -400, -271, 276, -539, 375, -539, -539, -539, -220, - -221, 151, -427, -400, -224, -3, -164, -163, 124, 125, - 127, 706, 439, 705, 709, 703, -481, 44, -533, 164, - 163, 88, -527, -529, 88, -528, 88, -528, -528, -528, - -528, -528, 88, 88, -530, 88, -530, -530, -527, -531, - 203, 88, -531, -532, 88, -532, -531, -400, -508, 14, - -433, -435, -400, 42, -239, -155, 42, -241, 23, -248, - 100, -400, 205, 184, 720, 38, 173, 104, 94, -126, - -107, 80, -126, -107, -107, 89, 174, -610, 110, 111, - -612, 94, 223, 214, -400, -124, 94, -576, -7, -12, - -8, -128, -10, -11, -53, -92, -214, 606, 608, -579, - -577, 88, 35, 491, 85, 19, -488, 270, 563, 444, - 297, 273, 420, -486, -468, -465, -463, -399, -461, -464, - -463, -491, -376, 523, -156, 506, 505, 351, -427, -427, - -427, -427, -427, 109, 120, 399, 110, 111, -422, -443, - 35, 347, 348, -423, -423, -423, -423, -423, -423, -423, - -423, -423, -423, -423, -423, -425, -425, -431, -441, -520, - 88, 140, 138, 142, 139, 122, -425, -425, -423, -423, - -290, -292, 163, 164, -312, -399, 170, 89, 174, -427, - -603, -602, 124, -427, -427, -427, -427, -454, -456, -376, - 88, -400, -423, -599, -600, 578, 579, 580, 581, 582, - 583, 584, 585, 586, 587, 588, 435, 430, 436, 434, - 423, 442, 437, 438, 207, 595, 596, 589, 590, 591, - 592, 593, 594, -433, -433, -427, -599, -423, -433, -369, - 36, 35, -435, -435, -435, 89, -427, -613, 397, 396, - 398, -243, -400, -433, 89, 89, 89, 104, -435, -435, - -433, -423, -433, -433, -433, -433, -600, -600, -601, 287, - 204, 206, 205, -369, -369, -369, -369, 151, -435, -435, - -369, -369, -369, -369, 151, -369, -369, -369, -369, -369, - -369, -369, -369, -369, -369, -369, -369, -369, 89, 89, - 89, 89, -427, 89, -427, -427, -427, -427, -427, 151, - -435, -240, -154, -558, -557, -427, 44, -155, -241, -658, - 713, 88, -376, -646, 94, 94, 742, -160, 173, 19, - 270, -160, 173, 721, 184, -160, 563, 19, -400, -400, - 94, 104, -400, 94, 104, 270, 563, 270, 563, -285, - -285, -285, 553, 554, 183, 187, 186, -400, 185, -400, - -400, 120, -400, -400, -400, 38, -271, -260, -447, -447, - -447, -621, -400, 95, 94, -469, -466, -463, -400, -400, - -459, -400, -389, -285, -447, -447, -447, -447, -285, -321, - 56, 57, 58, -463, -201, 59, 60, -549, 64, -214, - 88, 34, 88, -248, -605, 38, -246, -400, -617, -141, - 26, 301, -357, -425, -425, -427, 420, 563, 270, -463, - 301, -665, -412, -412, -390, -389, -414, -409, -414, -414, - -357, -410, -412, -412, -427, -414, -410, -357, -400, 523, - -357, -357, -504, -389, -412, 94, -411, -400, -411, -447, - -389, -390, -390, -285, -285, -335, -342, -336, -343, 293, - 267, 428, 429, 263, 261, 11, 262, -351, 340, -448, - 571, -316, -317, 80, 45, -319, 291, 468, 464, 303, - 307, 98, 308, 501, 309, 272, 311, 312, 313, 328, - 330, 283, 314, 315, 316, 492, 317, 178, 329, 318, - 319, 320, 446, -311, 6, 382, 44, 54, 55, 515, - 514, 619, 14, 304, -400, 471, 608, 34, 39, 263, - 267, 262, -621, -619, 34, -400, 34, -469, -463, -400, - -400, 174, 274, -231, -233, -230, -226, -227, -232, -360, - -362, -229, 88, -285, -218, -400, -481, 174, 551, 553, - 554, -651, -482, -651, -482, 274, 35, 491, -485, 491, - 35, -459, -479, 547, 549, -474, 94, 492, -464, -484, - 85, 170, -557, -482, -482, -484, -484, 160, 174, -649, - 552, 553, 257, -240, 104, -639, -637, -400, 612, -400, - -287, -285, -621, -468, -459, -400, -539, -287, -287, -287, - -402, -402, 88, 173, 39, -400, -539, -400, -400, -400, - -356, 174, -355, 19, -401, -400, 38, 94, 173, -165, - -163, 126, -427, -6, 705, -427, -6, -6, -427, -6, - -427, -537, 166, -292, 104, 104, -379, 94, -379, 104, - -531, 104, 104, 622, 89, 94, -240, 690, -242, 23, - -237, -236, -427, -550, 64, -216, 88, -214, 34, 270, - -539, -277, 130, 130, 130, 27, -400, 26, -126, -107, - -608, 173, 174, -246, -488, -467, -464, -490, 151, -400, - -475, 174, 14, 745, 92, 274, -634, -633, 483, 89, - 174, -561, 275, 570, 94, 742, 499, 251, 252, 109, - 399, 110, 111, -520, -435, -431, -425, -425, -423, -423, - -429, 288, -429, 119, -300, 169, 168, -300, -427, 743, - -426, -602, 126, -427, 38, 174, 38, 174, 86, 174, - 89, -527, -427, 173, 174, 89, 89, 19, 19, 140, - 89, -427, 89, 89, 89, 89, 19, 19, -427, 89, - 173, 89, 89, 89, 89, 86, 89, 174, 89, 89, - 89, 89, 174, 174, 174, -435, -435, -427, -435, 89, - 89, 89, -427, -427, -427, -435, 89, -427, -427, -427, - -427, -427, -427, -427, -427, -427, -427, -246, -427, -427, - -498, 518, -498, -498, -498, 89, -498, 89, 174, 89, - 174, 89, 89, 174, 174, 174, 174, 89, -242, 88, - 104, 174, 737, -383, -382, 94, -161, 274, -400, 721, - -400, -161, -400, -400, 130, -161, -400, 721, 94, 94, - -285, -389, -285, -389, 614, 42, 42, 184, 188, 188, - 187, -400, 94, 39, 26, 26, 338, -136, 609, -270, - 88, 88, -285, -285, -285, -623, 469, -400, -635, 174, - 44, -633, 563, -197, 351, -451, 86, -204, 358, 19, - 14, -285, -285, -285, -285, -299, 38, -472, 85, -551, - -436, -597, 689, -250, 89, -243, -595, -596, -243, -249, - -400, -549, 88, 89, 174, 19, -225, -286, -400, -143, - 24, -400, -462, -400, -400, -400, -460, 86, -400, -390, - -357, -357, -414, -357, -357, 174, 25, -412, -414, -414, - -277, -410, -277, 173, -277, -389, -526, 38, -247, 174, - 23, 293, -284, -397, -281, -283, 278, -417, -282, 281, - -591, 279, 277, 114, 282, 336, 115, 272, -397, -397, - 278, -320, 274, 38, -397, -338, 272, 402, 336, 279, - 23, 293, -337, 272, 115, -400, 278, 282, 279, 277, - -396, 130, -388, 160, 274, 46, 446, -396, 620, 293, - -396, -396, -396, -396, -396, -396, -396, 310, 310, -396, - -396, -396, -396, -396, -396, -396, -396, -396, -396, -396, - 179, -396, -396, -396, -396, -396, -396, 88, 305, 306, - 338, 609, 124, 622, 611, -462, 274, 538, 538, -624, - 469, 34, 426, 426, 427, -635, 422, 45, 34, -205, - 420, -341, -339, -411, 34, -363, -364, -365, -366, -368, - -367, 71, 75, 77, 81, 72, 73, 74, 526, 78, - 83, 76, 34, 174, -398, -403, 38, -400, 94, -398, - -218, -233, -231, -398, 88, -482, -650, -652, 555, 552, - 558, -484, -484, 104, 274, 88, 130, -484, -484, 44, - -399, -647, 559, 553, -242, -267, 723, 174, 85, -287, - -261, -262, -263, -264, -292, -376, 740, 209, 212, 214, - 215, 216, 217, 219, 220, 221, 222, 223, 226, 227, - 224, 225, 287, 204, 205, 206, 207, 191, 210, 615, - 192, 193, 194, 168, 169, 195, 198, 199, 200, 201, - 197, 228, 229, 230, 231, 232, 233, 234, 235, 237, - 236, 238, 239, 240, 241, 242, 243, 244, 245, -400, - -271, 94, 19, -267, -357, -221, -233, -400, 94, -400, - 151, 127, -6, 125, -169, -168, -167, 128, 703, 709, - 127, 127, 127, 89, 89, 89, 89, 174, 89, 89, - 89, 174, 89, 174, 104, -564, 528, -242, 94, -155, - 664, 174, -234, 40, 41, -551, -250, 89, -595, -285, - 94, -427, -400, 94, -427, 205, 173, 501, -400, -577, - 89, -490, 174, 274, 173, 173, -465, 449, -399, -467, - 23, 14, -376, 42, -383, 130, 742, -400, 89, -429, - -429, 119, -425, -422, 89, 127, -427, 125, -290, -427, - -290, -291, -297, 170, 208, 287, 207, 206, 204, 163, - 164, -310, -456, 614, -234, 89, -400, -435, -427, -427, - -423, 89, -427, -427, 19, -400, -310, -423, -427, -427, - -427, -239, -239, 89, 89, -497, -498, -497, -497, 89, - 89, 89, 89, -497, 89, 89, 89, 89, 89, 89, - 89, 89, 89, 89, 89, 89, 174, 88, -498, -498, - -427, -498, -427, -498, -498, -427, 104, 106, 104, 106, - -557, -155, -659, 66, 711, 65, 491, 109, 341, 174, - 104, 94, 743, 174, 130, 420, -400, 19, 173, 94, - -400, 94, 19, 270, -400, 19, 19, -285, -285, -285, - 188, 94, -636, 345, 420, 563, 270, 420, 345, 563, - 270, -509, 104, -137, 124, 94, 457, -272, -273, -274, - -275, -276, 140, 175, 176, -261, -247, 88, -247, -626, - 530, 471, 481, -396, 374, -419, -418, 422, 45, -544, - 492, 477, 478, -466, 301, -389, 151, -632, 101, 130, - 85, 386, 390, 392, 394, 393, 391, 387, 388, 389, - -445, -446, -444, -448, -389, 94, -619, 88, 88, -214, - 38, 138, -204, 358, 19, 88, 88, 38, -521, 371, - -292, 43, 174, 88, 89, 174, 64, 174, 130, 89, - 174, -1, -400, -285, -225, -400, 19, 174, -618, 173, - 104, -400, -459, -412, -357, -427, -427, -357, -412, -412, - -414, -400, -277, -521, -292, 38, -336, 267, 262, -494, - 338, 339, -495, -511, 341, -513, 88, -289, -376, -282, - -590, -591, -447, -400, 115, -590, 115, 88, -289, -376, - -376, -339, -376, -400, -400, -400, -400, -346, -345, -376, - -349, 35, -350, -400, -400, -400, -400, 115, -400, 115, - -315, 44, 51, 52, 53, -396, -396, 211, -318, 44, - 491, 493, 494, -349, 104, 104, 104, 104, 94, 94, - 94, -396, -396, 104, 94, -403, 94, -592, 187, 48, - 49, 104, 104, 104, 104, 44, 94, -323, 44, 321, - 325, 322, 323, 324, 94, 104, 44, 104, 44, 104, - 44, -400, 88, -593, -594, 94, -509, 94, 88, 104, - 94, 263, -462, 94, 85, -626, -396, 426, -481, 130, - 130, -419, -628, 98, 472, -628, -631, 351, -207, 563, - 35, -251, 267, 262, -619, -471, -470, -376, -230, -230, - -230, -230, -230, -230, 71, 82, 71, -244, 88, 71, - 76, 71, 76, 71, 76, 71, -365, 71, 82, -471, - -232, -247, -403, 89, -644, -643, -642, -640, 79, 275, - 80, -433, -484, 552, 556, 557, -467, -415, 94, -474, - -155, -285, -285, -542, 331, 332, 89, 174, -292, -400, - -359, 21, 173, 123, -6, -165, -167, -427, -6, -427, - 705, 439, 706, 94, 104, 104, -572, 512, 507, 509, - -155, -573, 499, 14, -236, -235, 47, 89, 64, -239, - 743, 743, 743, 743, 94, -400, 104, 19, -464, -459, - 151, 151, -400, 450, -475, 94, 470, 94, 270, 743, - 94, -383, -422, -427, 89, 38, 89, 89, -528, -528, - -527, -530, -527, -300, -300, 89, 88, -234, 89, 89, - 26, 89, 89, 89, 89, -427, 89, 89, 174, 174, - 89, -547, 572, -548, 649, -497, -497, -497, -497, -497, + 88, 88, 88, 88, 88, -239, 174, -238, 88, -238, + -239, -219, -218, 35, 36, 35, 36, 35, 36, 35, + 36, -657, 717, 88, 104, 741, 255, -252, -400, -253, + -400, -160, 19, 747, -400, 726, -637, 35, 617, 379, + 617, 617, 379, 617, 264, 18, 367, 57, 368, 557, + 14, 186, 187, 188, -400, 185, 278, -400, -447, 280, + -447, -447, -447, -269, -400, 301, 449, 277, 605, 277, + -200, -447, 19, -447, -447, -447, -447, 276, -447, 26, + 274, 274, 274, 274, -447, 575, 130, 130, 62, -248, + 297, -228, -285, 174, -606, -247, 88, -616, 190, -638, + -637, 550, 736, 737, 738, 85, -412, 138, 142, -412, + -357, 20, -357, 26, 26, 303, 303, 303, -412, 343, + -665, -666, 19, 140, -410, -666, -410, -410, -412, -667, + 276, 538, 46, 304, 303, -240, -241, 24, -240, 532, + 528, -504, 533, 534, -414, -666, -413, -412, -412, -413, + -412, -412, 384, -412, 35, 379, 380, 274, 277, 568, + 378, 721, -665, -665, 34, 34, -539, -539, -285, -539, + -400, 280, -462, -539, 603, -389, -400, -539, -539, -539, + -340, -341, -285, -617, 279, 738, -651, -650, 555, -653, + 557, 179, -481, 179, -481, 91, -461, 305, 305, 174, + 130, 26, -482, 130, 141, -481, -481, -482, -482, -310, + 44, -399, 170, -400, 94, -310, 44, -648, -647, -285, + -239, -219, -218, 89, 89, 89, 617, -539, -539, -539, + -539, -539, -539, -539, -540, -539, -539, -539, -539, -539, + -407, -260, -400, -271, 280, -539, 379, -539, -539, -539, + -220, -221, 151, -427, -400, -224, -3, -164, -163, 124, + 125, 127, 711, 444, 710, 714, 708, -481, 44, -533, + 164, 163, 88, -527, -529, 88, -528, 88, -528, -528, + -528, -528, -528, -528, -528, -528, -528, 88, 88, -530, + 88, -530, -530, -527, -531, 203, 88, -531, -532, 88, + -532, -531, -400, -508, 14, -433, -435, -400, 42, -239, + -155, 42, -241, 23, -248, 100, -400, 205, 184, 725, + 38, 173, 104, 94, -126, -107, 80, -126, -107, -107, + 89, 174, -610, 110, 111, -612, 94, 223, 214, -400, + -124, 94, -576, -7, -12, -8, -128, -10, -11, -53, + -92, -214, 611, 613, -579, -577, 88, 35, 496, 85, + 19, -488, 274, 568, 449, 301, 277, 425, -486, -468, + -465, -463, -399, -461, -464, -463, -491, -376, 528, -156, + 511, 510, 355, -427, -427, -427, -427, -427, 109, 120, + 404, 110, 111, -422, -443, 35, 351, 352, -423, -423, + -423, -423, -423, -423, -423, -423, -423, -423, -423, -423, + -425, -425, -431, -441, -520, 88, 140, 138, 142, 139, + 122, -425, -425, -423, -423, -290, -292, 163, 164, -312, + -399, 170, 89, 174, -427, -603, -602, 124, -427, -427, + -427, -427, -454, -456, -376, 88, -454, -400, -423, -599, + -600, 583, 584, 585, 586, 587, 588, 589, 590, 591, + 592, 593, 440, 435, 441, 439, 428, 447, 442, 443, + 207, 600, 601, 594, 595, 596, 597, 598, 599, -433, + -433, -427, -599, -423, -433, -369, 36, 35, -435, -435, + -435, 89, -427, -613, 402, 401, 403, -243, -400, -433, + 89, 89, 89, 104, -435, -435, -433, -423, -433, -433, + -433, -433, -600, -600, -601, 291, 204, 206, 205, -369, + -369, -369, -369, 151, -435, -435, -369, -369, -369, -369, + 151, -369, -369, -369, -369, -369, -369, -369, -369, -369, + -369, -369, -369, -369, 89, 89, 89, 89, -427, 89, + -427, -427, -427, -427, -427, 151, -435, -240, -154, -558, + -557, -427, 44, -155, -241, -658, 718, 88, -376, -646, + 94, 94, 747, -160, 173, 19, 274, -160, 173, 726, + 184, -160, 568, 19, -400, -400, 94, 104, -400, 94, + 104, 274, 568, 274, 568, -285, -285, -285, 558, 559, + 183, 187, 186, -400, 185, -400, -400, 120, -400, -400, + -400, 38, -271, -260, -447, -447, -447, -621, -400, 95, + 94, -469, -466, -463, -400, -400, -459, -400, -389, -285, + -447, -447, -447, -447, -285, -321, 56, 57, 58, -463, + -201, 59, 60, -549, 64, -214, 88, 34, 88, -248, + -605, 38, -246, -400, -617, -141, 26, 305, -357, -425, + -425, -427, 425, 568, 274, -463, 305, -665, -412, -412, + -390, -389, -414, -409, -414, -414, -357, -410, -412, -412, + -427, -414, -410, -357, -400, 528, -357, -357, -504, -389, + -412, 94, -411, -400, -411, -447, -389, -390, -390, -285, + -285, -335, -342, -336, -343, 297, 271, 433, 434, 267, + 265, 11, 266, -351, 344, -448, 576, -316, -317, 80, + 45, -319, 295, 473, 469, 307, 311, 98, 312, 506, + 313, 276, 315, 316, 317, 332, 334, 287, 318, 319, + 320, 497, 321, 178, 333, 322, 323, 324, 451, -311, + 6, 386, 44, 54, 55, 520, 519, 624, 14, 308, + -400, 476, 613, 34, 39, 267, 271, 266, -621, -619, + 34, -400, 34, -469, -463, -400, -400, 174, 278, -231, + -233, -230, -226, -227, -232, -360, -362, -229, 88, -285, + -218, -400, -481, 174, 556, 558, 559, -651, -482, -651, + -482, 278, 35, 496, -485, 496, 35, -459, -479, 552, + 554, -474, 94, 497, -464, -484, 85, 170, -557, -482, + -482, -484, -484, 160, 174, -649, 557, 558, 261, -240, + 104, -639, -637, -400, 617, -400, -287, -285, -621, -468, + -459, -400, -539, -287, -287, -287, -402, -402, 88, 173, + 39, -400, -539, -400, -400, -400, -356, 174, -355, 19, + -401, -400, 38, 94, 173, -165, -163, 126, -427, -6, + 710, -427, -6, -6, -427, -6, -427, -537, 166, -292, + 104, 104, -379, 94, -379, 104, -531, 104, 104, 627, + 89, 94, -240, 695, -242, 23, -237, -236, -427, -550, + 64, -216, 88, -214, 34, 274, -539, -277, 130, 130, + 130, 27, -400, 26, -126, -107, -608, 173, 174, -246, + -488, -467, -464, -490, 151, -400, -475, 174, 14, 750, + 92, 278, -634, -633, 488, 89, 174, -561, 279, 575, + 94, 747, 504, 255, 256, 109, 404, 110, 111, -520, + -435, -431, -425, -425, -423, -423, -429, 292, -429, 119, + -300, 169, 168, -300, -427, 748, -426, -602, 126, -427, + 38, 174, 38, 174, 86, 174, 89, -527, -427, 89, + 173, 174, 89, 89, 19, 19, 140, 89, -427, 89, + 89, 89, 89, 19, 19, -427, 89, 173, 89, 89, + 89, 89, 86, 89, 174, 89, 89, 89, 89, 174, + 174, 174, -435, -435, -427, -435, 89, 89, 89, -427, + -427, -427, -435, 89, -427, -427, -427, -427, -427, -427, + -427, -427, -427, -427, -246, -427, -427, -498, 523, -498, + -498, -498, 89, -498, 89, 174, 89, 174, 89, 89, + 174, 174, 174, 174, 89, -242, 88, 104, 174, 742, + -383, -382, 94, -161, 278, -400, 726, -400, -161, -400, + -400, 130, -161, -400, 726, 94, 94, -285, -389, -285, + -389, 619, 42, 42, 184, 188, 188, 187, -400, 94, + 39, 26, 26, 342, -136, 614, -270, 88, 88, -285, + -285, -285, -623, 474, -400, -635, 174, 44, -633, 568, + -197, 355, -451, 86, -204, 362, 19, 14, -285, -285, + -285, -285, -299, 38, -472, 85, -551, -436, -597, 694, + -250, 89, -243, -595, -596, -243, -249, -400, -549, 88, + 89, 174, 19, -225, -286, -400, -143, 24, -400, -462, + -400, -400, -400, -460, 86, -400, -390, -357, -357, -414, + -357, -357, 174, 25, -412, -414, -414, -277, -410, -277, + 173, -277, -389, -526, 38, -247, 174, 23, 297, -284, + -397, -281, -283, 282, -417, -282, 285, -591, 283, 281, + 114, 286, 340, 115, 276, -397, -397, 282, -320, 278, + 38, -397, -338, 276, 407, 340, 283, 23, 297, -337, + 276, 115, -400, 282, 286, 283, 281, -396, 130, -388, + 160, 278, 46, 451, -396, 625, 297, -396, -396, -396, + -396, -396, -396, -396, 314, 314, -396, -396, -396, -396, + -396, -396, -396, -396, -396, -396, -396, 179, -396, -396, + -396, -396, -396, -396, 88, 309, 310, 342, 614, 124, + 627, 616, -462, 278, 543, 543, -624, 474, 34, 431, + 431, 432, -635, 427, 45, 34, -205, 425, -341, -339, + -411, 34, -363, -364, -365, -366, -368, -367, 71, 75, + 77, 81, 72, 73, 74, 531, 78, 83, 76, 34, + 174, -398, -403, 38, -400, 94, -398, -218, -233, -231, + -398, 88, -482, -650, -652, 560, 557, 563, -484, -484, + 104, 278, 88, 130, -484, -484, 44, -399, -647, 564, + 558, -242, -267, 728, 174, 85, -287, -261, -262, -263, + -264, -292, -376, 745, 209, 212, 214, 215, 216, 217, + 219, 220, 221, 222, 223, 226, 227, 228, 229, 230, + 231, 224, 225, 291, 204, 205, 206, 207, 191, 210, + 620, 192, 193, 194, 168, 169, 195, 198, 199, 200, + 201, 197, 232, 233, 234, 235, 236, 237, 238, 239, + 241, 240, 242, 243, 244, 245, 246, 247, 248, 249, + -400, -271, 94, 19, -267, -357, -221, -233, -400, 94, + -400, 151, 127, -6, 125, -169, -168, -167, 128, 708, + 714, 127, 127, 127, 89, 89, 89, 89, 174, 89, + 89, 89, 174, 89, 174, 104, -564, 533, -242, 94, + -155, 669, 174, -234, 40, 41, -551, -250, 89, -595, + -285, 94, -427, -400, 94, -427, 205, 173, 506, -400, + -577, 89, -490, 174, 278, 173, 173, -465, 454, -399, + -467, 23, 14, -376, 42, -383, 130, 747, -400, 89, + -429, -429, 119, -425, -422, 89, 127, -427, 125, -290, + -427, -290, -291, -297, 170, 208, 291, 207, 206, 204, + 163, 164, -310, -456, 619, -234, 89, 619, -400, -435, + -427, -427, -423, 89, -427, -427, 19, -400, -310, -423, + -427, -427, -427, -239, -239, 89, 89, -497, -498, -497, + -497, 89, 89, 89, 89, -497, 89, 89, 89, 89, + 89, 89, 89, 89, 89, 89, 89, 89, 174, 88, + -498, -498, -427, -498, -427, -498, -498, -427, 104, 106, + 104, 106, -557, -155, -659, 66, 716, 65, 496, 109, + 345, 174, 104, 94, 748, 174, 130, 425, -400, 19, + 173, 94, -400, 94, 19, 274, -400, 19, 19, -285, + -285, -285, 188, 94, -636, 349, 425, 568, 274, 425, + 349, 568, 274, -509, 104, -137, 124, 94, 462, -272, + -273, -274, -275, -276, 140, 175, 176, -261, -247, 88, + -247, -626, 535, 476, 486, -396, 378, -419, -418, 427, + 45, -544, 497, 482, 483, -466, 305, -389, 151, -632, + 101, 130, 85, 390, 394, 396, 398, 397, 399, 395, + 391, 392, 393, -445, -446, -444, -448, -389, 94, -619, + 88, 88, -214, 38, 138, -204, 362, 19, 88, 88, + 38, -521, 375, -292, 43, 174, 88, 89, 174, 64, + 174, 130, 89, 174, -1, -400, -285, -225, -400, 19, + 174, -618, 173, 104, -400, -459, -412, -357, -427, -427, + -357, -412, -412, -414, -400, -277, -521, -292, 38, -336, + 271, 266, -494, 342, 343, -495, -511, 345, -513, 88, + -289, -376, -282, -590, -591, -447, -400, 115, -590, 115, + 88, -289, -376, -376, -339, -376, -400, -400, -400, -400, + -346, -345, -376, -349, 35, -350, -400, -400, -400, -400, + 115, -400, 115, -315, 44, 51, 52, 53, -396, -396, + 211, -318, 44, 496, 498, 499, -349, 104, 104, 104, + 104, 94, 94, 94, -396, -396, 104, 94, -403, 94, + -592, 187, 48, 49, 104, 104, 104, 104, 44, 94, + -323, 44, 325, 329, 326, 327, 328, 94, 104, 44, + 104, 44, 104, 44, -400, 88, -593, -594, 94, -509, + 94, 88, 104, 94, 267, -462, 94, 85, -626, -396, + 431, -481, 130, 130, -419, -628, 98, 477, -628, -631, + 355, -207, 568, 35, -251, 271, 266, -619, -471, -470, + -376, -230, -230, -230, -230, -230, -230, 71, 82, 71, + -244, 88, 71, 76, 71, 76, 71, 76, 71, -365, + 71, 82, -471, -232, -247, -403, 89, -644, -643, -642, + -640, 79, 279, 80, -433, -484, 557, 561, 562, -467, + -415, 94, -474, -155, -285, -285, -542, 335, 336, 89, + 174, -292, -400, -359, 21, 173, 123, -6, -165, -167, + -427, -6, -427, 710, 444, 711, 94, 104, 104, -572, + 517, 512, 514, -155, -573, 504, 14, -236, -235, 47, + 89, 64, -239, 748, 748, 748, 748, 94, -400, 104, + 19, -464, -459, 151, 151, -400, 455, -475, 94, 475, + 94, 274, 748, 94, -383, -422, -427, 89, 38, 89, + 89, -528, -528, -527, -530, -527, -300, -300, 89, 88, + -234, 88, 89, 89, 26, 89, 89, 89, 89, -427, + 89, 89, 174, 174, 89, -547, 577, -548, 654, -497, -497, -497, -497, -497, -497, -497, -497, -497, -497, -497, - -497, -497, -497, -427, -438, -437, 293, 89, 174, 89, - 174, 89, 513, 718, 718, 513, 718, 718, 89, 174, - -599, 174, -391, 346, -391, -382, 94, -400, 94, 721, - -400, 743, 743, 721, -400, 94, -285, -389, -254, 529, - -211, 124, -212, 122, 46, 94, -400, 19, -400, -400, - 338, -400, 338, -400, -400, 94, -142, 622, 88, -139, - 610, 94, 89, 174, -376, 89, 38, -278, -279, -280, - -289, -281, -283, 38, -627, 98, -622, 94, -400, 95, - -400, -628, 172, 424, 44, 473, 474, 489, 419, 104, - 104, 479, -620, -400, -206, 270, 420, -206, -630, 55, - 130, 94, -285, -444, -388, 160, 312, -277, -400, 374, - -354, -353, -400, 94, -278, -214, -285, -285, 94, -278, - -278, -214, -522, 373, 23, 104, 150, 115, -436, -559, - -558, 64, -214, -243, -551, -596, -557, -400, 89, -248, - 86, 173, -233, -286, -400, 151, -357, -277, -357, -357, - -412, -522, -214, -506, 342, 88, -504, 88, -504, 115, - 387, -514, -512, 293, -344, 48, 50, -292, -588, -400, - -586, -588, -400, -586, -586, -447, -427, -344, -289, 274, - 34, 262, -347, 390, 384, 385, 390, 392, 394, 393, - -476, 337, 120, -476, 174, -234, 174, -400, -310, -310, - 34, 94, 94, -287, 89, 174, 130, 94, -139, -138, - -427, -215, -218, 274, 85, 270, -627, -622, 130, -482, - 94, 94, -628, 94, 94, -632, 130, -288, 270, -389, - 174, -251, -251, -357, 19, 174, 130, -256, -255, 85, - 86, -257, 85, -255, -255, 71, -245, 94, 71, 71, - 71, -357, -642, -641, 26, -591, -591, -591, 89, 89, - -258, 26, -263, 44, 374, -358, 22, 23, 151, 127, - 125, 127, 127, -400, 89, 89, -534, 691, -568, -570, - 507, 23, 23, -258, -574, 696, 94, 450, 48, 49, - -216, 64, -214, -551, -240, 743, -459, -475, 492, -285, - 174, 743, -290, -329, 94, -427, 89, -427, -427, 89, - 94, 89, 94, 89, -239, 23, -498, -427, -498, -427, - -498, 89, 174, 89, 89, 89, 174, 89, 89, -427, - 89, -599, -392, 205, 94, -392, -400, -400, 19, -401, - -209, 274, -277, -213, 369, 88, 365, -211, 184, 88, - 94, -400, 19, -400, -509, 338, -509, 338, 270, -400, - -267, -140, 611, 104, -138, 94, -452, 616, -274, -292, - 268, -214, 89, 174, -214, 94, -625, 483, -510, 379, - 104, 44, 104, 172, 475, -545, -198, 98, -287, 35, - -251, -198, -629, 98, 130, 742, 88, -396, -396, -396, - -209, 374, -400, 89, 174, -396, -396, 89, -210, 53, - -400, 89, 89, -308, 14, -523, 292, 104, 150, 104, - 150, 104, 17, 275, 89, -551, -398, -233, -400, -357, - -618, 173, -357, -523, -496, 343, 104, -423, 88, -423, - 88, -505, 340, 88, 89, 174, -400, -376, -305, -304, - -302, 109, 120, 44, 464, -303, 98, 160, 326, 329, - 328, 304, 327, -334, -416, 85, 684, 467, 384, 385, - -448, 691, 602, 699, 38, 277, 114, 115, 451, -417, - 88, 88, 86, 346, 88, 88, -588, 89, -344, -376, - 44, -347, 44, -348, 408, -457, -457, -457, -457, 337, - -345, -400, 160, -310, 89, -594, 94, 89, -462, 270, - -400, -625, 94, -484, -630, 94, -198, -287, -619, -239, - -233, -470, -557, -427, 88, -427, 89, 88, 71, 11, - 21, 17, -420, -400, -427, -435, 726, 728, 729, 276, - -6, 706, 439, -325, 692, 94, 23, 94, -566, 94, - -564, 94, -435, -551, -158, -322, -388, 309, 89, -328, - 140, 14, 89, 89, 89, -497, -497, -497, -500, -499, - -503, 513, 338, 521, -435, 89, 89, 94, 94, 89, - 89, 94, 94, 94, 721, 420, -209, 38, 457, 24, - 628, 370, -246, 366, 367, 368, -400, 94, -435, -215, - 742, 374, -400, 19, 94, -509, 94, -509, -400, 338, - 38, 94, 89, 94, 94, -265, -292, -202, 14, -308, - -280, -202, 23, 14, 172, 423, 44, 104, 44, 476, - 94, -206, 130, 110, 111, -384, -385, 94, -454, -310, - -312, 94, -400, -353, -420, -420, -306, -214, 38, -307, - -351, -448, -209, 30, 374, -157, -156, -306, 88, -524, - 178, 104, 150, 104, 104, -471, -357, -357, -524, -513, - 23, 89, -491, 89, -491, 88, 130, -423, -512, -515, - 64, -302, 109, -423, 94, -312, -313, 44, 325, 321, - 130, 130, -314, 44, 305, 306, -324, 88, 336, 17, - 104, 211, 88, 700, 88, 115, 115, -285, -454, -454, - -589, 386, 387, 388, 395, 390, 391, 389, 392, 393, - 394, -589, -454, -454, 88, -477, -476, -423, -457, 130, - -458, 283, 400, 401, 98, 14, 384, 385, 405, 404, - 403, 409, 410, 414, 415, 411, 413, 412, 416, 417, - 418, 406, 407, 408, 423, 434, -396, 160, -400, 173, - -629, -240, -357, -246, -587, -400, 277, 23, 23, -543, - 14, 727, 88, 88, -400, -400, -380, 693, 104, 94, - 509, -572, -535, 694, -562, -504, -310, 130, 89, 78, - 615, 617, 89, -502, 122, 475, 479, -421, -424, 104, - 106, 202, 172, -498, -498, 89, 89, -400, -400, -285, - 94, 104, 89, 119, 119, 89, 89, -387, -386, 94, - -400, 374, -400, -267, 94, -267, 94, 338, -509, -2, - 616, -203, 63, 559, 94, 95, 470, 94, 95, 104, - 423, -198, 94, 743, 174, 130, 89, -510, -492, 293, - -214, 174, -351, -388, -400, -158, -492, -309, -352, -400, - 94, -541, 187, 372, 14, 104, 150, 104, -239, -525, - 187, 372, -495, 89, 89, 89, -491, 104, 89, -519, - -516, 88, -351, 295, 140, 94, 94, 104, 88, -552, - 34, 94, 38, -427, -455, 88, 89, 89, 89, 89, - -454, 110, 111, -396, -396, 94, 94, 383, -396, -396, - -396, -396, -396, -396, 88, 94, 94, -396, -396, -396, - -396, 130, -396, -396, -310, -396, 173, -400, 89, 89, - 174, 729, 88, -435, -435, 88, 23, -534, -536, 695, - 94, -571, 512, -565, -563, 507, 508, 509, 510, 94, - 616, 68, 618, -501, -502, 479, -421, -424, 689, 519, - 519, 519, 94, -400, 94, 743, 174, 130, -400, 374, - -267, -267, -509, 94, -268, -400, 336, 492, -385, 94, - -457, -493, 345, 23, -351, -396, -510, -493, 89, 174, - -396, -396, 372, 104, 150, 104, -240, 372, -507, 344, - 89, -519, -351, -518, -517, 343, 296, 88, 89, -427, - -439, -396, 89, 88, 89, -327, -326, 613, -454, -457, - 86, -457, 86, -457, 86, -457, 86, 89, 104, 104, - -400, 104, 104, 104, 104, 104, 104, -491, 104, 104, - 104, 104, 110, 111, 104, 104, -310, -400, -400, 277, - -153, 88, 89, 89, -381, -400, -566, -325, 94, -575, - 275, -569, -570, 511, -563, 23, 509, 23, 23, -159, - 174, 68, 119, 520, 520, 520, -211, -212, -211, -212, - -267, -386, 94, -400, 94, -267, -266, 38, 514, 450, - 23, -494, -310, -352, -420, -420, 104, 104, 89, 174, - -400, 292, 88, -434, -428, -427, 292, 89, -400, -427, - -478, 702, 701, -333, -331, -332, 85, 526, 334, 335, - 89, -589, -589, -589, -589, -334, 89, 89, 174, -433, - 89, 174, -380, -582, 88, 104, -568, -567, -569, 23, - -566, 23, -566, -566, 516, 14, -501, -211, -211, -267, - 94, -376, 88, -506, -517, -516, -434, 89, 174, -476, - 89, -332, 85, -331, 85, 18, 17, -457, -457, -457, - -457, 88, 89, -400, -585, 34, 89, -581, -580, -377, - -576, -400, 512, 513, 94, -566, 130, 617, -662, -661, - 717, -491, -496, 89, -428, -478, -330, 331, 332, 34, - 187, -330, -433, -584, -583, -378, 89, 174, 173, 94, - 618, 94, 89, -513, 109, 44, 333, 89, 174, 130, - -580, -400, -583, 44, -427, 173, -400, + -497, -497, -497, -497, -497, -497, -497, -427, -438, -437, + 297, 89, 174, 89, 174, 89, 518, 723, 723, 518, + 723, 723, 89, 174, -599, 174, -391, 350, -391, -382, + 94, -400, 94, 726, -400, 748, 748, 726, -400, 94, + -285, -389, -254, 534, -211, 124, -212, 122, 46, 94, + -400, 19, -400, -400, 342, -400, 342, -400, -400, 94, + -142, 627, 88, -139, 615, 94, 89, 174, -376, 89, + 38, -278, -279, -280, -289, -281, -283, 38, -627, 98, + -622, 94, -400, 95, -400, -628, 172, 429, 44, 478, + 479, 494, 424, 104, 104, 484, -620, -400, -206, 274, + 425, -206, -630, 55, 130, 94, -285, -444, -388, 160, + 316, -277, -400, 378, -354, -353, -400, 94, -278, -214, + -285, -285, 94, -278, -278, -214, -522, 377, 23, 104, + 150, 115, -436, -559, -558, 64, -214, -243, -551, -596, + -557, -400, 89, -248, 86, 173, -233, -286, -400, 151, + -357, -277, -357, -357, -412, -522, -214, -506, 346, 88, + -504, 88, -504, 115, 391, -514, -512, 297, -344, 48, + 50, -292, -588, -400, -586, -588, -400, -586, -586, -447, + -427, -344, -289, 278, 34, 266, -347, 394, 388, 389, + 394, 396, 398, 397, 399, -476, 341, 120, -476, 174, + -234, 174, -400, -310, -310, 34, 94, 94, -287, 89, + 174, 130, 94, -139, -138, -427, -215, -218, 278, 85, + 274, -627, -622, 130, -482, 94, 94, -628, 94, 94, + -632, 130, -288, 274, -389, 174, -251, -251, -357, 19, + 174, 130, -256, -255, 85, 86, -257, 85, -255, -255, + 71, -245, 94, 71, 71, 71, -357, -642, -641, 26, + -591, -591, -591, 89, 89, -258, 26, -263, 44, 378, + -358, 22, 23, 151, 127, 125, 127, 127, -400, 89, + 89, -534, 696, -568, -570, 512, 23, 23, -258, -574, + 701, 94, 455, 48, 49, -216, 64, -214, -551, -240, + 748, -459, -475, 497, -285, 174, 748, -290, -329, 94, + -329, -427, 89, -427, -427, 89, 94, 89, 94, 89, + -239, 23, -498, -427, -498, -427, -498, 89, 174, 89, + 89, 89, 174, 89, 89, -427, 89, -599, -392, 205, + 94, -392, -400, -400, 19, -401, -209, 278, -277, -213, + 373, 88, 369, -211, 184, 88, 94, -400, 19, -400, + -509, 342, -509, 342, 274, -400, -267, -140, 616, 104, + -138, 94, -452, 621, -274, -292, 272, -214, 89, 174, + -214, 94, -625, 488, -510, 383, 104, 44, 104, 172, + 480, -545, -198, 98, -287, 35, -251, -198, -629, 98, + 130, 747, 88, -396, -396, -396, -209, 378, -400, 89, + 174, -396, -396, 89, -210, 53, -400, 89, 89, -308, + 14, -523, 296, 104, 150, 104, 150, 104, 17, 279, + 89, -551, -398, -233, -400, -357, -618, 173, -357, -523, + -496, 347, 104, -423, 88, -423, 88, -505, 344, 88, + 89, 174, -400, -376, -305, -304, -302, 109, 120, 44, + 469, -303, 98, 160, 330, 333, 332, 308, 331, -334, + -416, 85, 689, 472, 388, 389, -448, 696, 607, 704, + 38, 281, 114, 115, 456, -417, 88, 88, 86, 350, + 88, 88, -588, 89, -344, -376, 44, -347, 44, -348, + 413, -457, -457, -457, -457, -457, 341, -345, -400, 160, + -310, 89, -594, 94, 89, -462, 274, -400, -625, 94, + -484, -630, 94, -198, -287, -619, -239, -233, -470, -557, + -427, 88, -427, 89, 88, 71, 11, 21, 17, -420, + -400, -427, -435, 731, 733, 734, 280, -6, 711, 444, + -325, 697, 94, 23, 94, -566, 94, -564, 94, -435, + -551, -158, -322, -388, 313, 89, -328, 140, 14, 89, + 89, 89, 89, -497, -497, -497, -500, -499, -503, 518, + 342, 526, -435, 89, 89, 94, 94, 89, 89, 94, + 94, 94, 726, 425, -209, 38, 462, 24, 633, 374, + -246, 370, 371, 372, -400, 94, -435, -215, 747, 378, + -400, 19, 94, -509, 94, -509, -400, 342, 38, 94, + 89, 94, 94, -265, -292, -202, 14, -308, -280, -202, + 23, 14, 172, 428, 44, 104, 44, 481, 94, -206, + 130, 110, 111, -384, -385, 94, -454, -310, -312, 94, + -400, -353, -420, -420, -306, -214, 38, -307, -351, -448, + -209, 30, 378, -157, -156, -306, 88, -524, 178, 104, + 150, 104, 104, -471, -357, -357, -524, -513, 23, 89, + -491, 89, -491, 88, 130, -423, -512, -515, 64, -302, + 109, -423, 94, -312, -313, 44, 329, 325, 130, 130, + -314, 44, 309, 310, -324, 88, 340, 17, 104, 211, + 88, 705, 88, 115, 115, -285, -454, -454, -589, 390, + 391, 392, 400, 394, 395, 393, 396, 397, 398, 399, + -589, -454, -454, 88, -477, -476, -423, -457, 130, -458, + 287, 405, 406, 98, 14, 388, 389, 410, 409, 408, + 414, 415, 419, 420, 416, 418, 417, 421, 422, 423, + 411, 412, 57, 413, 428, 439, 435, -396, 160, -400, + 173, -629, -240, -357, -246, -587, -400, 281, 23, 23, + -543, 14, 732, 88, 88, -400, -400, -380, 698, 104, + 94, 514, -572, -535, 699, -562, -504, -310, 130, 89, + 78, 620, 622, 89, -502, 122, 480, 484, -421, -424, + 104, 106, 202, 172, -498, -498, 89, 89, -400, -400, + -285, 94, 104, 89, 119, 119, 89, 89, -387, -386, + 94, -400, 378, -400, -267, 94, -267, 94, 342, -509, + -2, 621, -203, 63, 564, 94, 95, 475, 94, 95, + 104, 428, -198, 94, 748, 174, 130, 89, -510, -492, + 297, -214, 174, -351, -388, -400, -158, -492, -309, -352, + -400, 94, -541, 187, 376, 14, 104, 150, 104, -239, + -525, 187, 376, -495, 89, 89, 89, -491, 104, 89, + -519, -516, 88, -351, 299, 140, 94, 94, 104, 88, + -552, 34, 94, 38, -427, -455, 88, 89, 89, 89, + 89, -454, 110, 111, -396, -396, 94, 94, 387, -396, + -396, -396, -396, -396, -396, 88, 94, 94, -396, -396, + -396, -396, 130, -396, -396, -396, -310, -396, 173, -400, + 89, 89, 174, 734, 88, -435, -435, 88, 23, -534, + -536, 700, 94, -571, 517, -565, -563, 512, 513, 514, + 515, 94, 621, 68, 623, -501, -502, 484, -421, -424, + 694, 524, 524, 524, 94, -400, 94, 748, 174, 130, + -400, 378, -267, -267, -509, 94, -268, -400, 340, 497, + -385, 94, -457, -493, 349, 23, -351, -396, -510, -493, + 89, 174, -396, -396, 376, 104, 150, 104, -240, 376, + -507, 348, 89, -519, -351, -518, -517, 347, 300, 88, + 89, -427, -439, -396, 89, 88, 89, -327, -326, 618, + -454, -457, 86, -457, 86, -457, 86, -457, 86, 89, + 104, 104, -400, 104, 104, 104, 104, 104, 104, -491, + 104, 104, 104, 104, 110, 111, 104, 104, 104, -310, + -400, -400, 281, -153, 88, 89, 89, -381, -400, -566, + -325, 94, -575, 279, -569, -570, 516, -563, 23, 514, + 23, 23, -159, 174, 68, 119, 525, 525, 525, -211, + -212, -211, -212, -267, -386, 94, -400, 94, -267, -266, + 38, 519, 455, 23, -494, -310, -352, -420, -420, 104, + 104, 89, 174, -400, 296, 88, -434, -428, -427, 296, + 89, -400, -427, -478, 707, 706, -333, -331, -332, 85, + 531, 338, 339, 89, -589, -589, -589, -589, -334, 89, + 89, 174, -433, 89, 174, -380, -582, 88, 104, -568, + -567, -569, 23, -566, 23, -566, -566, 521, 14, -501, + -211, -211, -267, 94, -376, 88, -506, -517, -516, -434, + 89, 174, -476, 89, -332, 85, -331, 85, 18, 17, + -457, -457, -457, -457, 88, 89, -400, -585, 34, 89, + -581, -580, -377, -576, -400, 517, 518, 94, -566, 130, + 622, -662, -661, 722, -491, -496, 89, -428, -478, -330, + 335, 336, 34, 187, -330, -433, -584, -583, -378, 89, + 174, 173, 94, 623, 94, 89, -513, 109, 44, 337, + 89, 174, 130, -580, -400, -583, 44, -427, 173, -400, } var yyDef = [...]int{ @@ -11501,51 +11615,50 @@ var yyDef = [...]int{ 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 0, - 336, 337, 338, 339, 340, 341, 1061, 1062, 1063, 1064, - 1065, 1066, 1067, 1068, 1069, 1070, 0, 0, 0, 817, - 0, 804, 782, 783, 743, 0, 0, 0, 0, 0, - 0, 0, 600, 601, 602, 603, 604, 605, 606, 607, - 608, 609, 610, 611, 612, 613, 614, 615, 616, 617, - 618, 619, 620, 621, 622, 623, 624, 625, 626, 627, - 628, 629, 630, 631, 632, 633, 634, 635, 636, 637, - 638, 639, 640, 641, 642, 456, 457, 458, 459, 460, + 336, 337, 338, 339, 340, 341, 1062, 1063, 1064, 1065, + 1066, 1067, 1068, 1069, 1070, 1071, 0, 0, 0, 818, + 0, 805, 783, 784, 744, 0, 0, 0, 0, 0, + 0, 0, 601, 602, 603, 604, 605, 606, 607, 608, + 609, 610, 611, 612, 613, 614, 615, 616, 617, 618, + 619, 620, 621, 622, 623, 624, 625, 626, 627, 628, + 629, 630, 631, 632, 633, 634, 635, 636, 637, 638, + 639, 640, 641, 642, 643, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 0, 370, 366, - 278, 279, 280, 281, 282, 283, 284, 378, 379, 577, - 0, 0, 0, 0, 877, -2, 122, 0, 0, 0, - 0, 0, 359, 0, 350, 350, 0, 0, 1071, 1072, - 1073, 1074, 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, - 1083, -2, 0, 0, 795, 744, 745, 746, 747, 748, - 749, 750, 751, 752, 753, 754, 755, 756, 757, 758, - 759, 760, 761, 762, 439, 440, 441, 435, 436, 438, - 437, -2, 0, 0, 795, 0, 0, 0, 885, 0, - 0, 0, 930, 948, 23, 0, 7, 9, 10, 11, + 278, 279, 280, 281, 282, 283, 284, 378, 379, 578, + 0, 0, 0, 0, 878, -2, 122, 0, 0, 0, + 0, 0, 359, 0, 350, 350, 0, 0, 1072, 1073, + 1074, 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, 1083, + 1084, -2, 0, 0, 796, 745, 746, 747, 748, 749, + 750, 751, 752, 753, 754, 755, 756, 757, 758, 759, + 760, 761, 762, 763, 439, 440, 441, 435, 436, 438, + 437, -2, 0, 0, 796, 0, 0, 0, 886, 0, + 0, 0, 931, 949, 23, 0, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 0, 0, 19, - 0, 19, 0, 0, 0, 1575, 1576, 1577, 1578, 2470, - 2440, -2, 2188, 2148, 2364, 2365, 2255, 2269, 2141, 2517, - 2518, 2519, 2520, 2521, 2522, 2523, 2524, 2525, 2526, 2527, - 2528, 2529, 2530, 2531, 2532, 2533, 2534, 2535, 2536, 2537, - 2538, 2539, 2540, 2541, 2542, 2543, 2544, 2545, 2546, 2547, - 2548, 2549, 2550, 2551, 2552, 2553, 2554, 2555, 2556, 2557, - 2558, 2559, 2560, 2561, 2562, 2563, 2564, 2565, 2566, 2567, - 2568, 2569, 2570, 2094, 2095, 2096, 2097, 2098, 2099, 2100, - 2101, 2102, 2103, 2104, 2105, 2106, 2107, 2108, 2109, 2110, + 0, 19, 0, 0, 0, 1580, 1581, 1582, 1583, 2485, + 2455, -2, 2199, 2158, 2379, 2380, 2270, 2284, 2151, 2532, + 2533, 2534, 2535, 2536, 2537, 2538, 2539, 2540, 2541, 2542, + 2543, 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, + 2553, 2554, 2555, 2556, 2557, 2558, 2559, 2560, 2561, 2562, + 2563, 2564, 2565, 2566, 2567, 2568, 2569, 2570, 2571, 2572, + 2573, 2574, 2575, 2576, 2577, 2578, 2579, 2580, 2581, 2582, + 2583, 2584, 2585, 2104, 2105, 2106, 2107, 2108, 2109, 2110, 2111, 2112, 2113, 2114, 2115, 2116, 2117, 2118, 2119, 2120, 2121, 2122, 2123, 2124, 2125, 2126, 2127, 2128, 2129, 2130, 2131, 2132, 2133, 2134, 2135, 2136, 2137, 2138, 2139, 2140, - 2142, 2143, 2144, 2145, 2146, 2147, 2149, 2150, 2151, 2152, - 2153, 2154, 2155, 2156, 2157, 2158, 2159, 2160, 2161, 2162, + 2141, 2142, 2143, 2144, 2145, 2146, 2147, 2148, 2149, 2150, + 2152, 2153, 2154, 2155, 2156, 2157, 2159, 2160, 2161, 2162, 2163, 2164, 2165, 2166, 2167, 2168, 2169, 2170, 2171, 2172, 2173, 2174, 2175, 2176, 2177, 2178, 2179, 2180, 2181, 2182, - 2183, 2184, 2185, 2186, 2187, 2189, 2190, 2191, 2192, 2193, - 2194, 2195, 2196, 2197, 2198, 2199, 2200, 2201, 2202, 2203, + 2183, 2184, 2185, 2186, 2187, 2188, 2189, 2190, 2191, 2192, + 2193, 2194, 2195, 2196, 2197, 2198, 2200, 2201, 2202, 2203, 2204, 2205, 2206, 2207, 2208, 2209, 2210, 2211, 2212, 2213, 2214, 2215, 2216, 2217, 2218, 2219, 2220, 2221, 2222, 2223, 2224, 2225, 2226, 2227, 2228, 2229, 2230, 2231, 2232, 2233, 2234, 2235, 2236, 2237, 2238, 2239, 2240, 2241, 2242, 2243, 2244, 2245, 2246, 2247, 2248, 2249, 2250, 2251, 2252, 2253, - 2254, 2256, 2257, 2258, 2259, 2260, 2261, 2262, 2263, 2264, - 2265, 2266, 2267, 2268, 2271, 2272, 2273, 2274, 2275, 2276, - 2277, 2278, 2279, 2280, 2281, 2282, 2283, 2284, 2285, 2286, + 2254, 2255, 2256, 2257, 2258, 2259, 2260, 2261, 2262, 2263, + 2264, 2265, 2266, 2267, 2268, 2269, 2271, 2272, 2273, 2274, + 2275, 2276, 2277, 2278, 2279, 2280, 2281, 2282, 2283, 2286, 2287, 2288, 2289, 2290, 2291, 2292, 2293, 2294, 2295, 2296, 2297, 2298, 2299, 2300, 2301, 2302, 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2310, 2311, 2312, 2313, 2314, 2315, 2316, @@ -11553,421 +11666,425 @@ var yyDef = [...]int{ 2327, 2328, 2329, 2330, 2331, 2332, 2333, 2334, 2335, 2336, 2337, 2338, 2339, 2340, 2341, 2342, 2343, 2344, 2345, 2346, 2347, 2348, 2349, 2350, 2351, 2352, 2353, 2354, 2355, 2356, - 2357, 2358, 2359, 2360, 2361, 2362, 2363, 2366, 2367, 2368, - 2369, 2370, 2371, 2372, 2373, 2374, 2375, 2376, 2377, 2378, - 2379, 2380, 2381, 2382, 2383, 2384, 2385, 2386, 2387, 2388, - 2389, 2390, 2391, 2392, 2393, 2394, 2395, 2396, -2, 2398, + 2357, 2358, 2359, 2360, 2361, 2362, 2363, 2364, 2365, 2366, + 2367, 2368, 2369, 2370, 2371, 2372, 2373, 2374, 2375, 2376, + 2377, 2378, 2381, 2382, 2383, 2384, 2385, 2386, 2387, 2388, + 2389, 2390, 2391, 2392, 2393, 2394, 2395, 2396, 2397, 2398, 2399, 2400, 2401, 2402, 2403, 2404, 2405, 2406, 2407, 2408, - 2409, 2410, 2411, 2412, 2413, 2414, 2415, 2416, 2417, 2418, + 2409, 2410, 2411, -2, 2413, 2414, 2415, 2416, 2417, 2418, 2419, 2420, 2421, 2422, 2423, 2424, 2425, 2426, 2427, 2428, 2429, 2430, 2431, 2432, 2433, 2434, 2435, 2436, 2437, 2438, - 2439, 2441, 2442, 2443, 2444, 2445, 2446, 2447, 2448, 2449, - 2450, 2451, 2452, 2453, 2454, 2455, -2, -2, -2, 2459, + 2439, 2440, 2441, 2442, 2443, 2444, 2445, 2446, 2447, 2448, + 2449, 2450, 2451, 2452, 2453, 2454, 2456, 2457, 2458, 2459, 2460, 2461, 2462, 2463, 2464, 2465, 2466, 2467, 2468, 2469, - 2471, 2472, 2473, 2474, 2475, 2476, 2477, 2478, 2479, 2480, - 2481, 2482, 2483, 2484, 2485, 2486, 2487, 2488, 2489, 2490, + 2470, -2, -2, -2, 2474, 2475, 2476, 2477, 2478, 2479, + 2480, 2481, 2482, 2483, 2484, 2486, 2487, 2488, 2489, 2490, 2491, 2492, 2493, 2494, 2495, 2496, 2497, 2498, 2499, 2500, - 2501, 2502, 2503, 2504, 2505, 2506, 2507, 0, 334, 332, - 2113, 2141, 2148, 2188, 2255, 2269, 2270, 2310, 2364, 2365, - 2397, 2440, 2456, 2457, 2458, 2470, 0, 0, 1101, 0, - 371, 784, 785, 818, 885, 913, 0, 805, 806, 0, - 741, 0, 1520, 412, 0, 2165, 416, 2447, 0, 0, - 0, 0, 738, 406, 407, 408, 409, 410, 411, 0, - 0, 1060, 0, 0, 2477, 402, 0, 365, 2257, 2469, - 1579, 0, 0, 0, 0, 0, 221, 1236, 223, 1238, - 227, 235, 0, 0, 0, 240, 241, 244, 245, 246, - 247, 248, 0, 252, 0, 254, 257, 0, 259, 260, - 0, 263, 264, 265, 0, 275, 276, 277, 1239, 1240, - 1241, 1242, 1243, 1244, 1245, 1246, -2, 150, 1099, 2047, - 1931, 0, 1938, 1951, 1962, 1669, 1670, 1671, 1672, 0, - 0, 0, 0, 0, 0, 1680, 1681, 0, 1724, 2521, - 2566, 2567, 0, 1690, 1691, 1692, 1693, 1694, 1695, 0, - 161, 173, 174, 1984, 1985, 1986, 1987, 1988, 1989, 1990, - 0, 1992, 1993, 1994, 0, 1654, 1575, 0, 2530, 2540, - 0, 2554, 2561, 2562, 2563, 2564, 2553, 0, 0, 1887, - 0, 1877, 0, 0, -2, -2, 0, 0, 2337, -2, - 2568, 2569, 2570, 2527, 2550, 2558, 2559, 2560, 2531, 2534, - 2557, 2523, 2524, 2525, 2518, 2519, 2520, 2522, 2536, 2538, - 2549, 0, 2545, 2555, 2556, 2445, 0, 0, 2494, 2532, - 2533, 0, 0, 0, 0, 0, 0, 2503, 2504, 2505, - 2506, 2507, 2489, 175, 176, -2, -2, -2, -2, -2, - -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, - -2, 1898, -2, 1900, -2, 1902, -2, 1904, -2, -2, - -2, -2, 1909, 1910, -2, 1912, -2, -2, -2, -2, - -2, -2, -2, 1889, 1890, 1891, 1892, 1881, 1882, 1883, - 1884, 1885, 1886, -2, -2, -2, 913, 1008, 0, 913, - 0, 886, 935, 938, 941, 944, 889, 0, 0, 123, - 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 360, 361, 349, 351, 0, 355, - 0, 0, 351, 348, 342, 0, 1301, 1301, 1301, 1301, - 0, 0, 0, 1301, 1301, 1301, 1301, 1301, 0, 1301, - 0, 0, 0, 0, 0, 1301, 0, 1137, 1248, 1249, - 1250, 1299, 1300, 1406, 0, 0, 0, 851, 0, 0, - 856, 899, 0, 901, 904, 800, 796, 797, 798, 799, - 77, 643, 0, 0, 0, 718, 718, 973, 973, 0, - 661, 0, 0, 0, 718, 0, 675, 667, 0, 0, - 0, 718, 0, 0, 906, 906, 0, 721, 728, 718, - 718, -2, 718, 718, 0, 713, 718, 0, 0, 0, - 1315, 681, 682, 683, 667, 667, 686, 687, 688, 698, - 699, 729, 2089, 0, 0, 577, 577, 0, 577, 0, - 0, 577, 0, 577, 577, 577, 0, 802, 2210, 2305, - 2182, 2275, 2123, 2257, 2469, 0, 307, 2337, 312, 0, - 2187, 2213, 0, 0, 2232, 0, -2, 0, 388, 913, - 0, 0, 885, 0, 0, 0, 577, 577, 577, 577, - 577, 577, 577, 1405, 577, 577, 577, 577, 577, 0, - 0, 0, 577, 0, 577, 577, 577, 0, 949, 950, - 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, - 5, 6, 19, 0, 0, 0, 0, 0, 0, 129, - 128, 0, 2048, 2084, 1997, 1998, 1999, 0, 2071, 2002, - 2075, 2075, 2075, 2075, 2032, 2033, 2034, 2035, 2036, 2037, - 2038, 2039, 2040, 2041, 2075, 2075, 0, 0, 2046, 2023, - 2073, 2073, 2073, 2071, 2050, 2003, 2004, 2005, 2006, 2007, - 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2078, - 2078, 2081, 2081, 2078, 2051, 2052, 2053, 2054, 2055, 2056, - 2057, 2058, 2059, 2060, 2061, 2062, 2063, 2064, 2065, 2066, - 2067, 2068, 0, 454, 452, 453, 1927, 0, 0, 913, - -2, 0, 0, 851, 0, 742, 1518, 0, 0, 413, - 1580, 0, 0, 417, 0, 418, 0, 0, 420, 0, + 2501, 2502, 2503, 2504, 2505, 2506, 2507, 2508, 2509, 2510, + 2511, 2512, 2513, 2514, 2515, 2516, 2517, 2518, 2519, 2520, + 2521, 2522, 0, 334, 332, 2123, 2151, 2158, 2199, 2270, + 2284, 2285, 2325, 2379, 2380, 2412, 2455, 2471, 2472, 2473, + 2485, 0, 0, 1102, 0, 371, 785, 786, 819, 886, + 914, 0, 806, 807, 0, 742, 0, 1524, 412, 0, + 2175, 416, 2462, 0, 0, 0, 0, 739, 406, 407, + 408, 409, 410, 411, 0, 0, 1061, 0, 0, 2492, + 402, 0, 365, 2272, 2484, 1584, 0, 0, 0, 0, + 0, 221, 1237, 223, 1239, 227, 235, 0, 0, 0, + 240, 241, 244, 245, 246, 247, 248, 0, 252, 0, + 254, 257, 0, 259, 260, 0, 263, 264, 265, 0, + 275, 276, 277, 1240, 1241, 1242, 1243, 1244, 1245, 1246, + 1247, -2, 150, 1100, 2057, 1937, 0, 1944, 1957, 1968, + 1674, 1675, 1676, 1677, 0, 0, 0, 0, 0, 0, + 1685, 1686, 0, 1730, 2536, 2581, 2582, 0, 1695, 1696, + 1697, 1698, 1699, 1700, 0, 2197, 161, 173, 174, 1990, + 1991, 1992, 1993, 1994, 1995, 1996, 0, 1998, 1999, 2000, + 0, 1659, 1580, 0, 2545, 2555, 0, 2569, 2576, 2577, + 2578, 2579, 2568, 0, 0, 1893, 0, 1883, 0, 0, + -2, -2, 0, 0, 2352, -2, 2583, 2584, 2585, 2542, + 2565, 2573, 2574, 2575, 2546, 2549, 2572, 2538, 2539, 2540, + 2533, 2534, 2535, 2537, 2551, 2553, 2564, 0, 2560, 2570, + 2571, 2460, 0, 0, 2509, 2547, 2548, 0, 0, 0, + 0, 0, 0, 2518, 2519, 2520, 2521, 2522, 2504, 175, + 176, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, 1904, -2, 1906, + -2, 1908, -2, 1910, -2, -2, -2, -2, 1915, 1916, + -2, 1918, -2, -2, -2, -2, -2, -2, -2, 1895, + 1896, 1897, 1898, 1887, 1888, 1889, 1890, 1891, 1892, -2, + -2, -2, 914, 1009, 0, 914, 0, 887, 936, 939, + 942, 945, 890, 0, 0, 123, 124, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 360, 361, 349, 351, 0, 355, 0, 0, 351, 348, + 342, 0, 1305, 1305, 1305, 1305, 0, 0, 0, 1305, + 1305, 1305, 1305, 1305, 0, 1305, 0, 0, 0, 0, + 0, 1305, 0, 1138, 1249, 1250, 1251, 1303, 1304, 1410, + 0, 0, 0, 852, 0, 0, 857, 900, 0, 902, + 905, 801, 797, 798, 799, 800, 77, 644, 0, 0, + 0, 719, 719, 974, 974, 0, 662, 0, 0, 0, + 719, 0, 676, 668, 0, 0, 0, 719, 0, 0, + 907, 907, 0, 722, 729, 719, 719, -2, 719, 719, + 0, 714, 719, 0, 0, 0, 1319, 682, 683, 684, + 668, 668, 687, 688, 689, 699, 700, 730, 2099, 0, + 0, 578, 578, 0, 578, 0, 0, 578, 0, 578, + 578, 578, 0, 803, 2225, 2320, 2192, 2290, 2133, 2272, + 2484, 0, 307, 2352, 312, 0, 2198, 2228, 0, 0, + 2247, 0, -2, 0, 388, 914, 0, 0, 886, 0, + 0, 0, 578, 578, 578, 578, 578, 578, 578, 1409, + 578, 578, 578, 578, 578, 0, 0, 0, 578, 0, + 578, 578, 578, 0, 950, 951, 953, 954, 955, 956, + 957, 958, 959, 960, 961, 962, 5, 6, 19, 0, + 0, 0, 0, 0, 0, 129, 128, 0, 2058, 2094, + 2003, 2004, 2005, 0, 2081, 2008, 2085, 2085, 2085, 2085, + 2038, 2039, 2040, 2041, 2042, 2043, 2044, 2045, 2046, 2047, + 2085, 2085, 2085, 2085, 2085, 2085, 0, 0, 2056, 2029, + 2083, 2083, 2083, 2081, 2060, 2009, 2010, 2011, 2012, 2013, + 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2088, + 2088, 2091, 2091, 2088, 2061, 2062, 2063, 2064, 2065, 2066, + 2067, 2068, 2069, 2070, 2071, 2072, 2073, 2074, 2075, 2076, + 2077, 2078, 0, 454, 452, 453, 1933, 0, 0, 914, + -2, 0, 0, 852, 0, 743, 1522, 0, 0, 413, + 1585, 0, 0, 417, 0, 418, 0, 0, 420, 0, 0, 0, 442, 0, 445, 428, 429, 430, 431, 432, 424, 0, 201, 0, 404, 405, 401, 0, 0, 367, - 0, 0, 0, 578, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 579, 0, 0, 0, 0, 0, 0, 232, 228, 236, 239, 249, 256, 0, 268, 270, 273, 229, 237, 242, 243, 250, 271, 230, 233, 234, 238, 272, 274, 231, 251, 255, 269, 253, 258, 261, 262, - 267, 0, 202, 0, 0, 0, 0, 0, 1937, 0, - 0, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - -2, 1931, 0, 0, 1675, 1676, 1677, 1678, 0, 1682, - 0, 1725, 0, 0, 0, 0, 0, 0, 1991, 1995, - 0, 0, 1927, 1927, 0, 0, 1927, 1923, 0, 0, - 0, 0, 0, 0, 1927, 1860, 0, 0, 1862, 1878, - 0, 0, 1864, 1865, 0, 1868, 1869, 1927, 0, 1927, - 1873, 1927, 1927, 1927, 1854, 1855, 0, 0, 0, 1923, - 1923, 1923, 1923, 0, 0, 1923, 1923, 1923, 1923, 1923, - 1923, 1923, 1923, 1923, 1923, 1923, 1923, 1923, 1923, 1923, - 1923, 1923, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 906, 0, 914, 0, -2, 0, - 932, 934, 936, 937, 939, 940, 942, 943, 945, 946, - 891, 0, 0, 125, 0, 0, 0, 106, 0, 0, - 104, 0, 0, 0, 0, 75, 81, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 353, 0, 358, 344, 2298, 0, 343, 0, 0, 0, - 0, 0, 0, 1098, 0, 0, 1301, 1301, 1301, 1138, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1301, - 1301, 1301, 1301, 0, 1321, 0, 0, 0, 0, 0, - 851, 855, 0, 900, 0, 0, 802, 801, 74, 78, - 645, 649, 650, 651, 0, 973, 0, 0, 654, 655, - 0, 656, 0, 0, 667, 718, 718, 673, 674, 669, - 668, 724, 725, 721, 0, 721, 721, 973, 0, 692, - 693, 694, 718, 718, 700, 907, 0, 701, 702, 721, - 0, 726, 727, 973, 0, 0, 973, 973, 0, 710, - 711, 0, 714, 718, 0, 717, 0, 0, 1301, 0, - 734, 669, 669, 2090, 2091, 0, 0, 1312, 0, 0, - 0, 0, 0, 0, 0, 737, 0, 0, 0, 472, - 473, 0, 0, 803, 0, 286, 290, 0, 293, 0, - 2305, 0, 2305, 0, 0, 300, 0, 0, 0, 0, - 0, 0, 330, 331, 0, 0, 0, 0, 321, 324, - 1512, 1513, 1233, 1234, 325, 326, 380, 381, 0, 906, - 931, 933, 927, 928, 929, 0, 0, 0, 0, 0, - 0, 0, 0, 577, 0, 0, 0, 0, 0, 778, - 0, 1116, 780, 0, 0, 577, 0, 0, 0, 981, - 975, 977, 1055, 161, 951, 8, 146, 143, 0, 19, - 0, 0, 19, 19, 0, 19, 335, 0, 2087, 2085, - 2086, 0, 2001, 2072, 0, 2028, 0, 2029, 2030, 2031, - 2042, 2043, 0, 0, 2024, 0, 2025, 2026, 2027, 2017, - 2078, 0, 2019, 2020, 0, 2021, 2022, 333, 451, 0, - 0, 1928, 1102, 0, 906, 883, 0, 911, 0, 0, - 577, 1520, 0, 0, 0, 0, 0, 414, 0, 425, - 419, 0, 426, 421, 422, 0, 0, 444, 446, 447, - 448, 449, 433, 434, 739, 398, 399, 400, 389, 390, - 391, 392, 393, 394, 395, 396, 397, 0, 0, 403, - 171, 0, 368, 369, 0, 0, 0, 215, 216, 217, - 218, 219, 220, 222, 206, 767, 769, 1225, 1237, 0, - 1228, 0, 225, 266, 198, 0, 0, 0, 1932, 1933, - 1934, 1935, 1936, 1941, 0, 1943, 1945, 1947, 1949, 0, - 1967, -2, -2, 1655, 1656, 1657, 1658, 1659, 1660, 1661, - 1662, 1663, 1664, 1665, 1666, 1667, 1668, 1952, 1965, 1966, - 0, 0, 0, 0, 0, 0, 1963, 1963, 1958, 0, - 1687, 1729, 1741, 1741, 1696, 1514, 1515, 1673, 0, 0, - 1722, 1726, 0, 0, 0, 0, 0, 0, 1280, 2071, - 0, 162, 1962, 1922, 1821, 1822, 1823, 1824, 1825, 1826, + 267, 0, 202, 0, 0, 0, 0, 0, 1943, 0, + 0, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + -2, 1937, 0, 0, 1680, 1681, 1682, 1683, 0, 1687, + 0, 1731, 0, 0, 0, 0, 0, 0, 0, 1997, + 2001, 0, 0, 1933, 1933, 0, 0, 1933, 1929, 0, + 0, 0, 0, 0, 0, 1933, 1866, 0, 0, 1868, + 1884, 0, 0, 1870, 1871, 0, 1874, 1875, 1933, 0, + 1933, 1879, 1933, 1933, 1933, 1860, 1861, 0, 0, 0, + 1929, 1929, 1929, 1929, 0, 0, 1929, 1929, 1929, 1929, + 1929, 1929, 1929, 1929, 1929, 1929, 1929, 1929, 1929, 1929, + 1929, 1929, 1929, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 907, 0, 915, 0, -2, + 0, 933, 935, 937, 938, 940, 941, 943, 944, 946, + 947, 892, 0, 0, 125, 0, 0, 0, 106, 0, + 0, 104, 0, 0, 0, 0, 75, 81, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 353, 0, 358, 344, 2313, 0, 343, 0, 0, + 0, 0, 0, 0, 1099, 0, 0, 1305, 1305, 1305, + 1139, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1305, 1305, 1305, 1305, 0, 1325, 0, 0, 0, 0, + 0, 852, 856, 0, 901, 0, 0, 803, 802, 74, + 78, 646, 650, 651, 652, 0, 974, 0, 0, 655, + 656, 0, 657, 0, 0, 668, 719, 719, 674, 675, + 670, 669, 725, 726, 722, 0, 722, 722, 974, 0, + 693, 694, 695, 719, 719, 701, 908, 0, 702, 703, + 722, 0, 727, 728, 974, 0, 0, 974, 974, 0, + 711, 712, 0, 715, 719, 0, 718, 0, 0, 1305, + 0, 735, 670, 670, 2100, 2101, 0, 0, 1316, 0, + 0, 0, 0, 0, 0, 0, 738, 0, 0, 0, + 472, 473, 0, 0, 804, 0, 286, 290, 0, 293, + 0, 2320, 0, 2320, 0, 0, 300, 0, 0, 0, + 0, 0, 0, 330, 331, 0, 0, 0, 0, 321, + 324, 1516, 1517, 1234, 1235, 325, 326, 380, 381, 0, + 907, 932, 934, 928, 929, 930, 0, 0, 0, 0, + 0, 0, 0, 0, 578, 0, 0, 0, 0, 0, + 779, 0, 1117, 781, 0, 0, 578, 0, 0, 0, + 982, 976, 978, 1056, 161, 952, 8, 146, 143, 0, + 19, 0, 0, 19, 19, 0, 19, 335, 0, 2097, + 2095, 2096, 0, 2007, 2082, 0, 2034, 0, 2035, 2036, + 2037, 2048, 2049, 2050, 2051, 2052, 2053, 0, 0, 2030, + 0, 2031, 2032, 2033, 2023, 2088, 0, 2025, 2026, 0, + 2027, 2028, 333, 451, 0, 0, 1934, 1103, 0, 907, + 884, 0, 912, 0, 0, 578, 1524, 0, 0, 0, + 0, 0, 414, 0, 425, 419, 0, 426, 421, 422, + 0, 0, 444, 446, 447, 448, 449, 433, 434, 740, + 398, 399, 400, 389, 390, 391, 392, 393, 394, 395, + 396, 397, 0, 0, 403, 171, 0, 368, 369, 0, + 0, 0, 215, 216, 217, 218, 219, 220, 222, 206, + 768, 770, 1226, 1238, 0, 1229, 0, 225, 266, 198, + 0, 0, 0, 1938, 1939, 1940, 1941, 1942, 1947, 0, + 1949, 1951, 1953, 1955, 0, 1973, -2, -2, 1660, 1661, + 1662, 1663, 1664, 1665, 1666, 1667, 1668, 1669, 1670, 1671, + 1672, 1673, 1958, 1971, 1972, 0, 0, 0, 0, 0, + 0, 1969, 1969, 1964, 0, 1692, 1735, 1747, 1747, 1701, + 1518, 1519, 1678, 0, 0, 1728, 1732, 0, 0, 0, + 0, 0, 0, 1283, 2081, 0, 0, 162, 1968, 1928, 1827, 1828, 1829, 1830, 1831, 1832, 1833, 1834, 1835, 1836, 1837, 1838, 1839, 1840, 1841, 1842, 1843, 1844, 1845, 1846, - 1847, 1848, 1849, 0, 0, 1931, 0, 0, 0, 0, - 1924, 1925, 0, 0, 0, 1809, 0, 0, 1815, 1816, - 1817, 0, 838, 0, 1888, 1861, 1879, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1850, - 1851, 1852, 1853, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1007, 1009, 0, 847, 849, 850, 880, 911, 887, - 0, 0, 0, 121, 126, 0, 1373, 112, 0, 0, - 0, 112, 0, 0, 0, 112, 0, 0, 0, 82, - 1209, 1316, 83, 1208, 1318, 0, 0, 0, 0, 0, - 0, 0, 362, 363, 0, 0, 357, 345, 2298, 347, - 0, 0, 0, 0, 1085, 0, 0, 0, 0, 0, - 0, 0, 1153, 1154, 0, 575, 1219, 0, 0, 0, - 1235, 1284, 1297, 0, 0, 0, 0, 0, 1379, 1139, - 1144, 1145, 1146, 1140, 1141, 1147, 1148, 829, 843, 824, - 0, 832, 0, 0, 902, 0, 0, 1024, 0, 647, - 0, 0, 653, 719, 720, 974, 657, 0, 0, 664, - 2257, 669, 973, 973, 676, 670, 677, 723, 678, 679, - 680, 721, 973, 973, 908, 718, 721, 703, 722, 721, - 1520, 707, 0, 712, 715, 716, 1520, 735, 1520, 0, - 733, 684, 685, 1381, 904, 470, 471, 476, 478, 0, - 537, 537, 537, 520, 537, 0, 0, 508, 2092, 0, - 0, 0, 0, 517, 2092, 0, 0, 2092, 2092, 2092, - 2092, 2092, 2092, 2092, 0, 0, 2092, 2092, 2092, 2092, - 2092, 2092, 2092, 2092, 2092, 2092, 2092, 0, 2092, 2092, - 2092, 2092, 2092, 1498, 2092, 0, 1313, 527, 528, 529, - 530, 535, 536, 0, 0, 481, 482, 0, 0, 0, - 0, 0, 570, 0, 0, 1152, 0, 575, 0, 0, - 1197, 0, 0, 986, 0, 987, 988, 989, 984, 1026, - 1050, 1050, 0, 1050, 1030, 1520, 0, 0, 0, 298, - 299, 287, 0, 288, 0, 0, 301, 302, 0, 304, - 305, 306, 313, 2182, 2275, 308, 310, 0, 0, 314, - 327, 328, 329, 0, 0, 319, 320, 0, 0, 383, - 384, 386, 0, 911, 1317, 1303, 79, 80, 2477, 763, - 764, 1516, 765, 766, 770, 0, 0, 773, 774, 775, - 776, 777, 1118, 0, 0, 1206, 0, 1210, 1212, 1303, - 973, 0, 982, 0, 978, 1056, 0, 1058, 0, 0, - 144, 19, 0, 137, 134, 0, 0, 0, 0, 0, - 2049, 1996, 2088, 0, 0, 0, 0, 2069, 0, 0, - 2018, 0, 0, 0, 127, 863, 911, 0, 857, 0, - 915, 916, 919, 807, 843, 809, 0, 811, 832, 0, - 0, 1519, 0, 0, 0, 0, 1581, 0, 427, 423, - 443, 0, 0, 0, 0, 209, 1222, 0, 210, 214, - 204, 0, 0, 0, 1227, 0, 1224, 1229, 0, 224, - 0, 0, 199, 200, 1364, 1373, 0, 0, 0, 1942, - 1944, 1946, 1948, 1950, 0, 1953, 1963, 1963, 1959, 0, - 1954, 0, 1956, 0, 1730, 1742, 1743, 1731, 1932, 1679, - 0, 1727, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 919, 0, 0, 0, 1797, 1799, 0, 0, 0, - 1804, 0, 1806, 1807, 1808, 1810, 0, 0, 0, 1814, - 0, 1859, 1880, 1863, 1866, 0, 1870, 0, 1872, 1874, - 1875, 1876, 0, 0, 0, 913, 913, 0, 0, 1766, - 1766, 1766, 0, 0, 0, 0, 1766, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1699, 0, 1700, 1701, 1702, 0, 1704, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1010, 857, 0, - 0, 0, 0, 0, 1371, 0, 102, 0, 107, 0, - 0, 103, 108, 0, 0, 105, 0, 0, 114, 84, - 0, 0, 1324, 1325, 0, 0, 0, 364, 352, 354, - 0, 346, 0, 1302, 0, 0, 0, 1089, 0, 0, - -2, 1118, 904, 0, 904, 1164, 2092, 0, 579, 0, - 0, 1221, 0, 1186, 0, 0, 0, -2, 0, 0, - 0, 1297, 0, 0, 0, 1383, 0, 819, 0, 823, - 840, 0, 844, 0, 0, 836, 828, 833, 0, 0, - 853, 820, 23, 905, 0, 0, 0, 789, 793, 644, - 0, 646, 652, 660, 658, 0, 662, 0, 663, 718, - 671, 672, 973, 695, 696, 0, 0, 973, 718, 718, - 706, 721, 730, 0, 731, 1520, 1383, 0, 0, 1312, - 1449, 1417, 498, 0, 1533, 1534, 538, 0, 1540, 1549, - 1301, 1619, 0, 1549, 0, 0, 1551, 1552, 0, 0, - 0, 0, 521, 522, 0, 507, 0, 0, 0, 0, - 0, 0, 506, 0, 0, 548, 0, 0, 0, 0, - 0, 2093, 2092, 2092, 0, 515, 516, 0, 519, 0, - 0, 0, 0, 0, 0, 0, 0, 2092, 2092, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1489, 0, 0, 0, 0, 0, 0, 0, 1504, 1505, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1164, - 2092, 0, 0, 0, 0, 579, 1216, 1216, 1184, 1202, - 0, 474, 475, 545, 0, 0, 0, 0, 0, 0, - 0, 1016, 0, 0, 0, 1015, 0, 0, 0, 0, - 0, 0, 0, 0, 904, 1051, 0, 1053, 1054, 1028, - -2, 0, 986, 1033, 1927, 0, 291, 292, 0, 0, - 297, 315, 317, 289, 0, 0, 0, 316, 318, 322, - 323, 382, 385, 387, 857, 76, 1304, 0, 0, 1407, - 0, 1119, 1120, 1122, 1123, 0, 2098, -2, -2, -2, + 1847, 1848, 1849, 1850, 1851, 1852, 1853, 1854, 1855, 0, + 0, 1937, 0, 0, 0, 0, 1930, 1931, 0, 0, + 0, 1815, 0, 0, 1821, 1822, 1823, 0, 839, 0, + 1894, 1867, 1885, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1856, 1857, 1858, 1859, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1008, 1010, 0, + 848, 850, 851, 881, 912, 888, 0, 0, 0, 121, + 126, 0, 1377, 112, 0, 0, 0, 112, 0, 0, + 0, 112, 0, 0, 0, 82, 1210, 1320, 83, 1209, + 1322, 0, 0, 0, 0, 0, 0, 0, 362, 363, + 0, 0, 357, 345, 2313, 347, 0, 0, 0, 0, + 1086, 0, 0, 0, 0, 0, 0, 0, 1154, 1155, + 0, 576, 1220, 0, 0, 0, 1236, 1287, 1301, 0, + 0, 0, 0, 0, 1383, 1140, 1145, 1146, 1147, 1141, + 1142, 1148, 1149, 830, 844, 825, 0, 833, 0, 0, + 903, 0, 0, 1025, 0, 648, 0, 0, 654, 720, + 721, 975, 658, 0, 0, 665, 2272, 670, 974, 974, + 677, 671, 678, 724, 679, 680, 681, 722, 974, 974, + 909, 719, 722, 704, 723, 722, 1524, 708, 0, 713, + 716, 717, 1524, 736, 1524, 0, 734, 685, 686, 1385, + 905, 470, 471, 476, 478, 0, 537, 537, 537, 520, + 537, 0, 0, 508, 2102, 0, 0, 0, 0, 517, + 2102, 0, 0, 2102, 2102, 2102, 2102, 2102, 2102, 2102, + 0, 0, 2102, 2102, 2102, 2102, 2102, 2102, 2102, 2102, + 2102, 2102, 2102, 0, 2102, 2102, 2102, 2102, 2102, 1502, + 2102, 0, 1317, 527, 528, 529, 530, 535, 536, 0, + 0, 481, 482, 0, 0, 0, 0, 0, 571, 0, + 0, 1153, 0, 576, 0, 0, 1198, 0, 0, 987, + 0, 988, 989, 990, 985, 1027, 1051, 1051, 0, 1051, + 1031, 1524, 0, 0, 0, 298, 299, 287, 0, 288, + 0, 0, 301, 302, 0, 304, 305, 306, 313, 2192, + 2290, 308, 310, 0, 0, 314, 327, 328, 329, 0, + 0, 319, 320, 0, 0, 383, 384, 386, 0, 912, + 1321, 1307, 79, 80, 2492, 764, 765, 1520, 766, 767, + 771, 0, 0, 774, 775, 776, 777, 778, 1119, 0, + 0, 1207, 0, 1211, 1213, 1307, 974, 0, 983, 0, + 979, 1057, 0, 1059, 0, 0, 144, 19, 0, 137, + 134, 0, 0, 0, 0, 0, 2059, 2002, 2098, 0, + 0, 0, 0, 2079, 0, 0, 2024, 0, 0, 0, + 127, 864, 912, 0, 858, 0, 916, 917, 920, 808, + 844, 810, 0, 812, 833, 0, 0, 1523, 0, 0, + 0, 0, 1586, 0, 427, 423, 443, 0, 0, 0, + 0, 209, 1223, 0, 210, 214, 204, 0, 0, 0, + 1228, 0, 1225, 1230, 0, 224, 0, 0, 199, 200, + 1368, 1377, 0, 0, 0, 1948, 1950, 1952, 1954, 1956, + 0, 1959, 1969, 1969, 1965, 0, 1960, 0, 1962, 0, + 1736, 1748, 1749, 1737, 1938, 1684, 0, 1733, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 920, 0, 0, + 0, 0, 1803, 1805, 0, 0, 0, 1810, 0, 1812, + 1813, 1814, 1816, 0, 0, 0, 1820, 0, 1865, 1886, + 1869, 1872, 0, 1876, 0, 1878, 1880, 1881, 1882, 0, + 0, 0, 914, 914, 0, 0, 1772, 1772, 1772, 0, + 0, 0, 0, 1772, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1705, 0, 1706, + 1707, 1708, 0, 1710, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1011, 858, 0, 0, 0, 0, + 0, 1375, 0, 102, 0, 107, 0, 0, 103, 108, + 0, 0, 105, 0, 0, 114, 84, 0, 0, 1328, + 1329, 0, 0, 0, 364, 352, 354, 0, 346, 0, + 1306, 0, 0, 0, 1090, 0, 0, -2, 1119, 905, + 0, 905, 1165, 2102, 0, 580, 0, 0, 1222, 0, + 1187, 0, 0, 0, -2, 0, 0, 0, 1301, 0, + 0, 0, 1387, 0, 820, 0, 824, 841, 0, 845, + 0, 0, 837, 829, 834, 0, 0, 854, 821, 23, + 906, 0, 0, 0, 790, 794, 645, 0, 647, 653, + 661, 659, 0, 663, 0, 664, 719, 672, 673, 974, + 696, 697, 0, 0, 974, 719, 719, 707, 722, 731, + 0, 732, 1524, 1387, 0, 0, 1316, 1453, 1421, 498, + 0, 1537, 1538, 538, 0, 1544, 1553, 1305, 1624, 0, + 1553, 0, 0, 1555, 1556, 0, 0, 0, 0, 521, + 522, 0, 507, 0, 0, 0, 0, 0, 0, 506, + 0, 0, 548, 0, 0, 0, 0, 0, 2103, 2102, + 2102, 0, 515, 516, 0, 519, 0, 0, 0, 0, + 0, 0, 0, 0, 2102, 2102, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1493, 0, 0, + 0, 0, 0, 0, 0, 1508, 1509, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1165, 2102, 0, 0, + 0, 0, 580, 1217, 1217, 1185, 1203, 0, 474, 475, + 545, 0, 0, 0, 0, 0, 0, 0, 1017, 0, + 0, 0, 1016, 0, 0, 0, 0, 0, 0, 0, + 0, 905, 1052, 0, 1054, 1055, 1029, -2, 0, 987, + 1034, 1933, 0, 291, 292, 0, 0, 297, 315, 317, + 289, 0, 0, 0, 316, 318, 322, 323, 382, 385, + 387, 858, 76, 1308, 0, 0, 1411, 0, 1120, 1121, + 1123, 1124, 0, 2108, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, 2166, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, - 2156, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, - -2, -2, -2, -2, -2, -2, -2, -2, -2, 1117, - 781, 1207, 0, 1214, 964, 976, 983, 1057, 1059, 162, - 979, 0, 147, 19, 146, 138, 139, 0, 19, 0, - 0, 0, 0, 2000, 2077, 2076, 2044, 0, 2045, 2074, - 2079, 0, 2082, 0, 455, 867, 0, 857, 859, 884, - 0, 0, 922, 920, 921, 808, 0, 0, 815, 913, - 1211, 0, 0, 0, 0, 0, 0, 0, 740, 172, - 450, 0, 0, 0, 0, 0, 768, 0, 1226, 206, - 0, 0, 226, 0, 0, 0, 1373, 1368, 1926, 1955, - 1957, 0, 1964, 1960, 1674, 1683, 1723, 0, 0, 0, - 0, 0, 1732, 2075, 2075, 1735, 2071, 2073, 2071, 1741, - 1741, 0, 1281, 0, 1282, 919, 163, 0, 0, 0, - 0, 1805, 0, 0, 0, 839, 0, 0, 0, 0, - 0, 1762, 1764, 1766, 1766, 1773, 1767, 1774, 1775, 1766, - 1766, 1766, 1766, 1780, 1766, 1766, 1766, 1766, 1766, 1766, - 1766, 1766, 1766, 1766, 1766, 1766, 0, 1760, 1703, 1705, - 0, 1708, 0, 1711, 1712, 0, 0, 0, 1985, 1986, - 848, 881, 0, 0, 894, 895, 896, 897, 898, 0, - 0, 65, 65, 1373, 0, 0, 0, 0, 0, 120, - 0, 0, 0, 0, 0, 0, 0, 1333, 1341, 0, - 356, 0, 85, 86, 88, 0, 0, 0, 0, 0, - 0, 0, 101, 1093, 0, 1087, 0, 0, 1104, 1105, - 1107, 0, 1110, 1111, 1112, 0, 0, 1526, 0, 1168, - 1165, 1166, 1167, 0, 0, 1216, 580, 581, 582, 583, - 0, 0, 0, 1220, 0, 0, 0, 1177, 0, 0, - 0, 1285, 1286, 1287, 1288, 1289, 1290, 1291, 1292, 1293, - 1294, -2, 1307, 0, 1520, 0, 0, 0, 1526, 1355, - 0, 0, 1360, 0, 0, 1526, 1526, 0, 1391, 0, - 1380, 0, 843, 845, 0, 0, 843, 0, 0, 852, - 0, 0, 1025, 851, 0, -2, 0, 0, 791, 0, - 648, 659, 665, 973, 689, 909, 910, 1520, 973, 973, - 718, 736, 732, 1391, 1382, 0, 477, 537, 0, 1437, - 0, 0, 1443, 0, 1450, 491, 0, 539, 0, 1539, - 1569, 1550, 1569, 1620, 1569, 1569, 1301, 0, 539, 0, - 0, 509, 0, 0, 0, 0, 0, 505, 542, 919, - 492, 494, 495, 496, 546, 547, 549, 0, 551, 552, - 511, 523, 524, 525, 526, 0, 0, 0, 518, 531, - 532, 533, 534, 493, 1466, 1467, 1468, 1471, 1472, 1473, - 1474, 0, 0, 1477, 1478, 1479, 1480, 1481, 1566, 1567, - 1568, 1482, 1483, 1484, 1485, 1486, 1487, 1488, 1506, 1507, - 1508, 1509, 1510, 1511, 1490, 1491, 1492, 1493, 1494, 1495, - 1496, 1497, 0, 0, 1501, 0, 0, 1087, 0, 485, - 486, 0, 488, 0, 0, 1168, 0, 0, 0, 0, - 0, 1216, 573, 0, 0, 574, 1186, 0, 1204, 0, - 1198, 1199, 0, 0, 821, 973, 375, 0, 1020, 1011, - 0, 993, 0, 995, 1017, 996, 1018, 0, 0, 1000, - 0, 1002, 0, 1004, 0, 998, 999, 1006, 997, 973, - 985, 1027, 1052, 1029, 1032, 1034, 1035, 1041, 0, 0, - 0, 0, 285, 294, 295, 296, 303, 0, 599, 309, - 925, 1517, 771, 772, 1408, 1409, 779, 0, 1124, 0, - 962, 0, 0, 142, 145, 0, 140, 0, 0, 0, - 0, 132, 130, 2070, 0, 0, 869, 186, 0, 0, - 925, 861, 0, 0, 917, 918, 0, 0, 843, 906, - 1521, 1522, 1523, 1524, 0, 1582, 415, 0, 1223, 206, - 211, 212, 213, 207, 205, 1230, 0, 1232, 0, 1366, - 0, 0, 1961, 1728, 1684, 0, 1686, 1688, 1733, 1734, - 1736, 1737, 1738, 1739, 1740, 1689, 0, 1283, 1798, 1800, - 0, 1802, 1803, 1811, 1812, 0, 1867, 1871, 0, 0, - 1858, 0, 0, 0, 0, 1771, 1772, 1776, 1777, 1778, - 1779, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, - 1790, 1791, 1792, 0, 913, 1761, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 892, 0, - 0, 0, 67, 0, 67, 1372, 1374, 113, 115, 0, - 109, 110, 111, 0, 0, 1055, 1347, 1520, 1335, 0, - 1327, 0, 1341, 0, 0, 0, 87, 0, 89, 0, - 2260, 0, 0, 0, 0, 1303, 1095, 0, 0, 1086, - 0, 1097, 1113, 1109, 0, 0, 0, 0, 1527, 1528, - 1530, 1531, 1532, 0, 1135, 0, 0, 1156, 1157, 1158, - 1182, 1170, 0, 585, 586, 0, 0, 0, 598, 594, - 595, 596, 576, 1215, 1193, 0, 0, 1193, 1180, 0, - 0, 1192, 0, 1308, 2092, 2092, 2092, 1347, 0, 0, - 0, 1451, 2092, 2092, 0, 1357, 1359, 1349, 0, 0, - 0, 1455, 1394, 0, 0, 1385, 0, 0, 841, 0, - 846, 843, 827, 837, 826, 834, 835, 854, 903, 1050, - 0, 0, 973, 790, 793, 794, 666, 704, 708, 705, - 973, 1394, 469, 1415, 0, 0, 0, 0, 0, 1447, - 0, 0, 1419, 0, 510, 540, 0, -2, 0, 1570, - 0, 1553, 1570, 0, 0, 1569, 0, 499, 539, 0, - 0, 0, 553, 0, 561, 562, 1252, 1252, 1252, 1252, - 559, 1615, 0, 560, 0, 544, 0, 550, 1469, 1470, - 0, 1475, 1476, 0, 1500, 0, 0, 480, 483, 0, - 1091, 1092, -2, 0, 0, 0, 565, 0, 0, 0, - 566, 567, 572, 1217, 1218, 1177, 0, 1193, 0, 1203, - 0, 1200, 1201, 913, 0, 0, 0, 990, 1021, 0, - 0, 991, 0, 992, 994, 1019, 0, 1013, 1001, 1003, - 1005, 373, 1036, 0, 0, 1038, 1039, 1040, 1031, 311, - 879, 0, 1121, 0, 0, 947, 0, 0, 980, 0, - 19, 0, 0, 135, 2080, 2083, 871, 0, 868, 187, - 0, 0, 0, 882, 863, 0, 860, 0, 923, 924, - 810, 843, 814, 813, 816, 1525, 208, 203, 1231, 1376, - 0, 1367, 0, 1639, 1698, 0, 1813, 0, 0, 1766, - 1763, 1766, 1765, 1766, 1757, 0, 1706, 0, 1709, 0, - 1713, 1714, 0, 1716, 1717, 1718, 0, 1720, 1721, 0, - 890, 0, 63, 0, 66, 64, 0, 0, 0, 119, - 1322, 0, 1347, 1326, 0, 0, 0, 1328, 0, 0, - 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, - 99, 0, 0, 1094, 0, 1088, 0, 0, 1106, 1108, - 0, 1142, 1455, 0, 1142, 1169, 1155, 0, 1136, 0, - 0, 587, 588, 0, 591, 597, 1171, 0, 0, 1174, - 1175, 1173, 1176, 0, 0, 1190, 0, 0, 0, 0, - 1295, 0, 1298, 1314, 0, 0, 0, -2, 1347, 0, - 0, 0, -2, 1354, 0, 1400, 0, 1392, 0, 1384, - 0, 1387, 0, 831, 842, 825, 973, 973, -2, 787, - 792, 0, 709, 1400, 1417, 0, 1438, 0, 0, 0, - 0, 0, 0, 0, 1418, 0, 1431, 541, 1571, -2, - 1585, 1587, 0, 1313, 1590, 1591, 0, 0, 0, 0, - 0, 0, 1646, 1599, 0, 0, 0, 1604, 1605, 1606, - 0, 0, 1609, 0, 0, 0, 1979, 1980, 0, 1618, - 0, 0, 0, 0, 0, 0, 0, 1547, 500, 501, - 0, 503, 504, 1252, 0, 555, 556, 557, 558, 1616, - 543, 497, 2092, 513, 1499, 1502, 1503, 484, 487, 0, - 0, 571, 568, 569, 1180, 1185, 1196, 1205, 822, 906, - 973, 376, 377, 1022, 0, 1012, 1014, 1045, 1042, 0, - 0, 926, 1125, 1213, 963, 971, 2494, 2496, 2493, 136, - 141, 0, 0, 873, 0, 870, 0, 864, 866, 197, - 867, 862, 912, 812, 157, 189, 0, 0, 1685, 0, - 0, 0, 1801, 1856, 1857, 1769, 1770, 1793, 0, 1758, - 0, 1752, 1753, 1754, 1759, 0, 0, 0, 0, 893, - 888, 68, 117, 116, 0, 0, 1323, 0, 0, 0, - 1339, 1340, 0, 1342, 1343, 1344, 0, 0, 0, 0, - 72, 0, 0, 0, 1303, 0, 1303, 0, 0, 0, - 0, 1096, 1090, 1100, 1114, 0, 1127, 1134, 1149, 1319, - 1529, 1133, 0, 0, 0, 584, 589, 0, 592, 593, - 1194, 1193, 0, 1178, 1179, 0, 1188, 0, 0, 1309, - 1310, 1311, 1182, 1452, 1453, 1454, 1410, 1356, 0, -2, - 1463, 0, 1361, 1350, 0, 1352, 1376, 1410, 0, 1388, - 0, 1395, 0, 1393, 1386, 830, 913, 788, 1397, 479, - 1449, 1439, 0, 1441, 0, 0, 0, 0, 1420, -2, - 0, 1586, 1588, 1589, 1592, 1593, 1594, 1651, 1652, 1653, - 0, 0, 1597, 1648, 1649, 1650, 1598, 0, 0, 0, - 1603, 0, 0, 0, 0, 1977, 1978, 1644, 0, 0, - 1554, 1556, 1557, 1558, 1559, 1560, 1561, 1562, 1563, 1564, - 1565, 1555, 0, 0, 0, 1546, 1548, 502, 554, 0, - 1253, 2092, 2092, 0, 0, 0, 1259, 1260, 2092, 2092, - 2092, 2092, 2092, 2092, 0, 0, 0, 2092, 2092, 2092, - 2092, 1274, 1275, 0, 2092, 2092, 0, 2092, 0, 0, - 1195, 372, 374, 0, 0, 1046, 1048, 1043, 1044, 965, - 0, 0, 0, 0, 131, 133, 148, 0, 872, 188, - 0, 869, 159, 0, 180, 0, 1377, 0, 1697, 0, - 0, 0, 1768, 1755, 0, 0, 0, 0, 0, 1981, - 1982, 1983, 0, 1707, 1710, 1715, 1719, 0, 1348, 1336, - 1337, 1338, 1334, 0, 0, 1345, 1346, 0, 70, 0, - 93, 0, 0, 94, 1303, 95, 1303, 0, 0, 1084, - 0, 0, 1150, 1151, 1159, 1160, 0, 1162, 1163, 1183, - 590, 1172, 1181, 1187, 1190, 0, 1252, 1296, 1412, 0, - 1358, 1312, 1465, 2092, 1182, 1363, 1412, 0, 1457, 2092, - 2092, 1378, 0, 1390, 0, 1402, 0, 1396, 906, 468, - 0, 1399, 1435, 1440, 1442, 1444, 0, 1448, 1446, 1421, - -2, 0, 1429, 0, 0, 1595, 1596, 0, 0, 1877, - 2092, 0, 0, 0, 1634, 0, 1252, 1252, 1252, 1252, - 0, 563, 564, 0, 0, 1256, 1257, 0, 0, 0, - 0, 0, 0, 0, 0, 1268, 1269, 0, 0, 0, - 0, 0, 0, 0, 512, 0, 0, 490, 1023, 1037, - 0, 972, 0, 0, 0, 0, 0, 871, 149, 0, - 158, 177, 0, 190, 191, 0, 0, 0, 0, 1369, - 0, 1642, 1643, 0, 1744, 0, 0, 0, 1748, 1749, - 1750, 1751, 118, 1341, 1341, 1303, 72, 0, 92, 0, - 96, 97, 0, 1303, 0, 1126, 0, 1161, 1189, 1191, - 1251, 1351, 0, 1449, 1464, 0, 1362, 1353, 1456, 0, - 0, 0, 1389, 1401, 0, 1404, 786, 1398, 1416, 0, - 1445, 1422, 1430, 0, 1425, 0, 0, 0, 1647, 0, - 1602, 0, 1608, 0, 1612, 1622, 1635, 0, 0, 1535, - 0, 1537, 0, 1541, 0, 1543, 0, 0, 1254, 1255, - 1258, 1261, 1262, 1263, 1264, 1265, 1266, 0, 1270, 1271, - 1272, 1273, 1276, 1277, 1278, 1279, 514, 489, 1047, 1049, - 0, 1927, 967, 968, 0, 875, 865, 873, 160, 164, - 0, 186, 183, 0, 192, 0, 0, 0, 0, 1365, - 0, 1640, 0, 1745, 1746, 1747, 1329, 1341, 1330, 1341, - 69, 71, 73, 91, 1303, 98, 0, 1128, 1129, 1143, - 0, 1437, 1469, 1458, 1459, 1460, 1403, 1436, 1424, 0, - -2, 1432, 0, 0, 1929, 1939, 1940, 1600, 1607, 0, - 1611, 1613, 1614, 1621, 1623, 1624, 0, 1636, 1637, 1638, - 1645, 1252, 1252, 1252, 1252, 1545, 1267, 966, 0, 0, - 874, 0, 858, 151, 0, 0, 181, 182, 184, 0, - 193, 0, 195, 196, 0, 0, 1756, 1331, 1332, 100, - 1130, 1413, 0, 1415, 1426, -2, 0, 1434, 0, 1601, - 1612, 1625, 0, 1626, 0, 0, 0, 1536, 1538, 1542, - 1544, 1927, 969, 876, 1375, 0, 165, 0, 167, 169, - 170, 1572, 178, 179, 185, 194, 0, 0, 1115, 1131, - 0, 0, 1417, 1433, 1930, 1610, 1627, 1629, 1630, 0, - 0, 1628, 0, 152, 153, 0, 166, 0, 0, 1370, - 1641, 1132, 1414, 1411, 1631, 1633, 1632, 970, 0, 0, - 168, 1573, 154, 155, 156, 0, 1574, + 1118, 782, 1208, 0, 1215, 965, 977, 984, 1058, 1060, + 162, 980, 0, 147, 19, 146, 138, 139, 0, 19, + 0, 0, 0, 0, 2006, 2087, 2086, 2054, 0, 2055, + 2084, 2089, 0, 2092, 0, 455, 868, 0, 858, 860, + 885, 0, 0, 923, 921, 922, 809, 0, 0, 816, + 914, 1212, 0, 0, 0, 0, 0, 0, 0, 741, + 172, 450, 0, 0, 0, 0, 0, 769, 0, 1227, + 206, 0, 0, 226, 0, 0, 0, 1377, 1372, 1932, + 1961, 1963, 0, 1970, 1966, 1679, 1688, 1729, 0, 0, + 0, 0, 0, 1738, 2085, 2085, 1741, 2081, 2083, 2081, + 1747, 1747, 0, 1284, 0, 1285, 920, 0, 163, 0, + 0, 0, 0, 1811, 0, 0, 0, 840, 0, 0, + 0, 0, 0, 1768, 1770, 1772, 1772, 1779, 1773, 1780, + 1781, 1772, 1772, 1772, 1772, 1786, 1772, 1772, 1772, 1772, + 1772, 1772, 1772, 1772, 1772, 1772, 1772, 1772, 0, 1766, + 1709, 1711, 0, 1714, 0, 1717, 1718, 0, 0, 0, + 1991, 1992, 849, 882, 0, 0, 895, 896, 897, 898, + 899, 0, 0, 65, 65, 1377, 0, 0, 0, 0, + 0, 120, 0, 0, 0, 0, 0, 0, 0, 1337, + 1345, 0, 356, 0, 85, 86, 88, 0, 0, 0, + 0, 0, 0, 0, 101, 1094, 0, 1088, 0, 0, + 1105, 1106, 1108, 0, 1111, 1112, 1113, 0, 0, 1530, + 0, 1169, 1166, 1167, 1168, 0, 0, 1217, 581, 582, + 583, 584, 0, 0, 0, 1221, 0, 0, 0, 1178, + 0, 0, 0, 1288, 1289, 1290, 1291, 1292, 1293, 1294, + 1295, 1296, 1297, 1298, -2, 1311, 0, 1524, 0, 0, + 0, 1530, 1359, 0, 0, 1364, 0, 0, 1530, 1530, + 0, 1395, 0, 1384, 0, 844, 846, 0, 0, 844, + 0, 0, 853, 0, 0, 1026, 852, 0, -2, 0, + 0, 792, 0, 649, 660, 666, 974, 690, 910, 911, + 1524, 974, 974, 719, 737, 733, 1395, 1386, 0, 477, + 537, 0, 1441, 0, 0, 1447, 0, 1454, 491, 0, + 539, 0, 1543, 1574, 1554, 1574, 1625, 1574, 1574, 1305, + 0, 539, 0, 0, 509, 0, 0, 0, 0, 0, + 505, 542, 920, 492, 494, 495, 496, 546, 547, 549, + 0, 551, 552, 511, 523, 524, 525, 526, 0, 0, + 0, 518, 531, 532, 533, 534, 493, 1470, 1471, 1472, + 1475, 1476, 1477, 1478, 0, 0, 1481, 1482, 1483, 1484, + 1485, 1571, 1572, 1573, 1486, 1487, 1488, 1489, 1490, 1491, + 1492, 1510, 1511, 1512, 1513, 1514, 1515, 1494, 1495, 1496, + 1497, 1498, 1499, 1500, 1501, 0, 0, 1505, 0, 0, + 1088, 0, 485, 486, 0, 488, 0, 0, 1169, 0, + 0, 0, 0, 0, 1217, 574, 0, 0, 575, 1187, + 0, 1205, 0, 1199, 1200, 0, 0, 822, 974, 375, + 0, 1021, 1012, 0, 994, 0, 996, 1018, 997, 1019, + 0, 0, 1001, 0, 1003, 0, 1005, 0, 999, 1000, + 1007, 998, 974, 986, 1028, 1053, 1030, 1033, 1035, 1036, + 1042, 0, 0, 0, 0, 285, 294, 295, 296, 303, + 0, 600, 309, 926, 1521, 772, 773, 1412, 1413, 780, + 0, 1125, 0, 963, 0, 0, 142, 145, 0, 140, + 0, 0, 0, 0, 132, 130, 2080, 0, 0, 870, + 186, 0, 0, 926, 862, 0, 0, 918, 919, 0, + 0, 844, 907, 1525, 1526, 1527, 1528, 0, 1587, 415, + 0, 1224, 206, 211, 212, 213, 207, 205, 1231, 0, + 1233, 0, 1370, 0, 0, 1967, 1734, 1689, 0, 1691, + 1693, 1739, 1740, 1742, 1743, 1744, 1745, 1746, 1694, 0, + 1286, 0, 1804, 1806, 0, 1808, 1809, 1817, 1818, 0, + 1873, 1877, 0, 0, 1864, 0, 0, 0, 0, 1777, + 1778, 1782, 1783, 1784, 1785, 1787, 1788, 1789, 1790, 1791, + 1792, 1793, 1794, 1795, 1796, 1797, 1798, 0, 914, 1767, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 893, 0, 0, 0, 67, 0, 67, 1376, + 1378, 113, 115, 0, 109, 110, 111, 0, 0, 1056, + 1351, 1524, 1339, 0, 1331, 0, 1345, 0, 0, 0, + 87, 0, 89, 0, 2275, 0, 0, 0, 0, 1307, + 1096, 0, 0, 1087, 0, 1098, 1114, 1110, 0, 0, + 0, 0, 1531, 1532, 1534, 1535, 1536, 0, 1136, 0, + 0, 1157, 1158, 1159, 1183, 1171, 0, 586, 587, 0, + 0, 0, 599, 595, 596, 597, 577, 1216, 1194, 0, + 0, 1194, 1181, 0, 0, 1193, 0, 1312, 2102, 2102, + 2102, 1351, 0, 0, 0, 1455, 2102, 2102, 0, 1361, + 1363, 1353, 0, 0, 0, 1459, 1398, 0, 0, 1389, + 0, 0, 842, 0, 847, 844, 828, 838, 827, 835, + 836, 855, 904, 1051, 0, 0, 974, 791, 794, 795, + 667, 705, 709, 706, 974, 1398, 469, 1419, 0, 0, + 0, 0, 0, 1451, 0, 0, 1423, 0, 510, 540, + 0, -2, 0, 1575, 0, 1557, 1575, 0, 0, 1574, + 0, 499, 539, 0, 0, 0, 553, 0, 562, 563, + 1253, 1253, 1253, 1253, 1253, 560, 1620, 0, 561, 0, + 544, 0, 550, 1473, 1474, 0, 1479, 1480, 0, 1504, + 0, 0, 480, 483, 0, 1092, 1093, -2, 0, 0, + 0, 566, 0, 0, 0, 567, 568, 573, 1218, 1219, + 1178, 0, 1194, 0, 1204, 0, 1201, 1202, 914, 0, + 0, 0, 991, 1022, 0, 0, 992, 0, 993, 995, + 1020, 0, 1014, 1002, 1004, 1006, 373, 1037, 0, 0, + 1039, 1040, 1041, 1032, 311, 880, 0, 1122, 0, 0, + 948, 0, 0, 981, 0, 19, 0, 0, 135, 2090, + 2093, 872, 0, 869, 187, 0, 0, 0, 883, 864, + 0, 861, 0, 924, 925, 811, 844, 815, 814, 817, + 1529, 208, 203, 1232, 1380, 0, 1371, 0, 1644, 1704, + 0, 0, 1819, 0, 0, 1772, 1769, 1772, 1771, 1772, + 1763, 0, 1712, 0, 1715, 0, 1719, 1720, 0, 1722, + 1723, 1724, 0, 1726, 1727, 0, 891, 0, 63, 0, + 66, 64, 0, 0, 0, 119, 1326, 0, 1351, 1330, + 0, 0, 0, 1332, 0, 0, 0, 0, 0, 90, + 0, 0, 0, 0, 0, 0, 99, 0, 0, 1095, + 0, 1089, 0, 0, 1107, 1109, 0, 1143, 1459, 0, + 1143, 1170, 1156, 0, 1137, 0, 0, 588, 589, 0, + 592, 598, 1172, 0, 0, 1175, 1176, 1174, 1177, 0, + 0, 1191, 0, 0, 0, 0, 1299, 0, 1302, 1318, + 0, 0, 0, -2, 1351, 0, 0, 0, -2, 1358, + 0, 1404, 0, 1396, 0, 1388, 0, 1391, 0, 832, + 843, 826, 974, 974, -2, 788, 793, 0, 710, 1404, + 1421, 0, 1442, 0, 0, 0, 0, 0, 0, 0, + 1422, 0, 1435, 541, 1576, -2, 1590, 1592, 0, 1317, + 1595, 1596, 0, 0, 0, 0, 0, 0, 1651, 1604, + 0, 0, 0, 1609, 1610, 1611, 0, 0, 1614, 0, + 0, 0, 1985, 1986, 0, 1623, 0, 0, 0, 0, + 0, 0, 0, 1551, 500, 501, 0, 503, 504, 1253, + 0, 555, 556, 557, 558, 559, 1621, 543, 497, 2102, + 513, 1503, 1506, 1507, 484, 487, 0, 0, 572, 569, + 570, 1181, 1186, 1197, 1206, 823, 907, 974, 376, 377, + 1023, 0, 1013, 1015, 1046, 1043, 0, 0, 927, 1126, + 1214, 964, 972, 2509, 2511, 2508, 136, 141, 0, 0, + 874, 0, 871, 0, 865, 867, 197, 868, 863, 913, + 813, 157, 189, 0, 0, 1690, 0, 0, 0, 1703, + 1807, 1862, 1863, 1775, 1776, 1799, 0, 1764, 0, 1758, + 1759, 1760, 1765, 0, 0, 0, 0, 894, 889, 68, + 117, 116, 0, 0, 1327, 0, 0, 0, 1343, 1344, + 0, 1346, 1347, 1348, 0, 0, 0, 0, 72, 0, + 0, 0, 1307, 0, 1307, 0, 0, 0, 0, 1097, + 1091, 1101, 1115, 0, 1128, 1135, 1150, 1323, 1533, 1134, + 0, 0, 0, 585, 590, 0, 593, 594, 1195, 1194, + 0, 1179, 1180, 0, 1189, 0, 0, 1313, 1314, 1315, + 1183, 1456, 1457, 1458, 1414, 1360, 0, -2, 1467, 0, + 1365, 1354, 0, 1356, 1380, 1414, 0, 1392, 0, 1399, + 0, 1397, 1390, 831, 914, 789, 1401, 479, 1453, 1443, + 0, 1445, 0, 0, 0, 0, 1424, -2, 0, 1591, + 1593, 1594, 1597, 1598, 1599, 1656, 1657, 1658, 0, 0, + 1602, 1653, 1654, 1655, 1603, 0, 0, 0, 1608, 0, + 0, 0, 0, 1983, 1984, 1649, 0, 0, 1558, 1560, + 1561, 1562, 1563, 1564, 1565, 1566, 1567, 1568, 1569, 1570, + 1559, 0, 0, 0, 1550, 1552, 502, 554, 0, 1254, + 2102, 2102, 0, 0, 0, 1260, 1261, 2102, 2102, 2102, + 2102, 2102, 2102, 0, 0, 0, 2102, 2102, 2102, 2102, + 1275, 1276, 1277, 0, 2102, 2102, 2102, 0, 2102, 0, + 0, 1196, 372, 374, 0, 0, 1047, 1049, 1044, 1045, + 966, 0, 0, 0, 0, 131, 133, 148, 0, 873, + 188, 0, 870, 159, 0, 180, 0, 1381, 0, 1702, + 0, 0, 0, 1774, 1761, 0, 0, 0, 0, 0, + 1987, 1988, 1989, 0, 1713, 1716, 1721, 1725, 0, 1352, + 1340, 1341, 1342, 1338, 0, 0, 1349, 1350, 0, 70, + 0, 93, 0, 0, 94, 1307, 95, 1307, 0, 0, + 1085, 0, 0, 1151, 1152, 1160, 1161, 0, 1163, 1164, + 1184, 591, 1173, 1182, 1188, 1191, 0, 1253, 1300, 1416, + 0, 1362, 1316, 1469, 2102, 1183, 1367, 1416, 0, 1461, + 2102, 2102, 1382, 0, 1394, 0, 1406, 0, 1400, 907, + 468, 0, 1403, 1439, 1444, 1446, 1448, 0, 1452, 1450, + 1425, -2, 0, 1433, 0, 0, 1600, 1601, 0, 0, + 1883, 2102, 0, 0, 0, 1639, 0, 1253, 1253, 1253, + 1253, 0, 564, 565, 0, 0, 1257, 1258, 0, 0, + 0, 0, 0, 0, 0, 0, 1269, 1270, 0, 0, + 0, 0, 0, 0, 0, 0, 512, 0, 0, 490, + 1024, 1038, 0, 973, 0, 0, 0, 0, 0, 872, + 149, 0, 158, 177, 0, 190, 191, 0, 0, 0, + 0, 1373, 0, 1647, 1648, 0, 1750, 0, 0, 0, + 1754, 1755, 1756, 1757, 118, 1345, 1345, 1307, 72, 0, + 92, 0, 96, 97, 0, 1307, 0, 1127, 0, 1162, + 1190, 1192, 1252, 1355, 0, 1453, 1468, 0, 1366, 1357, + 1460, 0, 0, 0, 1393, 1405, 0, 1408, 787, 1402, + 1420, 0, 1449, 1426, 1434, 0, 1429, 0, 0, 0, + 1652, 0, 1607, 0, 1613, 0, 1617, 1627, 1640, 0, + 0, 1539, 0, 1541, 0, 1545, 0, 1547, 0, 0, + 1255, 1256, 1259, 1262, 1263, 1264, 1265, 1266, 1267, 0, + 1271, 1272, 1273, 1274, 1278, 1279, 1280, 1281, 1282, 514, + 489, 1048, 1050, 0, 1933, 968, 969, 0, 876, 866, + 874, 160, 164, 0, 186, 183, 0, 192, 0, 0, + 0, 0, 1369, 0, 1645, 0, 1751, 1752, 1753, 1333, + 1345, 1334, 1345, 69, 71, 73, 91, 1307, 98, 0, + 1129, 1130, 1144, 0, 1441, 1473, 1462, 1463, 1464, 1407, + 1440, 1428, 0, -2, 1436, 0, 0, 1935, 1945, 1946, + 1605, 1612, 0, 1616, 1618, 1619, 1626, 1628, 1629, 0, + 1641, 1642, 1643, 1650, 1253, 1253, 1253, 1253, 1549, 1268, + 967, 0, 0, 875, 0, 859, 151, 0, 0, 181, + 182, 184, 0, 193, 0, 195, 196, 0, 0, 1762, + 1335, 1336, 100, 1131, 1417, 0, 1419, 1430, -2, 0, + 1438, 0, 1606, 1617, 1630, 0, 1631, 0, 0, 0, + 1540, 1542, 1546, 1548, 1933, 970, 877, 1379, 0, 165, + 0, 167, 169, 170, 1577, 178, 179, 185, 194, 0, + 0, 1116, 1132, 0, 0, 1421, 1437, 1936, 1615, 1632, + 1634, 1635, 0, 0, 1633, 0, 152, 153, 0, 166, + 0, 0, 1374, 1646, 1133, 1418, 1415, 1636, 1638, 1637, + 971, 0, 0, 168, 1578, 154, 155, 156, 0, 1579, } var yyTok1 = [...]int{ @@ -11976,14 +12093,14 @@ var yyTok1 = [...]int{ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 121, 3, 3, 3, 154, 144, 3, 88, 89, 151, 149, 174, 150, 173, 152, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 744, 741, - 131, 130, 132, 3, 745, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 749, 746, + 131, 130, 132, 3, 750, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 156, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 742, 143, 743, 157, + 3, 3, 3, 747, 143, 748, 157, } var yyTok2 = [...]int{ @@ -12109,7 +12226,8 @@ var yyTok3 = [...]int{ 58050, 725, 58051, 726, 58052, 727, 58053, 728, 58054, 729, 58055, 730, 58056, 731, 58057, 732, 58058, 733, 58059, 734, 58060, 735, 58061, 736, 58062, 737, 58063, 738, 58064, 739, - 58065, 740, 0, + 58065, 740, 58066, 741, 58067, 742, 58068, 743, 58069, 744, + 58070, 745, 0, } var yyErrorMessages = [...]struct { @@ -16916,61 +17034,78 @@ yydefault: } yyVAL.union = yyLOCAL case 559: - yyDollar = yyS[yypt-3 : yypt+1] + yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.AlterTableOption //line mysql_sql.y:4183 + { + var io *tree.IndexOption = nil + if yyDollar[4].indexOptionUnion() == nil { + io = tree.NewIndexOption() + io.IType = tree.INDEX_TYPE_BM25 + } else { + io = yyDollar[4].indexOptionUnion() + io.IType = tree.INDEX_TYPE_BM25 + } + var name = tree.Identifier(yyDollar[2].cstrUnion().Compare()) + yyLOCAL = tree.NewAlterOptionAlterReIndex(name, io) + } + yyVAL.union = yyLOCAL + case 560: + yyDollar = yyS[yypt-3 : yypt+1] + var yyLOCAL tree.AlterTableOption +//line mysql_sql.y:4196 { var checkType = yyDollar[1].str var enforce = yyDollar[3].boolValUnion() yyLOCAL = tree.NewAlterOptionAlterCheck(checkType, enforce) } yyVAL.union = yyLOCAL - case 560: + case 561: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.AlterTableOption -//line mysql_sql.y:4189 +//line mysql_sql.y:4202 { var checkType = yyDollar[1].str var enforce = yyDollar[3].boolValUnion() yyLOCAL = tree.NewAlterOptionAlterCheck(checkType, enforce) } yyVAL.union = yyLOCAL - case 561: + case 562: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.VisibleType -//line mysql_sql.y:4197 +//line mysql_sql.y:4210 { yyLOCAL = tree.VISIBLE_TYPE_VISIBLE } yyVAL.union = yyLOCAL - case 562: + case 563: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.VisibleType -//line mysql_sql.y:4201 +//line mysql_sql.y:4214 { yyLOCAL = tree.VISIBLE_TYPE_INVISIBLE } yyVAL.union = yyLOCAL - case 563: + case 564: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:4207 +//line mysql_sql.y:4220 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 564: + case 565: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:4211 +//line mysql_sql.y:4224 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 565: + case 566: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4217 +//line mysql_sql.y:4230 { var ifExists = yyDollar[3].boolValUnion() var name = yyDollar[4].exprUnion() @@ -16987,10 +17122,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 566: + case 567: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4235 +//line mysql_sql.y:4248 { var accountName = "" var dbName = yyDollar[3].str @@ -17006,10 +17141,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 567: + case 568: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4250 +//line mysql_sql.y:4263 { var accountName = "" var dbName = yyDollar[3].str @@ -17025,10 +17160,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 568: + case 569: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4265 +//line mysql_sql.y:4278 { var accountName = yyDollar[4].str var dbName = "" @@ -17044,10 +17179,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 569: + case 570: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4280 +//line mysql_sql.y:4293 { assignments := []*tree.VarAssignmentExpr{ { @@ -17060,20 +17195,20 @@ yydefault: yyLOCAL = &tree.SetVar{Assignments: assignments} } yyVAL.union = yyLOCAL - case 570: + case 571: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.AlterAccountAuthOption -//line mysql_sql.y:4293 +//line mysql_sql.y:4306 { yyLOCAL = tree.AlterAccountAuthOption{ Exist: false, } } yyVAL.union = yyLOCAL - case 571: + case 572: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.AlterAccountAuthOption -//line mysql_sql.y:4299 +//line mysql_sql.y:4312 { yyLOCAL = tree.AlterAccountAuthOption{ Exist: true, @@ -17083,10 +17218,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 572: + case 573: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4310 +//line mysql_sql.y:4323 { // Create temporary variables with meaningful names ifExists := yyDollar[3].boolValUnion() @@ -17099,10 +17234,10 @@ yydefault: yyLOCAL = tree.NewAlterUser(ifExists, users, role, miscOpt, commentOrAttribute) } yyVAL.union = yyLOCAL - case 573: + case 574: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4322 +//line mysql_sql.y:4335 { ifExists := yyDollar[3].boolValUnion() var Username = yyDollar[4].usernameRecordUnion().Username @@ -17114,10 +17249,10 @@ yydefault: yyLOCAL = tree.NewAlterUser(ifExists, users, nil, miscOpt, commentOrAttribute) } yyVAL.union = yyLOCAL - case 574: + case 575: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4333 +//line mysql_sql.y:4346 { ifExists := yyDollar[3].boolValUnion() var Username = yyDollar[4].usernameRecordUnion().Username @@ -17129,18 +17264,18 @@ yydefault: yyLOCAL = tree.NewAlterUser(ifExists, users, nil, miscOpt, commentOrAttribute) } yyVAL.union = yyLOCAL - case 575: + case 576: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.Role -//line mysql_sql.y:4345 +//line mysql_sql.y:4358 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 576: + case 577: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.Role -//line mysql_sql.y:4349 +//line mysql_sql.y:4362 { var UserName = yyDollar[3].str yyLOCAL = tree.NewRole( @@ -17148,66 +17283,66 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 577: + case 578: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:4357 +//line mysql_sql.y:4370 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 578: + case 579: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:4361 +//line mysql_sql.y:4374 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 579: + case 580: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.UserMiscOption -//line mysql_sql.y:4366 +//line mysql_sql.y:4379 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 580: + case 581: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.UserMiscOption -//line mysql_sql.y:4370 +//line mysql_sql.y:4383 { yyLOCAL = yyDollar[1].userMiscOptionUnion() } yyVAL.union = yyLOCAL - case 581: + case 582: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.UserMiscOption -//line mysql_sql.y:4386 +//line mysql_sql.y:4399 { yyLOCAL = tree.NewUserMiscOptionAccountUnlock() } yyVAL.union = yyLOCAL - case 582: + case 583: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.UserMiscOption -//line mysql_sql.y:4390 +//line mysql_sql.y:4403 { yyLOCAL = tree.NewUserMiscOptionAccountLock() } yyVAL.union = yyLOCAL - case 583: + case 584: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.UserMiscOption -//line mysql_sql.y:4394 +//line mysql_sql.y:4407 { yyLOCAL = tree.NewUserMiscOptionPasswordExpireNone() } yyVAL.union = yyLOCAL - case 584: + case 585: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.UserMiscOption -//line mysql_sql.y:4398 +//line mysql_sql.y:4411 { var Value = yyDollar[3].item.(int64) yyLOCAL = tree.NewUserMiscOptionPasswordExpireInterval( @@ -17215,34 +17350,34 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 585: + case 586: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.UserMiscOption -//line mysql_sql.y:4405 +//line mysql_sql.y:4418 { yyLOCAL = tree.NewUserMiscOptionPasswordExpireNever() } yyVAL.union = yyLOCAL - case 586: + case 587: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.UserMiscOption -//line mysql_sql.y:4409 +//line mysql_sql.y:4422 { yyLOCAL = tree.NewUserMiscOptionPasswordExpireDefault() } yyVAL.union = yyLOCAL - case 587: + case 588: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.UserMiscOption -//line mysql_sql.y:4413 +//line mysql_sql.y:4426 { yyLOCAL = tree.NewUserMiscOptionPasswordHistoryDefault() } yyVAL.union = yyLOCAL - case 588: + case 589: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.UserMiscOption -//line mysql_sql.y:4417 +//line mysql_sql.y:4430 { var Value = yyDollar[3].item.(int64) yyLOCAL = tree.NewUserMiscOptionPasswordHistoryCount( @@ -17250,18 +17385,18 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 589: + case 590: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.UserMiscOption -//line mysql_sql.y:4424 +//line mysql_sql.y:4437 { yyLOCAL = tree.NewUserMiscOptionPasswordReuseIntervalDefault() } yyVAL.union = yyLOCAL - case 590: + case 591: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.UserMiscOption -//line mysql_sql.y:4428 +//line mysql_sql.y:4441 { var Value = yyDollar[4].item.(int64) yyLOCAL = tree.NewUserMiscOptionPasswordReuseIntervalCount( @@ -17269,34 +17404,34 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 591: + case 592: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.UserMiscOption -//line mysql_sql.y:4435 +//line mysql_sql.y:4448 { yyLOCAL = tree.NewUserMiscOptionPasswordRequireCurrentNone() } yyVAL.union = yyLOCAL - case 592: + case 593: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.UserMiscOption -//line mysql_sql.y:4439 +//line mysql_sql.y:4452 { yyLOCAL = tree.NewUserMiscOptionPasswordRequireCurrentDefault() } yyVAL.union = yyLOCAL - case 593: + case 594: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.UserMiscOption -//line mysql_sql.y:4443 +//line mysql_sql.y:4456 { yyLOCAL = tree.NewUserMiscOptionPasswordRequireCurrentOptional() } yyVAL.union = yyLOCAL - case 594: + case 595: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.UserMiscOption -//line mysql_sql.y:4447 +//line mysql_sql.y:4460 { var Value = yyDollar[2].item.(int64) yyLOCAL = tree.NewUserMiscOptionFailedLoginAttempts( @@ -17304,10 +17439,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 595: + case 596: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.UserMiscOption -//line mysql_sql.y:4454 +//line mysql_sql.y:4467 { var Value = yyDollar[2].item.(int64) yyLOCAL = tree.NewUserMiscOptionPasswordLockTimeCount( @@ -17315,38 +17450,38 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 596: + case 597: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.UserMiscOption -//line mysql_sql.y:4461 +//line mysql_sql.y:4474 { yyLOCAL = tree.NewUserMiscOptionPasswordLockTimeUnbounded() } yyVAL.union = yyLOCAL - case 597: + case 598: yyDollar = yyS[yypt-3 : yypt+1] -//line mysql_sql.y:4467 +//line mysql_sql.y:4480 { yyVAL.item = nil } - case 598: + case 599: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:4472 +//line mysql_sql.y:4485 { yyVAL.item = nil } - case 643: + case 644: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4526 +//line mysql_sql.y:4539 { yyLOCAL = &tree.ShowSQLTasks{} } yyVAL.union = yyLOCAL - case 644: + case 645: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4532 +//line mysql_sql.y:4545 { stmt := &tree.ShowSQLTaskRuns{} if yyDollar[4].str != "" { @@ -17360,72 +17495,72 @@ yydefault: yyLOCAL = stmt } yyVAL.union = yyLOCAL - case 645: + case 646: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:4546 +//line mysql_sql.y:4559 { yyVAL.str = "" } - case 646: + case 647: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:4550 +//line mysql_sql.y:4563 { yyVAL.str = yyDollar[2].cstrUnion().Compare() } - case 647: + case 648: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL int64 -//line mysql_sql.y:4555 +//line mysql_sql.y:4568 { yyLOCAL = -1 } yyVAL.union = yyLOCAL - case 648: + case 649: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL int64 -//line mysql_sql.y:4559 +//line mysql_sql.y:4572 { yyLOCAL = sqlTaskInt64(yyDollar[2].item) } yyVAL.union = yyLOCAL - case 649: + case 650: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4565 +//line mysql_sql.y:4578 { yyLOCAL = &tree.ShowLogserviceReplicas{} } yyVAL.union = yyLOCAL - case 650: + case 651: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4571 +//line mysql_sql.y:4584 { yyLOCAL = &tree.ShowLogserviceStores{} } yyVAL.union = yyLOCAL - case 651: + case 652: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4577 +//line mysql_sql.y:4590 { yyLOCAL = &tree.ShowLogserviceSettings{} } yyVAL.union = yyLOCAL - case 652: + case 653: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4583 +//line mysql_sql.y:4596 { yyLOCAL = &tree.ShowRules{ RoleName: yyDollar[5].cstrUnion().Compare(), } } yyVAL.union = yyLOCAL - case 653: + case 654: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4591 +//line mysql_sql.y:4604 { yyLOCAL = &tree.ShowCollation{ Like: yyDollar[3].comparisionExprUnion(), @@ -17433,50 +17568,50 @@ yydefault: } } yyVAL.union = yyLOCAL - case 654: + case 655: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4600 +//line mysql_sql.y:4613 { yyLOCAL = &tree.ShowStages{ Like: yyDollar[3].comparisionExprUnion(), } } yyVAL.union = yyLOCAL - case 655: + case 656: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4608 +//line mysql_sql.y:4621 { yyLOCAL = &tree.ShowSnapShots{ Where: yyDollar[3].whereUnion(), } } yyVAL.union = yyLOCAL - case 656: + case 657: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4616 +//line mysql_sql.y:4629 { yyLOCAL = &tree.ShowPitr{ Where: yyDollar[3].whereUnion(), } } yyVAL.union = yyLOCAL - case 657: + case 658: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4624 +//line mysql_sql.y:4637 { yyLOCAL = &tree.ShowRecoveryWindow{ Level: tree.RECOVERYWINDOWLEVELACCOUNT, } } yyVAL.union = yyLOCAL - case 658: + case 659: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4630 +//line mysql_sql.y:4643 { yyLOCAL = &tree.ShowRecoveryWindow{ Level: tree.RECOVERYWINDOWLEVELDATABASE, @@ -17484,10 +17619,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 659: + case 660: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4637 +//line mysql_sql.y:4650 { yyLOCAL = &tree.ShowRecoveryWindow{ Level: tree.RECOVERYWINDOWLEVELTABLE, @@ -17496,10 +17631,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 660: + case 661: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4645 +//line mysql_sql.y:4658 { yyLOCAL = &tree.ShowRecoveryWindow{ Level: tree.RECOVERYWINDOWLEVELACCOUNT, @@ -17507,26 +17642,26 @@ yydefault: } } yyVAL.union = yyLOCAL - case 661: + case 662: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4654 +//line mysql_sql.y:4667 { yyLOCAL = &tree.ShowGrants{ShowGrantType: tree.GrantForUser} } yyVAL.union = yyLOCAL - case 662: + case 663: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4658 +//line mysql_sql.y:4671 { yyLOCAL = &tree.ShowGrants{Username: yyDollar[4].usernameRecordUnion().Username, Hostname: yyDollar[4].usernameRecordUnion().Hostname, Roles: yyDollar[5].rolesUnion(), ShowGrantType: tree.GrantForUser} } yyVAL.union = yyLOCAL - case 663: + case 664: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4662 +//line mysql_sql.y:4675 { s := &tree.ShowGrants{} roles := []*tree.Role{ @@ -17537,44 +17672,44 @@ yydefault: yyLOCAL = s } yyVAL.union = yyLOCAL - case 664: + case 665: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL []*tree.Role -//line mysql_sql.y:4673 +//line mysql_sql.y:4686 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 665: + case 666: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL []*tree.Role -//line mysql_sql.y:4677 +//line mysql_sql.y:4690 { yyLOCAL = yyDollar[2].rolesUnion() } yyVAL.union = yyLOCAL - case 666: + case 667: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4683 +//line mysql_sql.y:4696 { yyLOCAL = &tree.ShowTableStatus{DbName: yyDollar[5].str, Like: yyDollar[6].comparisionExprUnion(), Where: yyDollar[7].whereUnion()} } yyVAL.union = yyLOCAL - case 667: + case 668: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:4688 +//line mysql_sql.y:4701 { } - case 669: + case 670: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:4692 +//line mysql_sql.y:4705 { } - case 671: + case 672: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4697 +//line mysql_sql.y:4710 { yyLOCAL = &tree.ShowFunctionOrProcedureStatus{ Like: yyDollar[4].comparisionExprUnion(), @@ -17583,10 +17718,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 672: + case 673: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4707 +//line mysql_sql.y:4720 { yyLOCAL = &tree.ShowFunctionOrProcedureStatus{ Like: yyDollar[4].comparisionExprUnion(), @@ -17595,68 +17730,68 @@ yydefault: } } yyVAL.union = yyLOCAL - case 673: + case 674: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4717 +//line mysql_sql.y:4730 { yyLOCAL = &tree.ShowRolesStmt{ Like: yyDollar[3].comparisionExprUnion(), } } yyVAL.union = yyLOCAL - case 674: + case 675: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4725 +//line mysql_sql.y:4738 { yyLOCAL = &tree.ShowNodeList{} } yyVAL.union = yyLOCAL - case 675: + case 676: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4731 +//line mysql_sql.y:4744 { yyLOCAL = &tree.ShowLocks{} } yyVAL.union = yyLOCAL - case 676: + case 677: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4737 +//line mysql_sql.y:4750 { yyLOCAL = &tree.ShowTableNumber{DbName: yyDollar[4].str} } yyVAL.union = yyLOCAL - case 677: + case 678: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4743 +//line mysql_sql.y:4756 { yyLOCAL = &tree.ShowColumnNumber{Table: yyDollar[3].unresolvedObjectNameUnion(), DbName: yyDollar[4].str} } yyVAL.union = yyLOCAL - case 678: + case 679: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4749 +//line mysql_sql.y:4762 { yyLOCAL = &tree.ShowTableValues{Table: yyDollar[3].unresolvedObjectNameUnion(), DbName: yyDollar[4].str} } yyVAL.union = yyLOCAL - case 679: + case 680: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4755 +//line mysql_sql.y:4768 { yyLOCAL = &tree.ShowTableSize{Table: yyDollar[3].unresolvedObjectNameUnion(), DbName: yyDollar[4].str} } yyVAL.union = yyLOCAL - case 680: + case 681: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4761 +//line mysql_sql.y:4774 { s := yyDollar[2].statementUnion().(*tree.ShowTarget) s.Like = yyDollar[3].comparisionExprUnion() @@ -17664,74 +17799,74 @@ yydefault: yyLOCAL = s } yyVAL.union = yyLOCAL - case 681: + case 682: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4770 +//line mysql_sql.y:4783 { yyLOCAL = &tree.ShowTarget{Type: tree.ShowConfig} } yyVAL.union = yyLOCAL - case 682: + case 683: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4774 +//line mysql_sql.y:4787 { yyLOCAL = &tree.ShowTarget{Type: tree.ShowCharset} } yyVAL.union = yyLOCAL - case 683: + case 684: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4778 +//line mysql_sql.y:4791 { yyLOCAL = &tree.ShowTarget{Type: tree.ShowEngines} } yyVAL.union = yyLOCAL - case 684: + case 685: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4782 +//line mysql_sql.y:4795 { yyLOCAL = &tree.ShowTarget{DbName: yyDollar[3].str, Type: tree.ShowTriggers} } yyVAL.union = yyLOCAL - case 685: + case 686: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4786 +//line mysql_sql.y:4799 { yyLOCAL = &tree.ShowTarget{DbName: yyDollar[3].str, Type: tree.ShowEvents} } yyVAL.union = yyLOCAL - case 686: + case 687: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4790 +//line mysql_sql.y:4803 { yyLOCAL = &tree.ShowTarget{Type: tree.ShowPlugins} } yyVAL.union = yyLOCAL - case 687: + case 688: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4794 +//line mysql_sql.y:4807 { yyLOCAL = &tree.ShowTarget{Type: tree.ShowPrivileges} } yyVAL.union = yyLOCAL - case 688: + case 689: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4798 +//line mysql_sql.y:4811 { yyLOCAL = &tree.ShowTarget{Type: tree.ShowProfiles} } yyVAL.union = yyLOCAL - case 689: + case 690: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4804 +//line mysql_sql.y:4817 { yyLOCAL = &tree.ShowIndex{ TableName: yyDollar[4].unresolvedObjectNameUnion(), @@ -17740,20 +17875,20 @@ yydefault: } } yyVAL.union = yyLOCAL - case 690: + case 691: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:4813 +//line mysql_sql.y:4826 { } - case 691: + case 692: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:4815 +//line mysql_sql.y:4828 { } - case 695: + case 696: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4824 +//line mysql_sql.y:4837 { yyLOCAL = &tree.ShowVariables{ Global: yyDollar[2].boolValUnion(), @@ -17762,10 +17897,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 696: + case 697: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4834 +//line mysql_sql.y:4847 { yyLOCAL = &tree.ShowStatus{ Global: yyDollar[2].boolValUnion(), @@ -17774,58 +17909,58 @@ yydefault: } } yyVAL.union = yyLOCAL - case 697: + case 698: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:4843 +//line mysql_sql.y:4856 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 698: + case 699: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:4847 +//line mysql_sql.y:4860 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 699: + case 700: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:4851 +//line mysql_sql.y:4864 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 700: + case 701: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4857 +//line mysql_sql.y:4870 { yyLOCAL = &tree.ShowWarnings{} } yyVAL.union = yyLOCAL - case 701: + case 702: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4863 +//line mysql_sql.y:4876 { yyLOCAL = &tree.ShowErrors{} } yyVAL.union = yyLOCAL - case 702: + case 703: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4869 +//line mysql_sql.y:4882 { yyLOCAL = &tree.ShowProcessList{Full: yyDollar[2].fullOptUnion()} } yyVAL.union = yyLOCAL - case 703: + case 704: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4875 +//line mysql_sql.y:4888 { yyLOCAL = &tree.ShowSequences{ DBName: yyDollar[3].str, @@ -17833,10 +17968,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 704: + case 705: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4884 +//line mysql_sql.y:4897 { yyLOCAL = &tree.ShowTables{ Open: false, @@ -17848,10 +17983,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 705: + case 706: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4895 +//line mysql_sql.y:4908 { yyLOCAL = &tree.ShowTables{ Open: true, @@ -17862,10 +17997,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 706: + case 707: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4907 +//line mysql_sql.y:4920 { yyLOCAL = &tree.ShowDatabases{ Like: yyDollar[3].comparisionExprUnion(), @@ -17874,18 +18009,18 @@ yydefault: } } yyVAL.union = yyLOCAL - case 707: + case 708: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4915 +//line mysql_sql.y:4928 { yyLOCAL = &tree.ShowDatabases{Like: yyDollar[3].comparisionExprUnion(), Where: yyDollar[4].whereUnion()} } yyVAL.union = yyLOCAL - case 708: + case 709: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4921 +//line mysql_sql.y:4934 { yyLOCAL = &tree.ShowColumns{ Ext: false, @@ -17898,10 +18033,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 709: + case 710: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4933 +//line mysql_sql.y:4946 { yyLOCAL = &tree.ShowColumns{ Ext: true, @@ -17914,134 +18049,134 @@ yydefault: } } yyVAL.union = yyLOCAL - case 710: + case 711: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4947 +//line mysql_sql.y:4960 { yyLOCAL = &tree.ShowAccounts{Like: yyDollar[3].comparisionExprUnion()} } yyVAL.union = yyLOCAL - case 711: + case 712: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4953 +//line mysql_sql.y:4966 { yyLOCAL = &tree.ShowPublications{Like: yyDollar[3].comparisionExprUnion()} } yyVAL.union = yyLOCAL - case 712: + case 713: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4959 +//line mysql_sql.y:4972 { yyLOCAL = &tree.ShowPublicationCoverage{Name: yyDollar[4].str} } yyVAL.union = yyLOCAL - case 713: + case 714: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4965 +//line mysql_sql.y:4978 { yyLOCAL = &tree.ShowAccountUpgrade{} } yyVAL.union = yyLOCAL - case 714: + case 715: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4971 +//line mysql_sql.y:4984 { yyLOCAL = &tree.ShowSubscriptions{Like: yyDollar[3].comparisionExprUnion()} } yyVAL.union = yyLOCAL - case 715: + case 716: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4975 +//line mysql_sql.y:4988 { yyLOCAL = &tree.ShowSubscriptions{All: true, Like: yyDollar[4].comparisionExprUnion()} } yyVAL.union = yyLOCAL - case 716: + case 717: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4981 +//line mysql_sql.y:4994 { yyLOCAL = &tree.ShowCcprSubscriptions{TaskId: yyDollar[4].str} } yyVAL.union = yyLOCAL - case 717: + case 718: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:4985 +//line mysql_sql.y:4998 { yyLOCAL = &tree.ShowCcprSubscriptions{} } yyVAL.union = yyLOCAL - case 718: + case 719: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.ComparisonExpr -//line mysql_sql.y:4990 +//line mysql_sql.y:5003 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 719: + case 720: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.ComparisonExpr -//line mysql_sql.y:4994 +//line mysql_sql.y:5007 { yyLOCAL = tree.NewComparisonExpr(tree.LIKE, nil, yyDollar[2].exprUnion()) } yyVAL.union = yyLOCAL - case 720: + case 721: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.ComparisonExpr -//line mysql_sql.y:4998 +//line mysql_sql.y:5011 { yyLOCAL = tree.NewComparisonExpr(tree.ILIKE, nil, yyDollar[2].exprUnion()) } yyVAL.union = yyLOCAL - case 721: + case 722: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:5003 +//line mysql_sql.y:5016 { yyVAL.str = "" } - case 722: + case 723: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:5007 +//line mysql_sql.y:5020 { yyVAL.str = yyDollar[2].cstrUnion().Compare() } - case 723: + case 724: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.UnresolvedObjectName -//line mysql_sql.y:5013 +//line mysql_sql.y:5026 { yyLOCAL = yyDollar[2].unresolvedObjectNameUnion() } yyVAL.union = yyLOCAL - case 728: + case 729: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:5026 +//line mysql_sql.y:5039 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 729: + case 730: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:5030 +//line mysql_sql.y:5043 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 730: + case 731: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5036 +//line mysql_sql.y:5049 { yyLOCAL = &tree.ShowCreateTable{ Name: yyDollar[4].unresolvedObjectNameUnion(), @@ -18049,10 +18184,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 731: + case 732: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5044 +//line mysql_sql.y:5057 { yyLOCAL = &tree.ShowCreateView{ Name: yyDollar[4].unresolvedObjectNameUnion(), @@ -18060,10 +18195,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 732: + case 733: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5051 +//line mysql_sql.y:5064 { yyLOCAL = &tree.ShowCreateDatabase{ IfNotExists: yyDollar[4].ifNotExistsUnion(), @@ -18072,94 +18207,94 @@ yydefault: } } yyVAL.union = yyLOCAL - case 733: + case 734: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5059 +//line mysql_sql.y:5072 { yyLOCAL = &tree.ShowCreatePublications{Name: yyDollar[4].str} } yyVAL.union = yyLOCAL - case 734: + case 735: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5065 +//line mysql_sql.y:5078 { yyLOCAL = &tree.ShowBackendServers{} } yyVAL.union = yyLOCAL - case 735: + case 736: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.UnresolvedObjectName -//line mysql_sql.y:5071 +//line mysql_sql.y:5084 { tblName := yylex.(*Lexer).GetDbOrTblName(yyDollar[1].cstrUnion().Origin()) yyLOCAL = tree.NewUnresolvedObjectName(tblName) } yyVAL.union = yyLOCAL - case 736: + case 737: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.UnresolvedObjectName -//line mysql_sql.y:5076 +//line mysql_sql.y:5089 { dbName := yylex.(*Lexer).GetDbOrTblName(yyDollar[1].cstrUnion().Origin()) tblName := yylex.(*Lexer).GetDbOrTblName(yyDollar[3].cstrUnion().Origin()) yyLOCAL = tree.NewUnresolvedObjectName(dbName, tblName) } yyVAL.union = yyLOCAL - case 737: + case 738: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:5084 +//line mysql_sql.y:5097 { yyVAL.str = yylex.(*Lexer).GetDbOrTblName(yyDollar[1].cstrUnion().Origin()) } - case 738: + case 739: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.UnresolvedObjectName -//line mysql_sql.y:5090 +//line mysql_sql.y:5103 { tblName := yylex.(*Lexer).GetDbOrTblName(yyDollar[1].cstrUnion().Origin()) yyLOCAL = tree.NewUnresolvedObjectName(tblName) } yyVAL.union = yyLOCAL - case 739: + case 740: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.UnresolvedObjectName -//line mysql_sql.y:5095 +//line mysql_sql.y:5108 { dbName := yylex.(*Lexer).GetDbOrTblName(yyDollar[1].cstrUnion().Origin()) tblName := yylex.(*Lexer).GetDbOrTblName(yyDollar[3].cstrUnion().Origin()) yyLOCAL = tree.NewUnresolvedObjectName(dbName, tblName) } yyVAL.union = yyLOCAL - case 740: + case 741: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.UnresolvedObjectName -//line mysql_sql.y:5101 +//line mysql_sql.y:5114 { yyLOCAL = tree.NewUnresolvedObjectName(yyDollar[1].cstrUnion().Compare(), yyDollar[3].cstrUnion().Compare(), yyDollar[5].cstrUnion().Compare()) } yyVAL.union = yyLOCAL - case 741: + case 742: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5107 +//line mysql_sql.y:5120 { yyLOCAL = tree.NewTruncateTable(yyDollar[2].tableNameUnion()) } yyVAL.union = yyLOCAL - case 742: + case 743: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5111 +//line mysql_sql.y:5124 { yyLOCAL = tree.NewTruncateTable(yyDollar[3].tableNameUnion()) } yyVAL.union = yyLOCAL - case 763: + case 764: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5141 +//line mysql_sql.y:5154 { yyLOCAL = &tree.DropSQLTask{ IfExists: yyDollar[3].boolValUnion(), @@ -18167,56 +18302,56 @@ yydefault: } } yyVAL.union = yyLOCAL - case 764: + case 765: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5150 +//line mysql_sql.y:5163 { var ifExists = yyDollar[3].boolValUnion() var name = yyDollar[4].tableNamesUnion() yyLOCAL = tree.NewDropSequence(ifExists, name) } yyVAL.union = yyLOCAL - case 765: + case 766: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5158 +//line mysql_sql.y:5171 { var ifExists = yyDollar[3].boolValUnion() var name = yyDollar[4].exprUnion() yyLOCAL = tree.NewDropAccount(ifExists, name) } yyVAL.union = yyLOCAL - case 766: + case 767: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5166 +//line mysql_sql.y:5179 { var ifExists = yyDollar[3].boolValUnion() var users = yyDollar[4].usersUnion() yyLOCAL = tree.NewDropUser(ifExists, users) } yyVAL.union = yyLOCAL - case 767: + case 768: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []*tree.User -//line mysql_sql.y:5174 +//line mysql_sql.y:5187 { yyLOCAL = []*tree.User{yyDollar[1].userUnion()} } yyVAL.union = yyLOCAL - case 768: + case 769: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []*tree.User -//line mysql_sql.y:5178 +//line mysql_sql.y:5191 { yyLOCAL = append(yyDollar[1].usersUnion(), yyDollar[3].userUnion()) } yyVAL.union = yyLOCAL - case 769: + case 770: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.User -//line mysql_sql.y:5184 +//line mysql_sql.y:5197 { var Username = yyDollar[1].usernameRecordUnion().Username var Hostname = yyDollar[1].usernameRecordUnion().Hostname @@ -18228,20 +18363,20 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 770: + case 771: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5197 +//line mysql_sql.y:5210 { var ifExists = yyDollar[3].boolValUnion() var roles = yyDollar[4].rolesUnion() yyLOCAL = tree.NewDropRole(ifExists, roles) } yyVAL.union = yyLOCAL - case 771: + case 772: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5205 +//line mysql_sql.y:5218 { var name = tree.Identifier(yyDollar[4].cstrUnion().Compare()) var tableName = yyDollar[6].tableNameUnion() @@ -18249,126 +18384,126 @@ yydefault: yyLOCAL = tree.NewDropIndex(name, tableName, ifExists) } yyVAL.union = yyLOCAL - case 772: + case 773: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5214 +//line mysql_sql.y:5227 { var ifExists = yyDollar[4].boolValUnion() var names = yyDollar[5].tableNamesUnion() yyLOCAL = tree.NewDropTable(ifExists, names) } yyVAL.union = yyLOCAL - case 773: + case 774: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5220 +//line mysql_sql.y:5233 { var ifExists = yyDollar[3].boolValUnion() var names = yyDollar[4].tableNamesUnion() yyLOCAL = tree.NewDropTable(ifExists, names) } yyVAL.union = yyLOCAL - case 774: + case 775: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5228 +//line mysql_sql.y:5241 { var ifExists = yyDollar[3].boolValUnion() var names = yyDollar[4].tableNamesUnion() yyLOCAL = tree.NewDropConnector(ifExists, names) } yyVAL.union = yyLOCAL - case 775: + case 776: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5236 +//line mysql_sql.y:5249 { var ifExists = yyDollar[3].boolValUnion() var names = yyDollar[4].tableNamesUnion() yyLOCAL = tree.NewDropView(ifExists, names) } yyVAL.union = yyLOCAL - case 776: + case 777: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5244 +//line mysql_sql.y:5257 { var name = tree.Identifier(yyDollar[4].cstrUnion().Compare()) var ifExists = yyDollar[3].boolValUnion() yyLOCAL = tree.NewDropDatabase(name, ifExists) } yyVAL.union = yyLOCAL - case 777: + case 778: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5250 +//line mysql_sql.y:5263 { var name = tree.Identifier(yyDollar[4].cstrUnion().Compare()) var ifExists = yyDollar[3].boolValUnion() yyLOCAL = tree.NewDropDatabase(name, ifExists) } yyVAL.union = yyLOCAL - case 778: + case 779: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5258 +//line mysql_sql.y:5271 { yyLOCAL = tree.NewDeallocate(tree.Identifier(yyDollar[3].str), true) } yyVAL.union = yyLOCAL - case 779: + case 780: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5264 +//line mysql_sql.y:5277 { var name = yyDollar[3].functionNameUnion() var args = yyDollar[5].funcArgsUnion() yyLOCAL = tree.NewDropFunction(name, args) } yyVAL.union = yyLOCAL - case 780: + case 781: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5272 +//line mysql_sql.y:5285 { var name = yyDollar[3].procNameUnion() var ifExists = false yyLOCAL = tree.NewDropProcedure(name, ifExists) } yyVAL.union = yyLOCAL - case 781: + case 782: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5278 +//line mysql_sql.y:5291 { var name = yyDollar[5].procNameUnion() var ifExists = true yyLOCAL = tree.NewDropProcedure(name, ifExists) } yyVAL.union = yyLOCAL - case 784: + case 785: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5288 +//line mysql_sql.y:5301 { yyDollar[2].statementUnion().(*tree.Delete).With = yyDollar[1].withClauseUnion() yyLOCAL = yyDollar[2].statementUnion() } yyVAL.union = yyLOCAL - case 785: + case 786: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5293 +//line mysql_sql.y:5306 { yyDollar[2].statementUnion().(*tree.Delete).With = yyDollar[1].withClauseUnion() yyLOCAL = yyDollar[2].statementUnion() } yyVAL.union = yyLOCAL - case 786: + case 787: yyDollar = yyS[yypt-11 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5300 +//line mysql_sql.y:5313 { // Single-Table Syntax t := &tree.AliasedTableExpr{ @@ -18385,10 +18520,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 787: + case 788: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5316 +//line mysql_sql.y:5329 { // Multiple-Table Syntax yyLOCAL = &tree.Delete{ @@ -18398,10 +18533,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 788: + case 789: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5327 +//line mysql_sql.y:5340 { // Multiple-Table Syntax yyLOCAL = &tree.Delete{ @@ -18411,36 +18546,36 @@ yydefault: } } yyVAL.union = yyLOCAL - case 789: + case 790: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.TableExprs -//line mysql_sql.y:5338 +//line mysql_sql.y:5351 { yyLOCAL = tree.TableExprs{yyDollar[1].tableNameUnion()} } yyVAL.union = yyLOCAL - case 790: + case 791: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableExprs -//line mysql_sql.y:5342 +//line mysql_sql.y:5355 { yyLOCAL = append(yyDollar[1].tableExprsUnion(), yyDollar[3].tableNameUnion()) } yyVAL.union = yyLOCAL - case 791: + case 792: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.TableName -//line mysql_sql.y:5348 +//line mysql_sql.y:5361 { tblName := yylex.(*Lexer).GetDbOrTblName(yyDollar[1].cstrUnion().Origin()) prefix := tree.ObjectNamePrefix{ExplicitSchema: false} yyLOCAL = tree.NewTableName(tree.Identifier(tblName), prefix, nil) } yyVAL.union = yyLOCAL - case 792: + case 793: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.TableName -//line mysql_sql.y:5354 +//line mysql_sql.y:5367 { dbName := yylex.(*Lexer).GetDbOrTblName(yyDollar[1].cstrUnion().Origin()) tblName := yylex.(*Lexer).GetDbOrTblName(yyDollar[3].cstrUnion().Origin()) @@ -18448,40 +18583,40 @@ yydefault: yyLOCAL = tree.NewTableName(tree.Identifier(tblName), prefix, nil) } yyVAL.union = yyLOCAL - case 793: + case 794: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:5363 +//line mysql_sql.y:5376 { } - case 794: + case 795: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:5365 +//line mysql_sql.y:5378 { } - case 795: + case 796: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:5368 +//line mysql_sql.y:5381 { } - case 800: + case 801: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:5377 +//line mysql_sql.y:5390 { } - case 802: + case 803: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:5381 +//line mysql_sql.y:5394 { } - case 804: + case 805: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:5388 +//line mysql_sql.y:5401 { } - case 807: + case 808: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5394 +//line mysql_sql.y:5407 { rep := yyDollar[5].replaceUnion() rep.Table = yyDollar[3].tableExprUnion() @@ -18489,10 +18624,10 @@ yydefault: yyLOCAL = rep } yyVAL.union = yyLOCAL - case 808: + case 809: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.Replace -//line mysql_sql.y:5403 +//line mysql_sql.y:5416 { vc := tree.NewValuesClause(yyDollar[2].rowsExprsUnion()) yyLOCAL = &tree.Replace{ @@ -18500,20 +18635,20 @@ yydefault: } } yyVAL.union = yyLOCAL - case 809: + case 810: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.Replace -//line mysql_sql.y:5410 +//line mysql_sql.y:5423 { yyLOCAL = &tree.Replace{ Rows: yyDollar[1].selectUnion(), } } yyVAL.union = yyLOCAL - case 810: + case 811: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.Replace -//line mysql_sql.y:5416 +//line mysql_sql.y:5429 { yyLOCAL = &tree.Replace{ Columns: yyDollar[2].identifierListUnion(), @@ -18521,20 +18656,20 @@ yydefault: } } yyVAL.union = yyLOCAL - case 811: + case 812: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.Replace -//line mysql_sql.y:5423 +//line mysql_sql.y:5436 { yyLOCAL = &tree.Replace{ Rows: yyDollar[1].selectUnion(), } } yyVAL.union = yyLOCAL - case 812: + case 813: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.Replace -//line mysql_sql.y:5429 +//line mysql_sql.y:5442 { vc := tree.NewValuesClause(yyDollar[5].rowsExprsUnion()) yyLOCAL = &tree.Replace{ @@ -18543,10 +18678,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 813: + case 814: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.Replace -//line mysql_sql.y:5437 +//line mysql_sql.y:5450 { vc := tree.NewValuesClause(yyDollar[4].rowsExprsUnion()) yyLOCAL = &tree.Replace{ @@ -18554,10 +18689,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 814: + case 815: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.Replace -//line mysql_sql.y:5444 +//line mysql_sql.y:5457 { yyLOCAL = &tree.Replace{ Columns: yyDollar[2].identifierListUnion(), @@ -18565,10 +18700,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 815: + case 816: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.Replace -//line mysql_sql.y:5451 +//line mysql_sql.y:5464 { if yyDollar[2].assignmentsUnion() == nil { yylex.Error("the set list of replace can not be empty") @@ -18588,29 +18723,29 @@ yydefault: } } yyVAL.union = yyLOCAL - case 816: + case 817: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.Select -//line mysql_sql.y:5472 +//line mysql_sql.y:5485 { // MySQL treats TABLE as a query source, so ORDER BY and LIMIT belong to // the SELECT wrapper produced by the TABLE-to-SELECT rewrite. yyLOCAL = tree.NewSelect(makeSelectStarFromTable(yyDollar[2].tableNameUnion()), yyDollar[3].orderByUnion(), yyDollar[4].limitUnion()) } yyVAL.union = yyLOCAL - case 818: + case 819: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5481 +//line mysql_sql.y:5494 { yyDollar[2].statementUnion().(*tree.Insert).With = yyDollar[1].withClauseUnion() yyLOCAL = yyDollar[2].statementUnion() } yyVAL.union = yyLOCAL - case 819: + case 820: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5488 +//line mysql_sql.y:5501 { ins := yyDollar[4].insertUnion() ins.Table = yyDollar[2].tableExprUnion() @@ -18619,10 +18754,10 @@ yydefault: yyLOCAL = ins } yyVAL.union = yyLOCAL - case 820: + case 821: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:5496 +//line mysql_sql.y:5509 { ins := yyDollar[5].insertUnion() ins.Table = yyDollar[3].tableExprUnion() @@ -18631,26 +18766,26 @@ yydefault: yyLOCAL = ins } yyVAL.union = yyLOCAL - case 821: + case 822: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.IdentifierList -//line mysql_sql.y:5506 +//line mysql_sql.y:5519 { yyLOCAL = tree.IdentifierList{tree.Identifier(yyDollar[1].str)} } yyVAL.union = yyLOCAL - case 822: + case 823: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.IdentifierList -//line mysql_sql.y:5510 +//line mysql_sql.y:5523 { yyLOCAL = append(yyDollar[1].identifierListUnion(), tree.Identifier(yyDollar[3].str)) } yyVAL.union = yyLOCAL - case 823: + case 824: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.Insert -//line mysql_sql.y:5516 +//line mysql_sql.y:5529 { vc := tree.NewValuesClause(yyDollar[2].rowsExprsUnion()) yyLOCAL = &tree.Insert{ @@ -18658,20 +18793,20 @@ yydefault: } } yyVAL.union = yyLOCAL - case 824: + case 825: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.Insert -//line mysql_sql.y:5523 +//line mysql_sql.y:5536 { yyLOCAL = &tree.Insert{ Rows: yyDollar[1].selectUnion(), } } yyVAL.union = yyLOCAL - case 825: + case 826: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.Insert -//line mysql_sql.y:5529 +//line mysql_sql.y:5542 { vc := tree.NewValuesClause(yyDollar[5].rowsExprsUnion()) yyLOCAL = &tree.Insert{ @@ -18680,10 +18815,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 826: + case 827: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.Insert -//line mysql_sql.y:5537 +//line mysql_sql.y:5550 { vc := tree.NewValuesClause(yyDollar[4].rowsExprsUnion()) yyLOCAL = &tree.Insert{ @@ -18691,10 +18826,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 827: + case 828: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.Insert -//line mysql_sql.y:5544 +//line mysql_sql.y:5557 { yyLOCAL = &tree.Insert{ Columns: yyDollar[2].identifierListUnion(), @@ -18702,10 +18837,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 828: + case 829: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.Insert -//line mysql_sql.y:5551 +//line mysql_sql.y:5564 { if yyDollar[2].assignmentsUnion() == nil { yylex.Error("the set list of insert can not be empty") @@ -18724,58 +18859,58 @@ yydefault: } } yyVAL.union = yyLOCAL - case 829: + case 830: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.UpdateExprs -//line mysql_sql.y:5570 +//line mysql_sql.y:5583 { yyLOCAL = []*tree.UpdateExpr{} } yyVAL.union = yyLOCAL - case 830: + case 831: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.UpdateExprs -//line mysql_sql.y:5574 +//line mysql_sql.y:5587 { yyLOCAL = yyDollar[5].updateExprsUnion() } yyVAL.union = yyLOCAL - case 831: + case 832: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.UpdateExprs -//line mysql_sql.y:5578 +//line mysql_sql.y:5591 { yyLOCAL = []*tree.UpdateExpr{nil} } yyVAL.union = yyLOCAL - case 832: + case 833: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL []*tree.Assignment -//line mysql_sql.y:5583 +//line mysql_sql.y:5596 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 833: + case 834: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []*tree.Assignment -//line mysql_sql.y:5587 +//line mysql_sql.y:5600 { yyLOCAL = []*tree.Assignment{yyDollar[1].assignmentUnion()} } yyVAL.union = yyLOCAL - case 834: + case 835: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []*tree.Assignment -//line mysql_sql.y:5591 +//line mysql_sql.y:5604 { yyLOCAL = append(yyDollar[1].assignmentsUnion(), yyDollar[3].assignmentUnion()) } yyVAL.union = yyLOCAL - case 835: + case 836: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.Assignment -//line mysql_sql.y:5597 +//line mysql_sql.y:5610 { yyLOCAL = &tree.Assignment{ Column: tree.Identifier(yyDollar[1].str), @@ -18783,155 +18918,155 @@ yydefault: } } yyVAL.union = yyLOCAL - case 836: + case 837: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.IdentifierList -//line mysql_sql.y:5606 +//line mysql_sql.y:5619 { yyLOCAL = tree.IdentifierList{tree.Identifier(yyDollar[1].str)} } yyVAL.union = yyLOCAL - case 837: + case 838: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.IdentifierList -//line mysql_sql.y:5610 +//line mysql_sql.y:5623 { yyLOCAL = append(yyDollar[1].identifierListUnion(), tree.Identifier(yyDollar[3].str)) } yyVAL.union = yyLOCAL - case 838: + case 839: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:5616 +//line mysql_sql.y:5629 { yyVAL.str = yylex.(*Lexer).GetDbOrTblName(yyDollar[1].cstrUnion().Origin()) } - case 839: + case 840: yyDollar = yyS[yypt-3 : yypt+1] -//line mysql_sql.y:5620 +//line mysql_sql.y:5633 { yyVAL.str = yylex.(*Lexer).GetDbOrTblName(yyDollar[3].cstrUnion().Origin()) } - case 840: + case 841: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []tree.Exprs -//line mysql_sql.y:5626 +//line mysql_sql.y:5639 { yyLOCAL = []tree.Exprs{yyDollar[1].exprsUnion()} } yyVAL.union = yyLOCAL - case 841: + case 842: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []tree.Exprs -//line mysql_sql.y:5630 +//line mysql_sql.y:5643 { yyLOCAL = append(yyDollar[1].rowsExprsUnion(), yyDollar[3].exprsUnion()) } yyVAL.union = yyLOCAL - case 842: + case 843: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Exprs -//line mysql_sql.y:5636 +//line mysql_sql.y:5649 { yyLOCAL = yyDollar[3].exprsUnion() } yyVAL.union = yyLOCAL - case 843: + case 844: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:5641 +//line mysql_sql.y:5654 { } - case 845: + case 846: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.Exprs -//line mysql_sql.y:5645 +//line mysql_sql.y:5658 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 847: + case 848: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Exprs -//line mysql_sql.y:5652 +//line mysql_sql.y:5665 { yyLOCAL = tree.Exprs{yyDollar[1].exprUnion()} } yyVAL.union = yyLOCAL - case 848: + case 849: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Exprs -//line mysql_sql.y:5656 +//line mysql_sql.y:5669 { yyLOCAL = append(yyDollar[1].exprsUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 850: + case 851: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:5663 +//line mysql_sql.y:5676 { yyLOCAL = &tree.DefaultVal{} } yyVAL.union = yyLOCAL - case 851: + case 852: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.IdentifierList -//line mysql_sql.y:5668 +//line mysql_sql.y:5681 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 852: + case 853: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.IdentifierList -//line mysql_sql.y:5672 +//line mysql_sql.y:5685 { yyLOCAL = yyDollar[3].identifierListUnion() } yyVAL.union = yyLOCAL - case 853: + case 854: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.IdentifierList -//line mysql_sql.y:5678 +//line mysql_sql.y:5691 { yyLOCAL = tree.IdentifierList{tree.Identifier(yyDollar[1].cstrUnion().Compare())} } yyVAL.union = yyLOCAL - case 854: + case 855: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.IdentifierList -//line mysql_sql.y:5682 +//line mysql_sql.y:5695 { yyLOCAL = append(yyDollar[1].identifierListUnion(), tree.Identifier(yyDollar[3].cstrUnion().Compare())) } yyVAL.union = yyLOCAL - case 855: + case 856: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.TableExpr -//line mysql_sql.y:5688 +//line mysql_sql.y:5701 { yyLOCAL = yyDollar[2].tableNameUnion() } yyVAL.union = yyLOCAL - case 856: + case 857: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.TableExpr -//line mysql_sql.y:5692 +//line mysql_sql.y:5705 { yyLOCAL = yyDollar[1].tableNameUnion() } yyVAL.union = yyLOCAL - case 857: + case 858: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.ExportParam -//line mysql_sql.y:5697 +//line mysql_sql.y:5710 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 858: + case 859: yyDollar = yyS[yypt-10 : yypt+1] var yyLOCAL *tree.ExportParam -//line mysql_sql.y:5701 +//line mysql_sql.y:5714 { yyLOCAL = &tree.ExportParam{ Outfile: true, @@ -18946,15 +19081,15 @@ yydefault: } } yyVAL.union = yyLOCAL - case 859: + case 860: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:5716 +//line mysql_sql.y:5729 { yyVAL.str = "" } - case 860: + case 861: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:5720 +//line mysql_sql.y:5733 { str := strings.ToLower(yyDollar[2].str) if str != "csv" && str != "jsonline" && str != "parquet" { @@ -18963,18 +19098,18 @@ yydefault: } yyVAL.str = str } - case 861: + case 862: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL uint64 -//line mysql_sql.y:5730 +//line mysql_sql.y:5743 { yyLOCAL = uint64(0) } yyVAL.union = yyLOCAL - case 862: + case 863: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL uint64 -//line mysql_sql.y:5734 +//line mysql_sql.y:5747 { size, err := util.ParseDataSize(yyDollar[2].str) if err != nil { @@ -18984,10 +19119,10 @@ yydefault: yyLOCAL = size } yyVAL.union = yyLOCAL - case 863: + case 864: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.Fields -//line mysql_sql.y:5744 +//line mysql_sql.y:5757 { yyLOCAL = &tree.Fields{ Terminated: &tree.Terminated{ @@ -18999,10 +19134,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 864: + case 865: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.Fields -//line mysql_sql.y:5755 +//line mysql_sql.y:5768 { yyLOCAL = &tree.Fields{ Terminated: &tree.Terminated{ @@ -19014,10 +19149,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 865: + case 866: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL *tree.Fields -//line mysql_sql.y:5766 +//line mysql_sql.y:5779 { str := yyDollar[7].str if str != "\\" && len(str) > 1 { @@ -19040,10 +19175,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 866: + case 867: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.Fields -//line mysql_sql.y:5788 +//line mysql_sql.y:5801 { str := yyDollar[4].str if str != "\\" && len(str) > 1 { @@ -19066,10 +19201,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 867: + case 868: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.Lines -//line mysql_sql.y:5811 +//line mysql_sql.y:5824 { yyLOCAL = &tree.Lines{ TerminatedBy: &tree.Terminated{ @@ -19078,10 +19213,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 868: + case 869: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.Lines -//line mysql_sql.y:5819 +//line mysql_sql.y:5832 { yyLOCAL = &tree.Lines{ TerminatedBy: &tree.Terminated{ @@ -19090,18 +19225,18 @@ yydefault: } } yyVAL.union = yyLOCAL - case 869: + case 870: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:5828 +//line mysql_sql.y:5841 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 870: + case 871: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:5832 +//line mysql_sql.y:5845 { str := strings.ToLower(yyDollar[2].str) if str == "true" { @@ -19114,131 +19249,131 @@ yydefault: } } yyVAL.union = yyLOCAL - case 871: + case 872: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL int64 -//line mysql_sql.y:5845 +//line mysql_sql.y:5858 { yyLOCAL = 0 } yyVAL.union = yyLOCAL - case 872: + case 873: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL int64 -//line mysql_sql.y:5849 +//line mysql_sql.y:5862 { yyLOCAL = yyDollar[2].item.(int64) } yyVAL.union = yyLOCAL - case 873: + case 874: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:5854 +//line mysql_sql.y:5867 { yyLOCAL = []string{} } yyVAL.union = yyLOCAL - case 874: + case 875: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:5858 +//line mysql_sql.y:5871 { yyLOCAL = yyDollar[3].strsUnion() } yyVAL.union = yyLOCAL - case 875: + case 876: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:5864 +//line mysql_sql.y:5877 { yyLOCAL = make([]string, 0, 4) yyLOCAL = append(yyLOCAL, yyDollar[1].cstrUnion().Compare()) } yyVAL.union = yyLOCAL - case 876: + case 877: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:5869 +//line mysql_sql.y:5882 { yyLOCAL = append(yyDollar[1].strsUnion(), yyDollar[3].cstrUnion().Compare()) } yyVAL.union = yyLOCAL - case 878: + case 879: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.Select -//line mysql_sql.y:5876 +//line mysql_sql.y:5889 { yyLOCAL = &tree.Select{Select: yyDollar[1].selectStatementUnion()} } yyVAL.union = yyLOCAL - case 879: + case 880: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL *tree.Select -//line mysql_sql.y:5882 +//line mysql_sql.y:5895 { yyLOCAL = &tree.Select{Select: yyDollar[1].selectStatementUnion(), TimeWindow: yyDollar[2].timeWindowUnion(), OrderBy: yyDollar[3].orderByUnion(), Limit: yyDollar[4].limitUnion(), RankOption: yyDollar[5].rankOptionUnion(), Ep: yyDollar[6].exportParmUnion(), SelectLockInfo: yyDollar[7].selectLockInfoUnion()} } yyVAL.union = yyLOCAL - case 880: + case 881: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.Select -//line mysql_sql.y:5886 +//line mysql_sql.y:5899 { yyLOCAL = &tree.Select{Select: yyDollar[1].selectStatementUnion(), TimeWindow: yyDollar[2].timeWindowUnion(), OrderBy: yyDollar[3].orderByUnion(), Ep: yyDollar[4].exportParmUnion()} } yyVAL.union = yyLOCAL - case 881: + case 882: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.Select -//line mysql_sql.y:5890 +//line mysql_sql.y:5903 { yyLOCAL = &tree.Select{Select: yyDollar[1].selectStatementUnion(), TimeWindow: yyDollar[2].timeWindowUnion(), OrderBy: yyDollar[3].orderByUnion(), Limit: yyDollar[4].limitUnion(), RankOption: yyDollar[5].rankOptionUnion(), Ep: yyDollar[6].exportParmUnion()} } yyVAL.union = yyLOCAL - case 882: + case 883: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL *tree.Select -//line mysql_sql.y:5894 +//line mysql_sql.y:5907 { yyLOCAL = &tree.Select{Select: yyDollar[2].selectStatementUnion(), TimeWindow: yyDollar[3].timeWindowUnion(), OrderBy: yyDollar[4].orderByUnion(), Limit: yyDollar[5].limitUnion(), RankOption: yyDollar[6].rankOptionUnion(), Ep: yyDollar[7].exportParmUnion(), SelectLockInfo: yyDollar[8].selectLockInfoUnion(), With: yyDollar[1].withClauseUnion()} } yyVAL.union = yyLOCAL - case 883: + case 884: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.Select -//line mysql_sql.y:5898 +//line mysql_sql.y:5911 { yyLOCAL = &tree.Select{Select: yyDollar[2].selectStatementUnion(), OrderBy: yyDollar[3].orderByUnion(), Ep: yyDollar[4].exportParmUnion(), With: yyDollar[1].withClauseUnion()} } yyVAL.union = yyLOCAL - case 884: + case 885: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.Select -//line mysql_sql.y:5902 +//line mysql_sql.y:5915 { yyLOCAL = &tree.Select{Select: yyDollar[2].selectStatementUnion(), OrderBy: yyDollar[3].orderByUnion(), Limit: yyDollar[4].limitUnion(), RankOption: yyDollar[5].rankOptionUnion(), Ep: yyDollar[6].exportParmUnion(), With: yyDollar[1].withClauseUnion()} } yyVAL.union = yyLOCAL - case 885: + case 886: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.TimeWindow -//line mysql_sql.y:5907 +//line mysql_sql.y:5920 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 886: + case 887: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.TimeWindow -//line mysql_sql.y:5911 +//line mysql_sql.y:5924 { yyLOCAL = yyDollar[1].timeWindowUnion() } yyVAL.union = yyLOCAL - case 887: + case 888: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.TimeWindow -//line mysql_sql.y:5917 +//line mysql_sql.y:5930 { yyLOCAL = &tree.TimeWindow{ Interval: yyDollar[1].timeIntervalUnion(), @@ -19247,10 +19382,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 888: + case 889: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL *tree.Interval -//line mysql_sql.y:5927 +//line mysql_sql.y:5940 { str := fmt.Sprintf("%v", yyDollar[5].item) v, errStr := util.GetInt64(yyDollar[5].item) @@ -19265,18 +19400,18 @@ yydefault: } } yyVAL.union = yyLOCAL - case 889: + case 890: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.Sliding -//line mysql_sql.y:5942 +//line mysql_sql.y:5955 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 890: + case 891: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.Sliding -//line mysql_sql.y:5946 +//line mysql_sql.y:5959 { str := fmt.Sprintf("%v", yyDollar[3].item) v, errStr := util.GetInt64(yyDollar[3].item) @@ -19290,28 +19425,28 @@ yydefault: } } yyVAL.union = yyLOCAL - case 891: + case 892: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.Fill -//line mysql_sql.y:5960 +//line mysql_sql.y:5973 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 892: + case 893: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.Fill -//line mysql_sql.y:5964 +//line mysql_sql.y:5977 { yyLOCAL = &tree.Fill{ Mode: yyDollar[3].fillModeUnion(), } } yyVAL.union = yyLOCAL - case 893: + case 894: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.Fill -//line mysql_sql.y:5970 +//line mysql_sql.y:5983 { yyLOCAL = &tree.Fill{ Mode: tree.FillValue, @@ -19319,50 +19454,50 @@ yydefault: } } yyVAL.union = yyLOCAL - case 894: + case 895: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.FillMode -//line mysql_sql.y:5979 +//line mysql_sql.y:5992 { yyLOCAL = tree.FillPrev } yyVAL.union = yyLOCAL - case 895: + case 896: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.FillMode -//line mysql_sql.y:5983 +//line mysql_sql.y:5996 { yyLOCAL = tree.FillNext } yyVAL.union = yyLOCAL - case 896: + case 897: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.FillMode -//line mysql_sql.y:5987 +//line mysql_sql.y:6000 { yyLOCAL = tree.FillNone } yyVAL.union = yyLOCAL - case 897: + case 898: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.FillMode -//line mysql_sql.y:5991 +//line mysql_sql.y:6004 { yyLOCAL = tree.FillNull } yyVAL.union = yyLOCAL - case 898: + case 899: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.FillMode -//line mysql_sql.y:5995 +//line mysql_sql.y:6008 { yyLOCAL = tree.FillLinear } yyVAL.union = yyLOCAL - case 899: + case 900: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.With -//line mysql_sql.y:6001 +//line mysql_sql.y:6014 { yyLOCAL = &tree.With{ IsRecursive: false, @@ -19370,10 +19505,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 900: + case 901: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.With -//line mysql_sql.y:6008 +//line mysql_sql.y:6021 { yyLOCAL = &tree.With{ IsRecursive: true, @@ -19381,26 +19516,26 @@ yydefault: } } yyVAL.union = yyLOCAL - case 901: + case 902: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []*tree.CTE -//line mysql_sql.y:6017 +//line mysql_sql.y:6030 { yyLOCAL = []*tree.CTE{yyDollar[1].cteUnion()} } yyVAL.union = yyLOCAL - case 902: + case 903: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []*tree.CTE -//line mysql_sql.y:6021 +//line mysql_sql.y:6034 { yyLOCAL = append(yyDollar[1].cteListUnion(), yyDollar[3].cteUnion()) } yyVAL.union = yyLOCAL - case 903: + case 904: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.CTE -//line mysql_sql.y:6027 +//line mysql_sql.y:6040 { yyLOCAL = &tree.CTE{ Name: &tree.AliasClause{Alias: tree.Identifier(yyDollar[1].cstrUnion().Compare()), Cols: yyDollar[2].identifierListUnion()}, @@ -19408,74 +19543,74 @@ yydefault: } } yyVAL.union = yyLOCAL - case 904: + case 905: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.IdentifierList -//line mysql_sql.y:6035 +//line mysql_sql.y:6048 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 905: + case 906: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.IdentifierList -//line mysql_sql.y:6039 +//line mysql_sql.y:6052 { yyLOCAL = yyDollar[2].identifierListUnion() } yyVAL.union = yyLOCAL - case 906: + case 907: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.Limit -//line mysql_sql.y:6044 +//line mysql_sql.y:6057 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 907: + case 908: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.Limit -//line mysql_sql.y:6048 +//line mysql_sql.y:6061 { yyLOCAL = yyDollar[1].limitUnion() } yyVAL.union = yyLOCAL - case 908: + case 909: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.Limit -//line mysql_sql.y:6054 +//line mysql_sql.y:6067 { yyLOCAL = &tree.Limit{Count: yyDollar[2].exprUnion()} } yyVAL.union = yyLOCAL - case 909: + case 910: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.Limit -//line mysql_sql.y:6058 +//line mysql_sql.y:6071 { yyLOCAL = &tree.Limit{Offset: yyDollar[2].exprUnion(), Count: yyDollar[4].exprUnion()} } yyVAL.union = yyLOCAL - case 910: + case 911: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.Limit -//line mysql_sql.y:6062 +//line mysql_sql.y:6075 { yyLOCAL = &tree.Limit{Offset: yyDollar[4].exprUnion(), Count: yyDollar[2].exprUnion()} } yyVAL.union = yyLOCAL - case 911: + case 912: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.RankOption -//line mysql_sql.y:6067 +//line mysql_sql.y:6080 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 912: + case 913: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.RankOption -//line mysql_sql.y:6071 +//line mysql_sql.y:6084 { // Parse option strings to extract key=value pairs into a map optionMap := make(map[string]string) @@ -19510,140 +19645,140 @@ yydefault: } } yyVAL.union = yyLOCAL - case 913: + case 914: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.OrderBy -//line mysql_sql.y:6106 +//line mysql_sql.y:6119 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 914: + case 915: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.OrderBy -//line mysql_sql.y:6110 +//line mysql_sql.y:6123 { yyLOCAL = yyDollar[1].orderByUnion() } yyVAL.union = yyLOCAL - case 915: + case 916: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.OrderBy -//line mysql_sql.y:6116 +//line mysql_sql.y:6129 { yyLOCAL = yyDollar[3].orderByUnion() } yyVAL.union = yyLOCAL - case 916: + case 917: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.OrderBy -//line mysql_sql.y:6122 +//line mysql_sql.y:6135 { yyLOCAL = tree.OrderBy{yyDollar[1].orderUnion()} } yyVAL.union = yyLOCAL - case 917: + case 918: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.OrderBy -//line mysql_sql.y:6126 +//line mysql_sql.y:6139 { yyLOCAL = append(yyDollar[1].orderByUnion(), yyDollar[3].orderUnion()) } yyVAL.union = yyLOCAL - case 918: + case 919: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.Order -//line mysql_sql.y:6132 +//line mysql_sql.y:6145 { yyLOCAL = &tree.Order{Expr: yyDollar[1].exprUnion(), Direction: yyDollar[2].directionUnion(), NullsPosition: yyDollar[3].nullsPositionUnion()} } yyVAL.union = yyLOCAL - case 919: + case 920: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.Direction -//line mysql_sql.y:6137 +//line mysql_sql.y:6150 { yyLOCAL = tree.DefaultDirection } yyVAL.union = yyLOCAL - case 920: + case 921: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Direction -//line mysql_sql.y:6141 +//line mysql_sql.y:6154 { yyLOCAL = tree.Ascending } yyVAL.union = yyLOCAL - case 921: + case 922: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Direction -//line mysql_sql.y:6145 +//line mysql_sql.y:6158 { yyLOCAL = tree.Descending } yyVAL.union = yyLOCAL - case 922: + case 923: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.NullsPosition -//line mysql_sql.y:6150 +//line mysql_sql.y:6163 { yyLOCAL = tree.DefaultNullsPosition } yyVAL.union = yyLOCAL - case 923: + case 924: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.NullsPosition -//line mysql_sql.y:6154 +//line mysql_sql.y:6167 { yyLOCAL = tree.NullsFirst } yyVAL.union = yyLOCAL - case 924: + case 925: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.NullsPosition -//line mysql_sql.y:6158 +//line mysql_sql.y:6171 { yyLOCAL = tree.NullsLast } yyVAL.union = yyLOCAL - case 925: + case 926: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.SelectLockInfo -//line mysql_sql.y:6163 +//line mysql_sql.y:6176 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 926: + case 927: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.SelectLockInfo -//line mysql_sql.y:6167 +//line mysql_sql.y:6180 { yyLOCAL = &tree.SelectLockInfo{ LockType: tree.SelectLockForUpdate, } } yyVAL.union = yyLOCAL - case 927: + case 928: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.SelectStatement -//line mysql_sql.y:6175 +//line mysql_sql.y:6188 { yyLOCAL = &tree.ParenSelect{Select: yyDollar[2].selectUnion()} } yyVAL.union = yyLOCAL - case 928: + case 929: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.SelectStatement -//line mysql_sql.y:6179 +//line mysql_sql.y:6192 { yyLOCAL = &tree.ParenSelect{Select: &tree.Select{Select: yyDollar[2].selectStatementUnion()}} } yyVAL.union = yyLOCAL - case 929: + case 930: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.SelectStatement -//line mysql_sql.y:6183 +//line mysql_sql.y:6196 { valuesStmt := yyDollar[2].statementUnion().(*tree.ValuesStatement) yyLOCAL = &tree.ParenSelect{Select: &tree.Select{ @@ -19656,18 +19791,18 @@ yydefault: }} } yyVAL.union = yyLOCAL - case 930: + case 931: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.SelectStatement -//line mysql_sql.y:6197 +//line mysql_sql.y:6210 { yyLOCAL = yyDollar[1].selectStatementUnion() } yyVAL.union = yyLOCAL - case 931: + case 932: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.SelectStatement -//line mysql_sql.y:6201 +//line mysql_sql.y:6214 { yyLOCAL = &tree.UnionClause{ Type: yyDollar[2].unionTypeRecordUnion().Type, @@ -19678,10 +19813,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 932: + case 933: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.SelectStatement -//line mysql_sql.y:6211 +//line mysql_sql.y:6224 { yyLOCAL = &tree.UnionClause{ Type: yyDollar[2].unionTypeRecordUnion().Type, @@ -19692,10 +19827,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 933: + case 934: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.SelectStatement -//line mysql_sql.y:6221 +//line mysql_sql.y:6234 { yyLOCAL = &tree.UnionClause{ Type: yyDollar[2].unionTypeRecordUnion().Type, @@ -19706,10 +19841,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 934: + case 935: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.SelectStatement -//line mysql_sql.y:6231 +//line mysql_sql.y:6244 { yyLOCAL = &tree.UnionClause{ Type: yyDollar[2].unionTypeRecordUnion().Type, @@ -19720,10 +19855,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 935: + case 936: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.UnionTypeRecord -//line mysql_sql.y:6243 +//line mysql_sql.y:6256 { yyLOCAL = &tree.UnionTypeRecord{ Type: tree.UNION, @@ -19732,10 +19867,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 936: + case 937: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.UnionTypeRecord -//line mysql_sql.y:6251 +//line mysql_sql.y:6264 { yyLOCAL = &tree.UnionTypeRecord{ Type: tree.UNION, @@ -19744,10 +19879,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 937: + case 938: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.UnionTypeRecord -//line mysql_sql.y:6259 +//line mysql_sql.y:6272 { yyLOCAL = &tree.UnionTypeRecord{ Type: tree.UNION, @@ -19756,10 +19891,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 938: + case 939: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.UnionTypeRecord -//line mysql_sql.y:6268 +//line mysql_sql.y:6281 { yyLOCAL = &tree.UnionTypeRecord{ Type: tree.EXCEPT, @@ -19768,10 +19903,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 939: + case 940: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.UnionTypeRecord -//line mysql_sql.y:6276 +//line mysql_sql.y:6289 { yyLOCAL = &tree.UnionTypeRecord{ Type: tree.EXCEPT, @@ -19780,10 +19915,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 940: + case 941: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.UnionTypeRecord -//line mysql_sql.y:6284 +//line mysql_sql.y:6297 { yyLOCAL = &tree.UnionTypeRecord{ Type: tree.EXCEPT, @@ -19792,10 +19927,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 941: + case 942: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.UnionTypeRecord -//line mysql_sql.y:6292 +//line mysql_sql.y:6305 { yyLOCAL = &tree.UnionTypeRecord{ Type: tree.INTERSECT, @@ -19804,10 +19939,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 942: + case 943: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.UnionTypeRecord -//line mysql_sql.y:6300 +//line mysql_sql.y:6313 { yyLOCAL = &tree.UnionTypeRecord{ Type: tree.INTERSECT, @@ -19816,10 +19951,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 943: + case 944: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.UnionTypeRecord -//line mysql_sql.y:6308 +//line mysql_sql.y:6321 { yyLOCAL = &tree.UnionTypeRecord{ Type: tree.INTERSECT, @@ -19828,10 +19963,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 944: + case 945: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.UnionTypeRecord -//line mysql_sql.y:6316 +//line mysql_sql.y:6329 { yyLOCAL = &tree.UnionTypeRecord{ Type: tree.UT_MINUS, @@ -19840,10 +19975,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 945: + case 946: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.UnionTypeRecord -//line mysql_sql.y:6324 +//line mysql_sql.y:6337 { yyLOCAL = &tree.UnionTypeRecord{ Type: tree.UT_MINUS, @@ -19852,10 +19987,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 946: + case 947: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.UnionTypeRecord -//line mysql_sql.y:6332 +//line mysql_sql.y:6345 { yyLOCAL = &tree.UnionTypeRecord{ Type: tree.UT_MINUS, @@ -19864,10 +19999,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 947: + case 948: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.SelectStatement -//line mysql_sql.y:6342 +//line mysql_sql.y:6355 { yyLOCAL = &tree.SelectClause{ Distinct: tree.QuerySpecOptionDistinct&yyDollar[2].selectOptionsUnion() != 0, @@ -19880,146 +20015,146 @@ yydefault: } } yyVAL.union = yyLOCAL - case 948: + case 949: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL uint64 -//line mysql_sql.y:6356 +//line mysql_sql.y:6369 { yyLOCAL = tree.QuerySpecOptionNone } yyVAL.union = yyLOCAL - case 949: + case 950: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL uint64 -//line mysql_sql.y:6360 +//line mysql_sql.y:6373 { yyLOCAL = yyDollar[1].selectOptionsUnion() } yyVAL.union = yyLOCAL - case 950: + case 951: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL uint64 -//line mysql_sql.y:6366 +//line mysql_sql.y:6379 { yyLOCAL = yyDollar[1].selectOptionUnion() } yyVAL.union = yyLOCAL - case 951: + case 952: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL uint64 -//line mysql_sql.y:6370 +//line mysql_sql.y:6383 { yyLOCAL = yyDollar[1].selectOptionsUnion() | yyDollar[2].selectOptionUnion() } yyVAL.union = yyLOCAL - case 952: + case 953: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL uint64 -//line mysql_sql.y:6376 +//line mysql_sql.y:6389 { yyLOCAL = tree.QuerySpecOptionSqlSmallResult } yyVAL.union = yyLOCAL - case 953: + case 954: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL uint64 -//line mysql_sql.y:6380 +//line mysql_sql.y:6393 { yyLOCAL = tree.QuerySpecOptionSqlBigResult } yyVAL.union = yyLOCAL - case 954: + case 955: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL uint64 -//line mysql_sql.y:6384 +//line mysql_sql.y:6397 { yyLOCAL = tree.QuerySpecOptionSqlBufferResult } yyVAL.union = yyLOCAL - case 955: + case 956: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL uint64 -//line mysql_sql.y:6388 +//line mysql_sql.y:6401 { yyLOCAL = tree.QuerySpecOptionStraightJoin } yyVAL.union = yyLOCAL - case 956: + case 957: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL uint64 -//line mysql_sql.y:6392 +//line mysql_sql.y:6405 { yyLOCAL = tree.QuerySpecOptionHighPriority } yyVAL.union = yyLOCAL - case 957: + case 958: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL uint64 -//line mysql_sql.y:6396 +//line mysql_sql.y:6409 { yyLOCAL = tree.QuerySpecOptionSqlCalcFoundRows } yyVAL.union = yyLOCAL - case 958: + case 959: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL uint64 -//line mysql_sql.y:6400 +//line mysql_sql.y:6413 { yyLOCAL = tree.QuerySpecOptionSqlNoCache } yyVAL.union = yyLOCAL - case 959: + case 960: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL uint64 -//line mysql_sql.y:6404 +//line mysql_sql.y:6417 { yyLOCAL = tree.QuerySpecOptionAll } yyVAL.union = yyLOCAL - case 960: + case 961: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL uint64 -//line mysql_sql.y:6408 +//line mysql_sql.y:6421 { yyLOCAL = tree.QuerySpecOptionDistinct } yyVAL.union = yyLOCAL - case 961: + case 962: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL uint64 -//line mysql_sql.y:6412 +//line mysql_sql.y:6425 { yyLOCAL = tree.QuerySpecOptionDistinctRow } yyVAL.union = yyLOCAL - case 962: + case 963: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.Where -//line mysql_sql.y:6434 +//line mysql_sql.y:6447 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 963: + case 964: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.Where -//line mysql_sql.y:6438 +//line mysql_sql.y:6451 { yyLOCAL = &tree.Where{Type: tree.AstHaving, Expr: yyDollar[2].exprUnion()} } yyVAL.union = yyLOCAL - case 964: + case 965: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.GroupByClause -//line mysql_sql.y:6443 +//line mysql_sql.y:6456 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 965: + case 966: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.GroupByClause -//line mysql_sql.y:6447 +//line mysql_sql.y:6460 { exprsList := []tree.Exprs{yyDollar[3].exprsUnion()} yyLOCAL = &tree.GroupByClause{ @@ -20030,10 +20165,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 966: + case 967: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL *tree.GroupByClause -//line mysql_sql.y:6457 +//line mysql_sql.y:6470 { yyLOCAL = &tree.GroupByClause{ GroupByExprsList: yyDollar[6].rowsExprsUnion(), @@ -20043,10 +20178,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 967: + case 968: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.GroupByClause -//line mysql_sql.y:6466 +//line mysql_sql.y:6479 { yyLOCAL = &tree.GroupByClause{ GroupByExprsList: []tree.Exprs{yyDollar[5].exprsUnion()}, @@ -20056,10 +20191,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 968: + case 969: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.GroupByClause -//line mysql_sql.y:6475 +//line mysql_sql.y:6488 { yyLOCAL = &tree.GroupByClause{ GroupByExprsList: []tree.Exprs{yyDollar[5].exprsUnion()}, @@ -20069,106 +20204,106 @@ yydefault: } } yyVAL.union = yyLOCAL - case 969: + case 970: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []tree.Exprs -//line mysql_sql.y:6486 +//line mysql_sql.y:6499 { yyLOCAL = []tree.Exprs{yyDollar[2].exprsUnion()} } yyVAL.union = yyLOCAL - case 970: + case 971: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL []tree.Exprs -//line mysql_sql.y:6490 +//line mysql_sql.y:6503 { yyLOCAL = append(yyDollar[1].rowsExprsUnion(), yyDollar[4].exprsUnion()) } yyVAL.union = yyLOCAL - case 971: + case 972: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:6496 +//line mysql_sql.y:6509 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 972: + case 973: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:6500 +//line mysql_sql.y:6513 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 973: + case 974: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.Where -//line mysql_sql.y:6505 +//line mysql_sql.y:6518 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 974: + case 975: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.Where -//line mysql_sql.y:6509 +//line mysql_sql.y:6522 { yyLOCAL = &tree.Where{Type: tree.AstWhere, Expr: yyDollar[2].exprUnion()} } yyVAL.union = yyLOCAL - case 975: + case 976: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.SelectExprs -//line mysql_sql.y:6515 +//line mysql_sql.y:6528 { yyLOCAL = tree.SelectExprs{yyDollar[1].selectExprUnion()} } yyVAL.union = yyLOCAL - case 976: + case 977: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.SelectExprs -//line mysql_sql.y:6519 +//line mysql_sql.y:6532 { yyLOCAL = append(yyDollar[1].selectExprsUnion(), yyDollar[3].selectExprUnion()) } yyVAL.union = yyLOCAL - case 977: + case 978: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.SelectExpr -//line mysql_sql.y:6525 +//line mysql_sql.y:6538 { yyLOCAL = tree.SelectExpr{Expr: tree.StarExpr()} } yyVAL.union = yyLOCAL - case 978: + case 979: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.SelectExpr -//line mysql_sql.y:6529 +//line mysql_sql.y:6542 { yyLOCAL = tree.SelectExpr{Expr: yyDollar[1].exprUnion(), As: yyDollar[2].cstrUnion()} } yyVAL.union = yyLOCAL - case 979: + case 980: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.SelectExpr -//line mysql_sql.y:6533 +//line mysql_sql.y:6546 { yyLOCAL = tree.SelectExpr{Expr: tree.NewUnresolvedNameWithStar(yyDollar[1].cstrUnion())} } yyVAL.union = yyLOCAL - case 980: + case 981: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.SelectExpr -//line mysql_sql.y:6537 +//line mysql_sql.y:6550 { yyLOCAL = tree.SelectExpr{Expr: tree.NewUnresolvedNameWithStar(yyDollar[1].cstrUnion(), yyDollar[3].cstrUnion())} } yyVAL.union = yyLOCAL - case 981: + case 982: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.From -//line mysql_sql.y:6542 +//line mysql_sql.y:6555 { prefix := tree.ObjectNamePrefix{ExplicitSchema: false} tn := tree.NewTableName(tree.Identifier(""), prefix, nil) @@ -20177,28 +20312,28 @@ yydefault: } } yyVAL.union = yyLOCAL - case 982: + case 983: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.From -//line mysql_sql.y:6550 +//line mysql_sql.y:6563 { yyLOCAL = yyDollar[1].fromUnion() } yyVAL.union = yyLOCAL - case 983: + case 984: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.From -//line mysql_sql.y:6556 +//line mysql_sql.y:6569 { yyLOCAL = &tree.From{ Tables: tree.TableExprs{yyDollar[2].tableExprUnion()}, } } yyVAL.union = yyLOCAL - case 984: + case 985: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.TableExpr -//line mysql_sql.y:6564 +//line mysql_sql.y:6577 { if t, ok := yyDollar[1].tableExprUnion().(*tree.JoinTableExpr); ok { yyLOCAL = t @@ -20209,34 +20344,34 @@ yydefault: } } yyVAL.union = yyLOCAL - case 985: + case 986: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableExpr -//line mysql_sql.y:6574 +//line mysql_sql.y:6587 { yyLOCAL = &tree.JoinTableExpr{Left: yyDollar[1].tableExprUnion(), Right: yyDollar[3].tableExprUnion(), JoinType: tree.JOIN_TYPE_CROSS} } yyVAL.union = yyLOCAL - case 988: + case 989: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.TableExpr -//line mysql_sql.y:6584 +//line mysql_sql.y:6597 { yyLOCAL = yyDollar[1].joinTableExprUnion() } yyVAL.union = yyLOCAL - case 989: + case 990: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.TableExpr -//line mysql_sql.y:6588 +//line mysql_sql.y:6601 { yyLOCAL = yyDollar[1].applyTableExprUnion() } yyVAL.union = yyLOCAL - case 990: + case 991: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.JoinTableExpr -//line mysql_sql.y:6594 +//line mysql_sql.y:6607 { if strings.Contains(yyDollar[2].str, ":") { ss := strings.SplitN(yyDollar[2].str, ":", 2) @@ -20257,10 +20392,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 991: + case 992: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.JoinTableExpr -//line mysql_sql.y:6614 +//line mysql_sql.y:6627 { yyLOCAL = &tree.JoinTableExpr{ Left: yyDollar[1].tableExprUnion(), @@ -20270,10 +20405,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 992: + case 993: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.JoinTableExpr -//line mysql_sql.y:6623 +//line mysql_sql.y:6636 { yyLOCAL = &tree.JoinTableExpr{ Left: yyDollar[1].tableExprUnion(), @@ -20283,10 +20418,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 993: + case 994: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.JoinTableExpr -//line mysql_sql.y:6632 +//line mysql_sql.y:6645 { yyLOCAL = &tree.JoinTableExpr{ Left: yyDollar[1].tableExprUnion(), @@ -20295,10 +20430,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 994: + case 995: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.JoinTableExpr -//line mysql_sql.y:6640 +//line mysql_sql.y:6653 { yyLOCAL = &tree.JoinTableExpr{ Left: yyDollar[1].tableExprUnion(), @@ -20308,10 +20443,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 995: + case 996: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.ApplyTableExpr -//line mysql_sql.y:6651 +//line mysql_sql.y:6664 { yyLOCAL = &tree.ApplyTableExpr{ Left: yyDollar[1].tableExprUnion(), @@ -20320,27 +20455,27 @@ yydefault: } } yyVAL.union = yyLOCAL - case 996: + case 997: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:6661 +//line mysql_sql.y:6674 { yyVAL.str = tree.APPLY_TYPE_CROSS } - case 997: + case 998: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:6665 +//line mysql_sql.y:6678 { yyVAL.str = tree.APPLY_TYPE_OUTER } - case 998: + case 999: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:6671 +//line mysql_sql.y:6684 { yyVAL.str = tree.JOIN_TYPE_NATURAL } - case 999: + case 1000: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:6675 +//line mysql_sql.y:6688 { switch yyDollar[2].str { case tree.JOIN_TYPE_LEFT: @@ -20351,52 +20486,52 @@ yydefault: yyVAL.str = tree.JOIN_TYPE_NATURAL_FULL } } - case 1000: + case 1001: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:6688 +//line mysql_sql.y:6701 { yyVAL.str = tree.JOIN_TYPE_LEFT } - case 1001: + case 1002: yyDollar = yyS[yypt-3 : yypt+1] -//line mysql_sql.y:6692 +//line mysql_sql.y:6705 { yyVAL.str = tree.JOIN_TYPE_LEFT } - case 1002: + case 1003: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:6696 +//line mysql_sql.y:6709 { yyVAL.str = tree.JOIN_TYPE_RIGHT } - case 1003: + case 1004: yyDollar = yyS[yypt-3 : yypt+1] -//line mysql_sql.y:6700 +//line mysql_sql.y:6713 { yyVAL.str = tree.JOIN_TYPE_RIGHT } - case 1004: + case 1005: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:6704 +//line mysql_sql.y:6717 { yyVAL.str = tree.JOIN_TYPE_FULL } - case 1005: + case 1006: yyDollar = yyS[yypt-3 : yypt+1] -//line mysql_sql.y:6708 +//line mysql_sql.y:6721 { yyVAL.str = tree.JOIN_TYPE_FULL } - case 1006: + case 1007: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:6714 +//line mysql_sql.y:6727 { yyVAL.str = tree.JOIN_TYPE_DEDUP } - case 1007: + case 1008: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:6720 +//line mysql_sql.y:6733 { yyLOCAL = &tree.ValuesStatement{ Rows: yyDollar[2].rowsExprsUnion(), @@ -20405,148 +20540,148 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1008: + case 1009: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []tree.Exprs -//line mysql_sql.y:6730 +//line mysql_sql.y:6743 { yyLOCAL = []tree.Exprs{yyDollar[1].exprsUnion()} } yyVAL.union = yyLOCAL - case 1009: + case 1010: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []tree.Exprs -//line mysql_sql.y:6734 +//line mysql_sql.y:6747 { yyLOCAL = append(yyDollar[1].rowsExprsUnion(), yyDollar[3].exprsUnion()) } yyVAL.union = yyLOCAL - case 1010: + case 1011: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Exprs -//line mysql_sql.y:6740 +//line mysql_sql.y:6753 { yyLOCAL = yyDollar[3].exprsUnion() } yyVAL.union = yyLOCAL - case 1011: + case 1012: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.JoinCond -//line mysql_sql.y:6746 +//line mysql_sql.y:6759 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1012: + case 1013: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.JoinCond -//line mysql_sql.y:6750 +//line mysql_sql.y:6763 { yyLOCAL = &tree.OnJoinCond{Expr: yyDollar[2].exprUnion()} } yyVAL.union = yyLOCAL - case 1013: + case 1014: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:6756 +//line mysql_sql.y:6769 { yyVAL.str = yyDollar[1].str } - case 1014: + case 1015: yyDollar = yyS[yypt-3 : yypt+1] -//line mysql_sql.y:6762 +//line mysql_sql.y:6775 { yyVAL.str = yyDollar[2].str } - case 1015: + case 1016: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:6768 +//line mysql_sql.y:6781 { yyVAL.str = tree.JOIN_TYPE_STRAIGHT } - case 1016: + case 1017: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:6774 +//line mysql_sql.y:6787 { yyVAL.str = tree.JOIN_TYPE_INNER } - case 1017: + case 1018: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:6778 +//line mysql_sql.y:6791 { yyVAL.str = tree.JOIN_TYPE_INNER } - case 1018: + case 1019: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:6782 +//line mysql_sql.y:6795 { yyVAL.str = tree.JOIN_TYPE_CROSS } - case 1019: + case 1020: yyDollar = yyS[yypt-3 : yypt+1] -//line mysql_sql.y:6786 +//line mysql_sql.y:6799 { yyVAL.str = tree.JOIN_TYPE_CENTROIDX + ":" + yyDollar[2].str } - case 1020: + case 1021: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.JoinCond -//line mysql_sql.y:6792 +//line mysql_sql.y:6805 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1021: + case 1022: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.JoinCond -//line mysql_sql.y:6796 +//line mysql_sql.y:6809 { yyLOCAL = yyDollar[1].joinCondUnion() } yyVAL.union = yyLOCAL - case 1022: + case 1023: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.JoinCond -//line mysql_sql.y:6802 +//line mysql_sql.y:6815 { yyLOCAL = &tree.OnJoinCond{Expr: yyDollar[2].exprUnion()} } yyVAL.union = yyLOCAL - case 1023: + case 1024: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.JoinCond -//line mysql_sql.y:6806 +//line mysql_sql.y:6819 { yyLOCAL = &tree.UsingJoinCond{Cols: yyDollar[3].identifierListUnion()} } yyVAL.union = yyLOCAL - case 1024: + case 1025: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.IdentifierList -//line mysql_sql.y:6812 +//line mysql_sql.y:6825 { yyLOCAL = tree.IdentifierList{tree.Identifier(yyDollar[1].cstrUnion().Compare())} } yyVAL.union = yyLOCAL - case 1025: + case 1026: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.IdentifierList -//line mysql_sql.y:6816 +//line mysql_sql.y:6829 { yyLOCAL = append(yyDollar[1].identifierListUnion(), tree.Identifier(yyDollar[3].cstrUnion().Compare())) } yyVAL.union = yyLOCAL - case 1026: + case 1027: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.TableExpr -//line mysql_sql.y:6822 +//line mysql_sql.y:6835 { yyLOCAL = yyDollar[1].aliasedTableExprUnion() } yyVAL.union = yyLOCAL - case 1027: + case 1028: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableExpr -//line mysql_sql.y:6826 +//line mysql_sql.y:6839 { yyLOCAL = &tree.AliasedTableExpr{ Expr: yyDollar[1].parenTableExprUnion(), @@ -20557,10 +20692,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1028: + case 1029: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.TableExpr -//line mysql_sql.y:6836 +//line mysql_sql.y:6849 { if yyDollar[2].str != "" { yyLOCAL = &tree.AliasedTableExpr{ @@ -20574,26 +20709,26 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1029: + case 1030: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableExpr -//line mysql_sql.y:6849 +//line mysql_sql.y:6862 { yyLOCAL = yyDollar[2].tableExprUnion() } yyVAL.union = yyLOCAL - case 1030: + case 1031: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.ParenTableExpr -//line mysql_sql.y:6855 +//line mysql_sql.y:6868 { yyLOCAL = &tree.ParenTableExpr{Expr: yyDollar[1].selectStatementUnion().(*tree.ParenSelect).Select} } yyVAL.union = yyLOCAL - case 1031: + case 1032: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.TableExpr -//line mysql_sql.y:6861 +//line mysql_sql.y:6874 { name := tree.NewUnresolvedName(yyDollar[1].cstrUnion()) yyLOCAL = &tree.TableFunction{ @@ -20606,10 +20741,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1032: + case 1033: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.AliasedTableExpr -//line mysql_sql.y:6875 +//line mysql_sql.y:6888 { yyLOCAL = &tree.AliasedTableExpr{ Expr: yyDollar[1].tableNameUnion(), @@ -20620,34 +20755,34 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1033: + case 1034: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL []*tree.IndexHint -//line mysql_sql.y:6886 +//line mysql_sql.y:6899 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1035: + case 1036: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []*tree.IndexHint -//line mysql_sql.y:6893 +//line mysql_sql.y:6906 { yyLOCAL = []*tree.IndexHint{yyDollar[1].indexHintUnion()} } yyVAL.union = yyLOCAL - case 1036: + case 1037: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL []*tree.IndexHint -//line mysql_sql.y:6897 +//line mysql_sql.y:6910 { yyLOCAL = append(yyDollar[1].indexHintListUnion(), yyDollar[2].indexHintUnion()) } yyVAL.union = yyLOCAL - case 1037: + case 1038: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.IndexHint -//line mysql_sql.y:6903 +//line mysql_sql.y:6916 { yyLOCAL = &tree.IndexHint{ IndexNames: yyDollar[4].strsUnion(), @@ -20656,182 +20791,182 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1038: + case 1039: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.IndexHintType -//line mysql_sql.y:6913 +//line mysql_sql.y:6926 { yyLOCAL = tree.HintUse } yyVAL.union = yyLOCAL - case 1039: + case 1040: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.IndexHintType -//line mysql_sql.y:6917 +//line mysql_sql.y:6930 { yyLOCAL = tree.HintIgnore } yyVAL.union = yyLOCAL - case 1040: + case 1041: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.IndexHintType -//line mysql_sql.y:6921 +//line mysql_sql.y:6934 { yyLOCAL = tree.HintForce } yyVAL.union = yyLOCAL - case 1041: + case 1042: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.IndexHintScope -//line mysql_sql.y:6926 +//line mysql_sql.y:6939 { yyLOCAL = tree.HintForScan } yyVAL.union = yyLOCAL - case 1042: + case 1043: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.IndexHintScope -//line mysql_sql.y:6930 +//line mysql_sql.y:6943 { yyLOCAL = tree.HintForJoin } yyVAL.union = yyLOCAL - case 1043: + case 1044: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.IndexHintScope -//line mysql_sql.y:6934 +//line mysql_sql.y:6947 { yyLOCAL = tree.HintForOrderBy } yyVAL.union = yyLOCAL - case 1044: + case 1045: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.IndexHintScope -//line mysql_sql.y:6938 +//line mysql_sql.y:6951 { yyLOCAL = tree.HintForGroupBy } yyVAL.union = yyLOCAL - case 1045: + case 1046: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:6943 +//line mysql_sql.y:6956 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1046: + case 1047: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:6947 +//line mysql_sql.y:6960 { yyLOCAL = []string{yyDollar[1].cstrUnion().Compare()} } yyVAL.union = yyLOCAL - case 1047: + case 1048: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:6951 +//line mysql_sql.y:6964 { yyLOCAL = append(yyDollar[1].strsUnion(), yyDollar[3].cstrUnion().Compare()) } yyVAL.union = yyLOCAL - case 1048: + case 1049: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:6955 +//line mysql_sql.y:6968 { yyLOCAL = []string{yyDollar[1].str} } yyVAL.union = yyLOCAL - case 1049: + case 1050: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:6959 +//line mysql_sql.y:6972 { yyLOCAL = append(yyDollar[1].strsUnion(), yyDollar[3].str) } yyVAL.union = yyLOCAL - case 1050: + case 1051: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:6964 +//line mysql_sql.y:6977 { yyVAL.str = "" } - case 1051: + case 1052: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:6968 +//line mysql_sql.y:6981 { yyVAL.str = yyDollar[1].str } - case 1052: + case 1053: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:6972 +//line mysql_sql.y:6985 { yyVAL.str = yyDollar[2].str } - case 1053: + case 1054: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:6978 +//line mysql_sql.y:6991 { yyVAL.str = yylex.(*Lexer).GetDbOrTblName(yyDollar[1].cstrUnion().Origin()) } - case 1054: + case 1055: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:6982 +//line mysql_sql.y:6995 { yyVAL.str = yylex.(*Lexer).GetDbOrTblName(yyDollar[1].str) } - case 1055: + case 1056: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.CStr -//line mysql_sql.y:6987 +//line mysql_sql.y:7000 { yyLOCAL = tree.NewCStr("", 1) } yyVAL.union = yyLOCAL - case 1056: + case 1057: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.CStr -//line mysql_sql.y:6991 +//line mysql_sql.y:7004 { yyLOCAL = yyDollar[1].cstrUnion() } yyVAL.union = yyLOCAL - case 1057: + case 1058: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.CStr -//line mysql_sql.y:6995 +//line mysql_sql.y:7008 { yyLOCAL = yyDollar[2].cstrUnion() } yyVAL.union = yyLOCAL - case 1058: + case 1059: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.CStr -//line mysql_sql.y:6999 +//line mysql_sql.y:7012 { yyLOCAL = tree.NewCStr(yyDollar[1].str, 1) } yyVAL.union = yyLOCAL - case 1059: + case 1060: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.CStr -//line mysql_sql.y:7003 +//line mysql_sql.y:7016 { yyLOCAL = tree.NewCStr(yyDollar[2].str, 1) } yyVAL.union = yyLOCAL - case 1060: + case 1061: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:7009 +//line mysql_sql.y:7022 { yyVAL.str = yyDollar[1].cstrUnion().Compare() } - case 1084: + case 1085: yyDollar = yyS[yypt-10 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7052 +//line mysql_sql.y:7065 { cronExpr := "" timezone := "" @@ -20851,18 +20986,18 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1085: + case 1086: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.SQLTaskSchedule -//line mysql_sql.y:7072 +//line mysql_sql.y:7085 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1086: + case 1087: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.SQLTaskSchedule -//line mysql_sql.y:7076 +//line mysql_sql.y:7089 { yyLOCAL = &tree.SQLTaskSchedule{ CronExpr: yyDollar[2].str, @@ -20870,82 +21005,82 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1087: + case 1088: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:7084 +//line mysql_sql.y:7097 { yyVAL.str = "" } - case 1088: + case 1089: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:7088 +//line mysql_sql.y:7101 { yyVAL.str = yyDollar[2].str } - case 1089: + case 1090: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:7093 +//line mysql_sql.y:7106 { yyLOCAL = tree.Expr(nil) } yyVAL.union = yyLOCAL - case 1090: + case 1091: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:7097 +//line mysql_sql.y:7110 { yyLOCAL = yyDollar[3].exprUnion() } yyVAL.union = yyLOCAL - case 1091: + case 1092: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:7103 +//line mysql_sql.y:7116 { yyLOCAL = yyDollar[1].exprUnion() } yyVAL.union = yyLOCAL - case 1092: + case 1093: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:7107 +//line mysql_sql.y:7120 { yyLOCAL = tree.NewSubquery(yyDollar[1].selectUnion(), false) } yyVAL.union = yyLOCAL - case 1093: + case 1094: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL int64 -//line mysql_sql.y:7112 +//line mysql_sql.y:7125 { yyLOCAL = 0 } yyVAL.union = yyLOCAL - case 1094: + case 1095: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL int64 -//line mysql_sql.y:7116 +//line mysql_sql.y:7129 { yyLOCAL = sqlTaskInt64(yyDollar[2].item) } yyVAL.union = yyLOCAL - case 1095: + case 1096: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:7121 +//line mysql_sql.y:7134 { yyVAL.str = "" } - case 1096: + case 1097: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:7125 +//line mysql_sql.y:7138 { yyVAL.str = yyDollar[2].str } - case 1097: + case 1098: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7131 +//line mysql_sql.y:7144 { var Language = yyDollar[3].str var Name = tree.Identifier(yyDollar[5].str) @@ -20957,135 +21092,135 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1098: + case 1099: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:7144 +//line mysql_sql.y:7157 { yyVAL.str = yyDollar[1].cstrUnion().Compare() } - case 1099: + case 1100: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:7150 +//line mysql_sql.y:7163 { yyVAL.str = yyDollar[1].cstrUnion().Compare() } - case 1100: + case 1101: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7156 +//line mysql_sql.y:7169 { yyLOCAL = tree.NewCreateProcedure( yyDollar[2].sourceOptionalUnion(), yyDollar[4].procNameUnion(), yyDollar[6].procArgsUnion(), yyDollar[8].str, yyDollar[9].str, ) } yyVAL.union = yyLOCAL - case 1101: + case 1102: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.ProcedureName -//line mysql_sql.y:7164 +//line mysql_sql.y:7177 { prefix := tree.ObjectNamePrefix{ExplicitSchema: false} yyLOCAL = tree.NewProcedureName(tree.Identifier(yyDollar[1].cstrUnion().Compare()), prefix) } yyVAL.union = yyLOCAL - case 1102: + case 1103: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.ProcedureName -//line mysql_sql.y:7169 +//line mysql_sql.y:7182 { dbName := yylex.(*Lexer).GetDbOrTblName(yyDollar[1].cstrUnion().Origin()) prefix := tree.ObjectNamePrefix{SchemaName: tree.Identifier(dbName), ExplicitSchema: true} yyLOCAL = tree.NewProcedureName(tree.Identifier(yyDollar[3].cstrUnion().Compare()), prefix) } yyVAL.union = yyLOCAL - case 1103: + case 1104: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.ProcedureArgs -//line mysql_sql.y:7176 +//line mysql_sql.y:7189 { yyLOCAL = tree.ProcedureArgs(nil) } yyVAL.union = yyLOCAL - case 1105: + case 1106: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ProcedureArgs -//line mysql_sql.y:7183 +//line mysql_sql.y:7196 { yyLOCAL = tree.ProcedureArgs{yyDollar[1].procArgUnion()} } yyVAL.union = yyLOCAL - case 1106: + case 1107: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.ProcedureArgs -//line mysql_sql.y:7187 +//line mysql_sql.y:7200 { yyLOCAL = append(yyDollar[1].procArgsUnion(), yyDollar[3].procArgUnion()) } yyVAL.union = yyLOCAL - case 1107: + case 1108: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ProcedureArg -//line mysql_sql.y:7193 +//line mysql_sql.y:7206 { yyLOCAL = tree.ProcedureArg(yyDollar[1].procArgDeclUnion()) } yyVAL.union = yyLOCAL - case 1108: + case 1109: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.ProcedureArgDecl -//line mysql_sql.y:7199 +//line mysql_sql.y:7212 { yyLOCAL = tree.NewProcedureArgDecl(yyDollar[1].procArgTypeUnion(), yyDollar[2].unresolvedNameUnion(), yyDollar[3].columnTypeUnion()) } yyVAL.union = yyLOCAL - case 1109: + case 1110: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.InOutArgType -//line mysql_sql.y:7204 +//line mysql_sql.y:7217 { yyLOCAL = tree.TYPE_IN } yyVAL.union = yyLOCAL - case 1110: + case 1111: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.InOutArgType -//line mysql_sql.y:7208 +//line mysql_sql.y:7221 { yyLOCAL = tree.TYPE_IN } yyVAL.union = yyLOCAL - case 1111: + case 1112: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.InOutArgType -//line mysql_sql.y:7212 +//line mysql_sql.y:7225 { yyLOCAL = tree.TYPE_OUT } yyVAL.union = yyLOCAL - case 1112: + case 1113: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.InOutArgType -//line mysql_sql.y:7216 +//line mysql_sql.y:7229 { yyLOCAL = tree.TYPE_INOUT } yyVAL.union = yyLOCAL - case 1113: + case 1114: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:7221 +//line mysql_sql.y:7234 { yyVAL.str = "sql" } - case 1114: + case 1115: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:7225 +//line mysql_sql.y:7238 { yyVAL.str = yyDollar[2].str } - case 1115: + case 1116: yyDollar = yyS[yypt-14 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7231 +//line mysql_sql.y:7244 { if yyDollar[13].str == "" { yylex.Error("no function body error") @@ -21117,127 +21252,127 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1116: + case 1117: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.FunctionName -//line mysql_sql.y:7264 +//line mysql_sql.y:7277 { prefix := tree.ObjectNamePrefix{ExplicitSchema: false} yyLOCAL = tree.NewFuncName(tree.Identifier(yyDollar[1].cstrUnion().Compare()), prefix) } yyVAL.union = yyLOCAL - case 1117: + case 1118: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.FunctionName -//line mysql_sql.y:7269 +//line mysql_sql.y:7282 { dbName := yylex.(*Lexer).GetDbOrTblName(yyDollar[1].cstrUnion().Origin()) prefix := tree.ObjectNamePrefix{SchemaName: tree.Identifier(dbName), ExplicitSchema: true} yyLOCAL = tree.NewFuncName(tree.Identifier(yyDollar[3].cstrUnion().Compare()), prefix) } yyVAL.union = yyLOCAL - case 1118: + case 1119: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.FunctionArgs -//line mysql_sql.y:7276 +//line mysql_sql.y:7289 { yyLOCAL = tree.FunctionArgs(nil) } yyVAL.union = yyLOCAL - case 1120: + case 1121: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.FunctionArgs -//line mysql_sql.y:7283 +//line mysql_sql.y:7296 { yyLOCAL = tree.FunctionArgs{yyDollar[1].funcArgUnion()} } yyVAL.union = yyLOCAL - case 1121: + case 1122: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.FunctionArgs -//line mysql_sql.y:7287 +//line mysql_sql.y:7300 { yyLOCAL = append(yyDollar[1].funcArgsUnion(), yyDollar[3].funcArgUnion()) } yyVAL.union = yyLOCAL - case 1122: + case 1123: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.FunctionArg -//line mysql_sql.y:7293 +//line mysql_sql.y:7306 { yyLOCAL = tree.FunctionArg(yyDollar[1].funcArgDeclUnion()) } yyVAL.union = yyLOCAL - case 1123: + case 1124: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.FunctionArgDecl -//line mysql_sql.y:7299 +//line mysql_sql.y:7312 { yyLOCAL = tree.NewFunctionArgDecl(nil, yyDollar[1].columnTypeUnion(), nil) } yyVAL.union = yyLOCAL - case 1124: + case 1125: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.FunctionArgDecl -//line mysql_sql.y:7303 +//line mysql_sql.y:7316 { yyLOCAL = tree.NewFunctionArgDecl(yyDollar[1].unresolvedNameUnion(), yyDollar[2].columnTypeUnion(), nil) } yyVAL.union = yyLOCAL - case 1125: + case 1126: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FunctionArgDecl -//line mysql_sql.y:7307 +//line mysql_sql.y:7320 { yyLOCAL = tree.NewFunctionArgDecl(yyDollar[1].unresolvedNameUnion(), yyDollar[2].columnTypeUnion(), yyDollar[4].exprUnion()) } yyVAL.union = yyLOCAL - case 1126: + case 1127: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:7313 +//line mysql_sql.y:7326 { yyVAL.str = yyDollar[1].cstrUnion().Compare() } - case 1127: + case 1128: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.ReturnType -//line mysql_sql.y:7319 +//line mysql_sql.y:7332 { yyLOCAL = tree.NewReturnType(yyDollar[1].columnTypeUnion()) } yyVAL.union = yyLOCAL - case 1128: + case 1129: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:7325 +//line mysql_sql.y:7338 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 1129: + case 1130: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:7329 +//line mysql_sql.y:7342 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 1130: + case 1131: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:7334 +//line mysql_sql.y:7347 { yyVAL.str = "" } - case 1132: + case 1133: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:7341 +//line mysql_sql.y:7354 { yyVAL.str = yyDollar[2].str } - case 1133: + case 1134: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7347 +//line mysql_sql.y:7360 { var Replace bool var Name = yyDollar[5].tableNameUnion() @@ -21253,10 +21388,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1134: + case 1135: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7362 +//line mysql_sql.y:7375 { var Replace = yyDollar[2].sourceOptionalUnion() var Name = yyDollar[5].tableNameUnion() @@ -21272,10 +21407,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1135: + case 1136: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7379 +//line mysql_sql.y:7392 { var IfNotExists = yyDollar[3].ifNotExistsUnion() var Name = yyDollar[4].exprUnion() @@ -21291,10 +21426,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1136: + case 1137: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7394 +//line mysql_sql.y:7407 { var FromUri = yyDollar[4].str var SubscriptionAccountName = yyDollar[5].cstrUnion().Compare() @@ -21312,81 +21447,81 @@ yydefault: yyLOCAL = cs } yyVAL.union = yyLOCAL - case 1137: + case 1138: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:7413 +//line mysql_sql.y:7426 { yyVAL.str = yyDollar[1].str } - case 1138: + case 1139: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:7417 +//line mysql_sql.y:7430 { yyVAL.str = yyVAL.str + yyDollar[2].str } - case 1139: + case 1140: yyDollar = yyS[yypt-3 : yypt+1] -//line mysql_sql.y:7423 +//line mysql_sql.y:7436 { yyVAL.str = "ALGORITHM = " + yyDollar[3].str } - case 1140: + case 1141: yyDollar = yyS[yypt-3 : yypt+1] -//line mysql_sql.y:7427 +//line mysql_sql.y:7440 { yyVAL.str = "DEFINER = " } - case 1141: + case 1142: yyDollar = yyS[yypt-3 : yypt+1] -//line mysql_sql.y:7431 +//line mysql_sql.y:7444 { yyVAL.str = "SQL SECURITY " + yyDollar[3].str } - case 1142: + case 1143: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:7436 +//line mysql_sql.y:7449 { yyVAL.str = "" } - case 1143: + case 1144: yyDollar = yyS[yypt-4 : yypt+1] -//line mysql_sql.y:7440 +//line mysql_sql.y:7453 { yyVAL.str = "WITH " + yyDollar[2].str + " CHECK OPTION" } - case 1149: + case 1150: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:7454 +//line mysql_sql.y:7467 { yyVAL.str = "" } - case 1152: + case 1153: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:7462 +//line mysql_sql.y:7475 { yyVAL.str = yyDollar[1].cstrUnion().Compare() } - case 1153: + case 1154: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:7468 +//line mysql_sql.y:7481 { var str = yyDollar[1].cstrUnion().Compare() yyLOCAL = tree.NewNumVal(str, str, false, tree.P_char) } yyVAL.union = yyLOCAL - case 1154: + case 1155: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:7473 +//line mysql_sql.y:7486 { yyLOCAL = tree.NewParamExpr(yylex.(*Lexer).GetParamIndex()) } yyVAL.union = yyLOCAL - case 1155: + case 1156: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.AccountAuthOption -//line mysql_sql.y:7479 +//line mysql_sql.y:7492 { var Equal = yyDollar[2].str var AdminName = yyDollar[3].exprUnion() @@ -21398,36 +21533,36 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1156: + case 1157: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:7492 +//line mysql_sql.y:7505 { var str = yyDollar[1].str yyLOCAL = tree.NewNumVal(str, str, false, tree.P_char) } yyVAL.union = yyLOCAL - case 1157: + case 1158: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:7497 +//line mysql_sql.y:7510 { var str = yyDollar[1].cstrUnion().Compare() yyLOCAL = tree.NewNumVal(str, str, false, tree.P_char) } yyVAL.union = yyLOCAL - case 1158: + case 1159: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:7502 +//line mysql_sql.y:7515 { yyLOCAL = tree.NewParamExpr(yylex.(*Lexer).GetParamIndex()) } yyVAL.union = yyLOCAL - case 1159: + case 1160: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.AccountIdentified -//line mysql_sql.y:7508 +//line mysql_sql.y:7521 { yyLOCAL = *tree.NewAccountIdentified( tree.AccountIdentifiedByPassword, @@ -21435,10 +21570,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1160: + case 1161: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.AccountIdentified -//line mysql_sql.y:7515 +//line mysql_sql.y:7528 { yyLOCAL = *tree.NewAccountIdentified( tree.AccountIdentifiedByPassword, @@ -21446,10 +21581,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1161: + case 1162: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.AccountIdentified -//line mysql_sql.y:7522 +//line mysql_sql.y:7535 { yyLOCAL = *tree.NewAccountIdentified( tree.AccountIdentifiedByRandomPassword, @@ -21457,10 +21592,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1162: + case 1163: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.AccountIdentified -//line mysql_sql.y:7529 +//line mysql_sql.y:7542 { yyLOCAL = *tree.NewAccountIdentified( tree.AccountIdentifiedWithSSL, @@ -21468,10 +21603,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1163: + case 1164: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.AccountIdentified -//line mysql_sql.y:7536 +//line mysql_sql.y:7549 { yyLOCAL = *tree.NewAccountIdentified( tree.AccountIdentifiedWithSSL, @@ -21479,20 +21614,20 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1164: + case 1165: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.AccountStatus -//line mysql_sql.y:7544 +//line mysql_sql.y:7557 { as := tree.NewAccountStatus() as.Exist = false yyLOCAL = *as } yyVAL.union = yyLOCAL - case 1165: + case 1166: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.AccountStatus -//line mysql_sql.y:7550 +//line mysql_sql.y:7563 { as := tree.NewAccountStatus() as.Exist = true @@ -21500,10 +21635,10 @@ yydefault: yyLOCAL = *as } yyVAL.union = yyLOCAL - case 1166: + case 1167: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.AccountStatus -//line mysql_sql.y:7557 +//line mysql_sql.y:7570 { as := tree.NewAccountStatus() as.Exist = true @@ -21511,10 +21646,10 @@ yydefault: yyLOCAL = *as } yyVAL.union = yyLOCAL - case 1167: + case 1168: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.AccountStatus -//line mysql_sql.y:7564 +//line mysql_sql.y:7577 { as := tree.NewAccountStatus() as.Exist = true @@ -21522,20 +21657,20 @@ yydefault: yyLOCAL = *as } yyVAL.union = yyLOCAL - case 1168: + case 1169: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.AccountComment -//line mysql_sql.y:7572 +//line mysql_sql.y:7585 { ac := tree.NewAccountComment() ac.Exist = false yyLOCAL = *ac } yyVAL.union = yyLOCAL - case 1169: + case 1170: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.AccountComment -//line mysql_sql.y:7578 +//line mysql_sql.y:7591 { ac := tree.NewAccountComment() ac.Exist = true @@ -21543,10 +21678,10 @@ yydefault: yyLOCAL = *ac } yyVAL.union = yyLOCAL - case 1170: + case 1171: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7587 +//line mysql_sql.y:7600 { var IfNotExists = yyDollar[3].ifNotExistsUnion() var Users = yyDollar[4].usersUnion() @@ -21562,10 +21697,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1171: + case 1172: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7604 +//line mysql_sql.y:7617 { var IfNotExists = yyDollar[3].ifNotExistsUnion() var Name = tree.Identifier(yyDollar[4].cstrUnion().Compare()) @@ -21582,10 +21717,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1172: + case 1173: yyDollar = yyS[yypt-10 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7620 +//line mysql_sql.y:7633 { var IfNotExists = yyDollar[3].ifNotExistsUnion() var Name = tree.Identifier(yyDollar[4].cstrUnion().Compare()) @@ -21603,10 +21738,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1173: + case 1174: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7637 +//line mysql_sql.y:7650 { var IfNotExists = yyDollar[3].ifNotExistsUnion() var Name = tree.Identifier(yyDollar[4].cstrUnion().Compare()) @@ -21623,30 +21758,30 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1174: + case 1175: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.AccountsSetOption -//line mysql_sql.y:7655 +//line mysql_sql.y:7668 { yyLOCAL = &tree.AccountsSetOption{ All: true, } } yyVAL.union = yyLOCAL - case 1175: + case 1176: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.AccountsSetOption -//line mysql_sql.y:7661 +//line mysql_sql.y:7674 { yyLOCAL = &tree.AccountsSetOption{ SetAccounts: yyDollar[2].identifierListUnion(), } } yyVAL.union = yyLOCAL - case 1176: + case 1177: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7669 +//line mysql_sql.y:7682 { var IfNotExists = yyDollar[3].ifNotExistsUnion() var Name = tree.Identifier(yyDollar[4].cstrUnion().Compare()) @@ -21664,20 +21799,20 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1177: + case 1178: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.StageStatus -//line mysql_sql.y:7687 +//line mysql_sql.y:7700 { yyLOCAL = tree.StageStatus{ Exist: false, } } yyVAL.union = yyLOCAL - case 1178: + case 1179: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.StageStatus -//line mysql_sql.y:7693 +//line mysql_sql.y:7706 { yyLOCAL = tree.StageStatus{ Exist: true, @@ -21685,10 +21820,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1179: + case 1180: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.StageStatus -//line mysql_sql.y:7700 +//line mysql_sql.y:7713 { yyLOCAL = tree.StageStatus{ Exist: true, @@ -21696,20 +21831,20 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1180: + case 1181: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.StageComment -//line mysql_sql.y:7708 +//line mysql_sql.y:7721 { yyLOCAL = tree.StageComment{ Exist: false, } } yyVAL.union = yyLOCAL - case 1181: + case 1182: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.StageComment -//line mysql_sql.y:7714 +//line mysql_sql.y:7727 { yyLOCAL = tree.StageComment{ Exist: true, @@ -21717,18 +21852,18 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1182: + case 1183: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL int64 -//line mysql_sql.y:7723 +//line mysql_sql.y:7736 { yyLOCAL = int64(0) } yyVAL.union = yyLOCAL - case 1183: + case 1184: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL int64 -//line mysql_sql.y:7727 +//line mysql_sql.y:7740 { switch v := yyDollar[3].item.(type) { case int64: @@ -21740,20 +21875,20 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1184: + case 1185: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.StageUrl -//line mysql_sql.y:7739 +//line mysql_sql.y:7752 { yyLOCAL = tree.StageUrl{ Exist: false, } } yyVAL.union = yyLOCAL - case 1185: + case 1186: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.StageUrl -//line mysql_sql.y:7745 +//line mysql_sql.y:7758 { yyLOCAL = tree.StageUrl{ Exist: true, @@ -21761,20 +21896,20 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1186: + case 1187: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.StageCredentials -//line mysql_sql.y:7753 +//line mysql_sql.y:7766 { yyLOCAL = tree.StageCredentials{ Exist: false, } } yyVAL.union = yyLOCAL - case 1187: + case 1188: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.StageCredentials -//line mysql_sql.y:7759 +//line mysql_sql.y:7772 { yyLOCAL = tree.StageCredentials{ Exist: true, @@ -21782,61 +21917,61 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1188: + case 1189: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:7768 +//line mysql_sql.y:7781 { yyLOCAL = yyDollar[1].strsUnion() } yyVAL.union = yyLOCAL - case 1189: + case 1190: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:7772 +//line mysql_sql.y:7785 { yyLOCAL = append(yyDollar[1].strsUnion(), yyDollar[3].strsUnion()...) } yyVAL.union = yyLOCAL - case 1190: + case 1191: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:7777 +//line mysql_sql.y:7790 { yyLOCAL = []string{} } yyVAL.union = yyLOCAL - case 1191: + case 1192: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:7781 +//line mysql_sql.y:7794 { yyLOCAL = append(yyLOCAL, yyDollar[1].str) yyLOCAL = append(yyLOCAL, yyDollar[3].str) } yyVAL.union = yyLOCAL - case 1192: + case 1193: yyDollar = yyS[yypt-3 : yypt+1] -//line mysql_sql.y:7788 +//line mysql_sql.y:7801 { yyVAL.str = yyDollar[3].str } - case 1193: + case 1194: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:7793 +//line mysql_sql.y:7806 { yyVAL.str = "" } - case 1194: + case 1195: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:7797 +//line mysql_sql.y:7810 { yyVAL.str = yyDollar[2].str } - case 1195: + case 1196: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7803 +//line mysql_sql.y:7816 { var ifNotExists = yyDollar[3].boolValUnion() var name = tree.Identifier(yyDollar[4].cstrUnion().Compare()) @@ -21847,10 +21982,10 @@ yydefault: yyLOCAL = tree.NewAlterStage(ifNotExists, name, urlOption, credentialsOption, statusOption, comment) } yyVAL.union = yyLOCAL - case 1196: + case 1197: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7815 +//line mysql_sql.y:7828 { var ifExists = yyDollar[3].boolValUnion() var name = tree.Identifier(yyDollar[4].cstrUnion().Compare()) @@ -21861,154 +21996,154 @@ yydefault: yyLOCAL = tree.NewAlterPublication(ifExists, name, accountsSet, dbName, table, comment) } yyVAL.union = yyLOCAL - case 1197: + case 1198: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.AccountsSetOption -//line mysql_sql.y:7826 +//line mysql_sql.y:7839 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1198: + case 1199: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.AccountsSetOption -//line mysql_sql.y:7830 +//line mysql_sql.y:7843 { yyLOCAL = &tree.AccountsSetOption{ All: true, } } yyVAL.union = yyLOCAL - case 1199: + case 1200: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.AccountsSetOption -//line mysql_sql.y:7836 +//line mysql_sql.y:7849 { yyLOCAL = &tree.AccountsSetOption{ SetAccounts: yyDollar[2].identifierListUnion(), } } yyVAL.union = yyLOCAL - case 1200: + case 1201: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.AccountsSetOption -//line mysql_sql.y:7842 +//line mysql_sql.y:7855 { yyLOCAL = &tree.AccountsSetOption{ AddAccounts: yyDollar[3].identifierListUnion(), } } yyVAL.union = yyLOCAL - case 1201: + case 1202: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.AccountsSetOption -//line mysql_sql.y:7848 +//line mysql_sql.y:7861 { yyLOCAL = &tree.AccountsSetOption{ DropAccounts: yyDollar[3].identifierListUnion(), } } yyVAL.union = yyLOCAL - case 1202: + case 1203: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:7855 +//line mysql_sql.y:7868 { yyVAL.str = "" } - case 1203: + case 1204: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:7859 +//line mysql_sql.y:7872 { yyVAL.str = yyDollar[2].str } - case 1204: + case 1205: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.TableNames -//line mysql_sql.y:7864 +//line mysql_sql.y:7877 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1205: + case 1206: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.TableNames -//line mysql_sql.y:7868 +//line mysql_sql.y:7881 { yyLOCAL = yyDollar[2].tableNamesUnion() } yyVAL.union = yyLOCAL - case 1206: + case 1207: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7874 +//line mysql_sql.y:7887 { var ifExists = yyDollar[3].boolValUnion() var name = tree.Identifier(yyDollar[4].cstrUnion().Compare()) yyLOCAL = tree.NewDropPublication(ifExists, name) } yyVAL.union = yyLOCAL - case 1207: + case 1208: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7882 +//line mysql_sql.y:7895 { var ifExists = yyDollar[4].boolValUnion() var taskID = yyDollar[5].str yyLOCAL = tree.NewDropCcprSubscription(ifExists, taskID) } yyVAL.union = yyLOCAL - case 1208: + case 1209: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7890 +//line mysql_sql.y:7903 { var taskID = yyDollar[4].str yyLOCAL = tree.NewResumeCcprSubscription(taskID) } yyVAL.union = yyLOCAL - case 1209: + case 1210: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7897 +//line mysql_sql.y:7910 { var taskID = yyDollar[4].str yyLOCAL = tree.NewPauseCcprSubscription(taskID) } yyVAL.union = yyLOCAL - case 1210: + case 1211: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7904 +//line mysql_sql.y:7917 { var ifNotExists = yyDollar[3].boolValUnion() var name = tree.Identifier(yyDollar[4].cstrUnion().Compare()) yyLOCAL = tree.NewDropStage(ifNotExists, name) } yyVAL.union = yyLOCAL - case 1211: + case 1212: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7912 +//line mysql_sql.y:7925 { var ifExists = yyDollar[5].boolValUnion() var path = yyDollar[6].str yyLOCAL = tree.NewRemoveStageFiles(ifExists, path) } yyVAL.union = yyLOCAL - case 1212: + case 1213: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7920 +//line mysql_sql.y:7933 { var ifExists = yyDollar[3].boolValUnion() var name = tree.Identifier(yyDollar[4].cstrUnion().Compare()) yyLOCAL = tree.NewDropSnapShot(ifExists, name, "", "") } yyVAL.union = yyLOCAL - case 1213: + case 1214: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7926 +//line mysql_sql.y:7939 { var ifExists = yyDollar[3].boolValUnion() var name = tree.Identifier(yyDollar[4].cstrUnion().Compare()) @@ -22017,10 +22152,10 @@ yydefault: yyLOCAL = tree.NewDropSnapShot(ifExists, name, accountName, pubName) } yyVAL.union = yyLOCAL - case 1214: + case 1215: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:7936 +//line mysql_sql.y:7949 { var ifExists = yyDollar[3].boolValUnion() var name = tree.Identifier(yyDollar[4].cstrUnion().Compare()) @@ -22032,16 +22167,16 @@ yydefault: } yyVAL.union = yyLOCAL - case 1215: + case 1216: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:7949 +//line mysql_sql.y:7962 { yyVAL.str = yyDollar[1].cstrUnion().Compare() } - case 1216: + case 1217: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.AccountCommentOrAttribute -//line mysql_sql.y:7954 +//line mysql_sql.y:7967 { var Exist = false var IsComment bool @@ -22054,10 +22189,10 @@ yydefault: } yyVAL.union = yyLOCAL - case 1217: + case 1218: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.AccountCommentOrAttribute -//line mysql_sql.y:7966 +//line mysql_sql.y:7979 { var Exist = true var IsComment = true @@ -22069,10 +22204,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1218: + case 1219: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.AccountCommentOrAttribute -//line mysql_sql.y:7977 +//line mysql_sql.y:7990 { var Exist = true var IsComment = false @@ -22084,26 +22219,26 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1219: + case 1220: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []*tree.User -//line mysql_sql.y:8085 +//line mysql_sql.y:8098 { yyLOCAL = []*tree.User{yyDollar[1].userUnion()} } yyVAL.union = yyLOCAL - case 1220: + case 1221: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []*tree.User -//line mysql_sql.y:8089 +//line mysql_sql.y:8102 { yyLOCAL = append(yyDollar[1].usersUnion(), yyDollar[3].userUnion()) } yyVAL.union = yyLOCAL - case 1221: + case 1222: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.User -//line mysql_sql.y:8095 +//line mysql_sql.y:8108 { var Username = yyDollar[1].usernameRecordUnion().Username var Hostname = yyDollar[1].usernameRecordUnion().Hostname @@ -22115,26 +22250,26 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1222: + case 1223: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []*tree.User -//line mysql_sql.y:8108 +//line mysql_sql.y:8121 { yyLOCAL = []*tree.User{yyDollar[1].userUnion()} } yyVAL.union = yyLOCAL - case 1223: + case 1224: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []*tree.User -//line mysql_sql.y:8112 +//line mysql_sql.y:8125 { yyLOCAL = append(yyDollar[1].usersUnion(), yyDollar[3].userUnion()) } yyVAL.union = yyLOCAL - case 1224: + case 1225: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.User -//line mysql_sql.y:8118 +//line mysql_sql.y:8131 { var Username = yyDollar[1].usernameRecordUnion().Username var Hostname = yyDollar[1].usernameRecordUnion().Hostname @@ -22146,50 +22281,50 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1225: + case 1226: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.UsernameRecord -//line mysql_sql.y:8131 +//line mysql_sql.y:8144 { yyLOCAL = &tree.UsernameRecord{Username: yyDollar[1].str, Hostname: "%"} } yyVAL.union = yyLOCAL - case 1226: + case 1227: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.UsernameRecord -//line mysql_sql.y:8135 +//line mysql_sql.y:8148 { yyLOCAL = &tree.UsernameRecord{Username: yyDollar[1].str, Hostname: yyDollar[3].str} } yyVAL.union = yyLOCAL - case 1227: + case 1228: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.UsernameRecord -//line mysql_sql.y:8139 +//line mysql_sql.y:8152 { yyLOCAL = &tree.UsernameRecord{Username: yyDollar[1].str, Hostname: yyDollar[2].str} } yyVAL.union = yyLOCAL - case 1228: + case 1229: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.AccountIdentified -//line mysql_sql.y:8144 +//line mysql_sql.y:8157 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1229: + case 1230: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.AccountIdentified -//line mysql_sql.y:8148 +//line mysql_sql.y:8161 { yyLOCAL = yyDollar[1].userIdentifiedUnion() } yyVAL.union = yyLOCAL - case 1230: + case 1231: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.AccountIdentified -//line mysql_sql.y:8154 +//line mysql_sql.y:8167 { yyLOCAL = &tree.AccountIdentified{ Typ: tree.AccountIdentifiedByPassword, @@ -22197,20 +22332,20 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1231: + case 1232: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.AccountIdentified -//line mysql_sql.y:8161 +//line mysql_sql.y:8174 { yyLOCAL = &tree.AccountIdentified{ Typ: tree.AccountIdentifiedByRandomPassword, } } yyVAL.union = yyLOCAL - case 1232: + case 1233: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.AccountIdentified -//line mysql_sql.y:8167 +//line mysql_sql.y:8180 { yyLOCAL = &tree.AccountIdentified{ Typ: tree.AccountIdentifiedWithSSL, @@ -22218,16 +22353,16 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1233: + case 1234: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:8176 +//line mysql_sql.y:8189 { yyVAL.str = yyDollar[1].cstrUnion().Compare() } - case 1235: + case 1236: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8183 +//line mysql_sql.y:8196 { var IfNotExists = yyDollar[3].ifNotExistsUnion() var Roles = yyDollar[4].rolesUnion() @@ -22237,26 +22372,26 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1236: + case 1237: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []*tree.Role -//line mysql_sql.y:8194 +//line mysql_sql.y:8207 { yyLOCAL = []*tree.Role{yyDollar[1].roleUnion()} } yyVAL.union = yyLOCAL - case 1237: + case 1238: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []*tree.Role -//line mysql_sql.y:8198 +//line mysql_sql.y:8211 { yyLOCAL = append(yyDollar[1].rolesUnion(), yyDollar[3].roleUnion()) } yyVAL.union = yyLOCAL - case 1238: + case 1239: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.Role -//line mysql_sql.y:8204 +//line mysql_sql.y:8217 { var UserName = yyDollar[1].cstrUnion().Compare() yyLOCAL = tree.NewRole( @@ -22264,106 +22399,106 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1239: + case 1240: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.CStr -//line mysql_sql.y:8213 +//line mysql_sql.y:8226 { yyLOCAL = tree.NewCStr(yyDollar[1].str, 1) } yyVAL.union = yyLOCAL - case 1240: + case 1241: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.CStr -//line mysql_sql.y:8217 +//line mysql_sql.y:8230 { yyLOCAL = tree.NewCStr(yyDollar[1].str, 1) } yyVAL.union = yyLOCAL - case 1241: + case 1242: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.CStr -//line mysql_sql.y:8221 +//line mysql_sql.y:8234 { yyLOCAL = tree.NewCStr(yyDollar[1].str, 1) } yyVAL.union = yyLOCAL - case 1242: + case 1243: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.CStr -//line mysql_sql.y:8225 +//line mysql_sql.y:8238 { yyLOCAL = tree.NewCStr("lag", 1) } yyVAL.union = yyLOCAL - case 1243: + case 1244: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.CStr -//line mysql_sql.y:8229 +//line mysql_sql.y:8242 { yyLOCAL = tree.NewCStr("lead", 1) } yyVAL.union = yyLOCAL - case 1244: + case 1245: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.CStr -//line mysql_sql.y:8233 +//line mysql_sql.y:8246 { yyLOCAL = tree.NewCStr("first_value", 1) } yyVAL.union = yyLOCAL - case 1245: + case 1246: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.CStr -//line mysql_sql.y:8237 +//line mysql_sql.y:8250 { yyLOCAL = tree.NewCStr("last_value", 1) } yyVAL.union = yyLOCAL - case 1246: + case 1247: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.CStr -//line mysql_sql.y:8241 +//line mysql_sql.y:8254 { yyLOCAL = tree.NewCStr("nth_value", 1) } yyVAL.union = yyLOCAL - case 1247: + case 1248: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.IndexCategory -//line mysql_sql.y:8246 +//line mysql_sql.y:8259 { yyLOCAL = tree.INDEX_CATEGORY_NONE } yyVAL.union = yyLOCAL - case 1248: + case 1249: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.IndexCategory -//line mysql_sql.y:8250 +//line mysql_sql.y:8263 { yyLOCAL = tree.INDEX_CATEGORY_FULLTEXT } yyVAL.union = yyLOCAL - case 1249: + case 1250: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.IndexCategory -//line mysql_sql.y:8254 +//line mysql_sql.y:8267 { yyLOCAL = tree.INDEX_CATEGORY_SPATIAL } yyVAL.union = yyLOCAL - case 1250: + case 1251: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.IndexCategory -//line mysql_sql.y:8258 +//line mysql_sql.y:8271 { yyLOCAL = tree.INDEX_CATEGORY_UNIQUE } yyVAL.union = yyLOCAL - case 1251: + case 1252: yyDollar = yyS[yypt-11 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8264 +//line mysql_sql.y:8277 { var io *tree.IndexOption = nil if yyDollar[11].indexOptionUnion() == nil && yyDollar[5].indexTypeUnion() != tree.INDEX_TYPE_INVALID { @@ -22394,18 +22529,18 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1252: + case 1253: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8295 +//line mysql_sql.y:8308 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1253: + case 1254: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8299 +//line mysql_sql.y:8312 { // Merge the options if yyDollar[1].indexOptionUnion() == nil { @@ -22435,12 +22570,16 @@ yydefault: opt1.Async = opt2.Async } else if opt2.ForceSync { opt1.ForceSync = opt2.ForceSync + } else if opt2.Merge { + opt1.Merge = opt2.Merge } else if opt2.AutoUpdate { opt1.AutoUpdate = opt2.AutoUpdate } else if opt2.Day > 0 { opt1.Day = opt2.Day } else if opt2.Hour > 0 { opt1.Hour = opt2.Hour + } else if opt2.Second > 0 { + opt1.Second = opt2.Second } else if opt2.IntermediateGraphDegree > 0 { opt1.IntermediateGraphDegree = opt2.IntermediateGraphDegree } else if opt2.GraphDegree > 0 { @@ -22468,20 +22607,20 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1254: + case 1255: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8363 +//line mysql_sql.y:8380 { io := tree.NewIndexOption() io.KeyBlockSize = uint64(yyDollar[3].item.(int64)) yyLOCAL = io } yyVAL.union = yyLOCAL - case 1255: + case 1256: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8369 +//line mysql_sql.y:8386 { val := int64(yyDollar[3].item.(int64)) if val <= 0 { @@ -22494,60 +22633,60 @@ yydefault: yyLOCAL = io } yyVAL.union = yyLOCAL - case 1256: + case 1257: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8381 +//line mysql_sql.y:8398 { io := tree.NewIndexOption() io.AlgoParamVectorOpType = yyDollar[2].str yyLOCAL = io } yyVAL.union = yyLOCAL - case 1257: + case 1258: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8387 +//line mysql_sql.y:8404 { io := tree.NewIndexOption() io.Comment = yyDollar[2].str yyLOCAL = io } yyVAL.union = yyLOCAL - case 1258: + case 1259: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8393 +//line mysql_sql.y:8410 { io := tree.NewIndexOption() io.ParserName = yyDollar[3].cstrUnion().Compare() yyLOCAL = io } yyVAL.union = yyLOCAL - case 1259: + case 1260: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8399 +//line mysql_sql.y:8416 { io := tree.NewIndexOption() io.Visible = tree.VISIBLE_TYPE_VISIBLE yyLOCAL = io } yyVAL.union = yyLOCAL - case 1260: + case 1261: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8405 +//line mysql_sql.y:8422 { io := tree.NewIndexOption() io.Visible = tree.VISIBLE_TYPE_INVISIBLE yyLOCAL = io } yyVAL.union = yyLOCAL - case 1261: + case 1262: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8411 +//line mysql_sql.y:8428 { val := int64(yyDollar[3].item.(int64)) if val <= 0 { @@ -22559,10 +22698,10 @@ yydefault: yyLOCAL = io } yyVAL.union = yyLOCAL - case 1262: + case 1263: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8422 +//line mysql_sql.y:8439 { val := int64(yyDollar[3].item.(int64)) if val <= 0 { @@ -22574,10 +22713,10 @@ yydefault: yyLOCAL = io } yyVAL.union = yyLOCAL - case 1263: + case 1264: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8433 +//line mysql_sql.y:8450 { val := int64(yyDollar[3].item.(int64)) if val <= 0 { @@ -22589,10 +22728,10 @@ yydefault: yyLOCAL = io } yyVAL.union = yyLOCAL - case 1264: + case 1265: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8444 +//line mysql_sql.y:8461 { val := int64(yyDollar[3].item.(int64)) if val <= 0 { @@ -22604,10 +22743,10 @@ yydefault: yyLOCAL = io } yyVAL.union = yyLOCAL - case 1265: + case 1266: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8455 +//line mysql_sql.y:8472 { val := int64(yyDollar[3].item.(int64)) if val <= 0 { @@ -22619,10 +22758,10 @@ yydefault: yyLOCAL = io } yyVAL.union = yyLOCAL - case 1266: + case 1267: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8466 +//line mysql_sql.y:8483 { val := int64(yyDollar[3].item.(int64)) if val <= 0 { @@ -22634,40 +22773,40 @@ yydefault: yyLOCAL = io } yyVAL.union = yyLOCAL - case 1267: + case 1268: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8477 +//line mysql_sql.y:8494 { io := tree.NewIndexOption() io.IncludeColumns = yyDollar[3].unresolveNamesUnion() yyLOCAL = io } yyVAL.union = yyLOCAL - case 1268: + case 1269: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8483 +//line mysql_sql.y:8500 { io := tree.NewIndexOption() io.Quantization = yyDollar[2].str yyLOCAL = io } yyVAL.union = yyLOCAL - case 1269: + case 1270: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8489 +//line mysql_sql.y:8506 { io := tree.NewIndexOption() io.DistributionMode = yyDollar[2].str yyLOCAL = io } yyVAL.union = yyLOCAL - case 1270: + case 1271: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8495 +//line mysql_sql.y:8512 { val := int64(yyDollar[3].item.(int64)) if val <= 0 { @@ -22679,10 +22818,10 @@ yydefault: yyLOCAL = io } yyVAL.union = yyLOCAL - case 1271: + case 1272: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8506 +//line mysql_sql.y:8523 { val := int64(yyDollar[3].item.(int64)) if val <= 0 { @@ -22694,10 +22833,10 @@ yydefault: yyLOCAL = io } yyVAL.union = yyLOCAL - case 1272: + case 1273: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8517 +//line mysql_sql.y:8534 { val := int64(yyDollar[3].item.(int64)) if val <= 0 { @@ -22709,10 +22848,10 @@ yydefault: yyLOCAL = io } yyVAL.union = yyLOCAL - case 1273: + case 1274: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8528 +//line mysql_sql.y:8545 { val := int64(yyDollar[3].item.(int64)) if val <= 0 { @@ -22724,50 +22863,60 @@ yydefault: yyLOCAL = io } yyVAL.union = yyLOCAL - case 1274: + case 1275: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8539 +//line mysql_sql.y:8556 { io := tree.NewIndexOption() io.Async = true yyLOCAL = io } yyVAL.union = yyLOCAL - case 1275: + case 1276: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8545 +//line mysql_sql.y:8562 { io := tree.NewIndexOption() io.ForceSync = true yyLOCAL = io } yyVAL.union = yyLOCAL - case 1276: + case 1277: + yyDollar = yyS[yypt-1 : yypt+1] + var yyLOCAL *tree.IndexOption +//line mysql_sql.y:8568 + { + io := tree.NewIndexOption() + io.Merge = true + yyLOCAL = io + } + yyVAL.union = yyLOCAL + case 1278: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8551 +//line mysql_sql.y:8574 { io := tree.NewIndexOption() io.AutoUpdate = true yyLOCAL = io } yyVAL.union = yyLOCAL - case 1277: + case 1279: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8557 +//line mysql_sql.y:8580 { io := tree.NewIndexOption() io.AutoUpdate = false yyLOCAL = io } yyVAL.union = yyLOCAL - case 1278: + case 1280: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8563 +//line mysql_sql.y:8586 { val := int64(yyDollar[3].item.(int64)) if val < 0 { @@ -22779,10 +22928,10 @@ yydefault: yyLOCAL = io } yyVAL.union = yyLOCAL - case 1279: + case 1281: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.IndexOption -//line mysql_sql.y:8574 +//line mysql_sql.y:8597 { val := int64(yyDollar[3].item.(int64)) if val < 0 || val > 23 { @@ -22794,26 +22943,41 @@ yydefault: yyLOCAL = io } yyVAL.union = yyLOCAL - case 1280: + case 1282: + yyDollar = yyS[yypt-3 : yypt+1] + var yyLOCAL *tree.IndexOption +//line mysql_sql.y:8608 + { + val := int64(yyDollar[3].item.(int64)) + if val < 0 { + yylex.Error("SECOND should be greater than or equal to 0") + return 1 + } + io := tree.NewIndexOption() + io.Second = val + yyLOCAL = io + } + yyVAL.union = yyLOCAL + case 1283: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []*tree.KeyPart -//line mysql_sql.y:8588 +//line mysql_sql.y:8622 { yyLOCAL = []*tree.KeyPart{yyDollar[1].keyPartUnion()} } yyVAL.union = yyLOCAL - case 1281: + case 1284: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []*tree.KeyPart -//line mysql_sql.y:8592 +//line mysql_sql.y:8626 { yyLOCAL = append(yyDollar[1].keyPartsUnion(), yyDollar[3].keyPartUnion()) } yyVAL.union = yyLOCAL - case 1282: + case 1285: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.KeyPart -//line mysql_sql.y:8598 +//line mysql_sql.y:8632 { // Order is parsed but just ignored as MySQL dtree. var ColName = yyDollar[1].unresolvedNameUnion() @@ -22828,10 +22992,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1283: + case 1286: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.KeyPart -//line mysql_sql.y:8612 +//line mysql_sql.y:8646 { var ColName *tree.UnresolvedName var Length int @@ -22845,90 +23009,98 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1284: + case 1287: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.IndexType -//line mysql_sql.y:8626 +//line mysql_sql.y:8660 { yyLOCAL = tree.INDEX_TYPE_INVALID } yyVAL.union = yyLOCAL - case 1285: + case 1288: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.IndexType -//line mysql_sql.y:8630 +//line mysql_sql.y:8664 { yyLOCAL = tree.INDEX_TYPE_BTREE } yyVAL.union = yyLOCAL - case 1286: + case 1289: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.IndexType -//line mysql_sql.y:8634 +//line mysql_sql.y:8668 { yyLOCAL = tree.INDEX_TYPE_IVFFLAT } yyVAL.union = yyLOCAL - case 1287: + case 1290: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.IndexType -//line mysql_sql.y:8638 +//line mysql_sql.y:8672 { yyLOCAL = tree.INDEX_TYPE_HNSW } yyVAL.union = yyLOCAL - case 1288: + case 1291: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.IndexType -//line mysql_sql.y:8642 +//line mysql_sql.y:8676 { yyLOCAL = tree.INDEX_TYPE_IVFPQ } yyVAL.union = yyLOCAL - case 1289: + case 1292: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.IndexType -//line mysql_sql.y:8646 +//line mysql_sql.y:8680 { yyLOCAL = tree.INDEX_TYPE_CAGRA } yyVAL.union = yyLOCAL - case 1290: + case 1293: + yyDollar = yyS[yypt-2 : yypt+1] + var yyLOCAL tree.IndexType +//line mysql_sql.y:8684 + { + yyLOCAL = tree.INDEX_TYPE_BM25 + } + yyVAL.union = yyLOCAL + case 1294: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.IndexType -//line mysql_sql.y:8650 +//line mysql_sql.y:8688 { yyLOCAL = tree.INDEX_TYPE_MASTER } yyVAL.union = yyLOCAL - case 1291: + case 1295: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.IndexType -//line mysql_sql.y:8654 +//line mysql_sql.y:8692 { yyLOCAL = tree.INDEX_TYPE_HASH } yyVAL.union = yyLOCAL - case 1292: + case 1296: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.IndexType -//line mysql_sql.y:8658 +//line mysql_sql.y:8696 { yyLOCAL = tree.INDEX_TYPE_RTREE } yyVAL.union = yyLOCAL - case 1293: + case 1297: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.IndexType -//line mysql_sql.y:8662 +//line mysql_sql.y:8700 { yyLOCAL = tree.INDEX_TYPE_BSI } yyVAL.union = yyLOCAL - case 1294: + case 1298: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8668 +//line mysql_sql.y:8706 { var IfNotExists = yyDollar[3].ifNotExistsUnion() var Name = tree.Identifier(yyDollar[4].str) @@ -22942,10 +23114,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1295: + case 1299: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8682 +//line mysql_sql.y:8720 { var t = tree.NewCloneDatabase() t.DstDatabase = tree.Identifier(yyDollar[4].str) @@ -22955,10 +23127,10 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1296: + case 1300: yyDollar = yyS[yypt-10 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8691 +//line mysql_sql.y:8729 { var DbName = tree.Identifier(yyDollar[4].str) var FromUri = yyDollar[6].str @@ -22976,92 +23148,92 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1297: + case 1301: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.SubscriptionOption -//line mysql_sql.y:8709 +//line mysql_sql.y:8747 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1298: + case 1302: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.SubscriptionOption -//line mysql_sql.y:8713 +//line mysql_sql.y:8751 { var From = tree.Identifier(yyDollar[2].str) var Publication = tree.Identifier(yyDollar[4].cstrUnion().Compare()) yyLOCAL = tree.NewSubscriptionOption(From, Publication) } yyVAL.union = yyLOCAL - case 1301: + case 1305: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:8724 +//line mysql_sql.y:8762 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 1302: + case 1306: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:8728 +//line mysql_sql.y:8766 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 1303: + case 1307: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:8733 +//line mysql_sql.y:8771 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 1304: + case 1308: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:8737 +//line mysql_sql.y:8775 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 1305: + case 1309: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL []tree.CreateOption -//line mysql_sql.y:8742 +//line mysql_sql.y:8780 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1306: + case 1310: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []tree.CreateOption -//line mysql_sql.y:8746 +//line mysql_sql.y:8784 { yyLOCAL = yyDollar[1].createOptionsUnion() } yyVAL.union = yyLOCAL - case 1307: + case 1311: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []tree.CreateOption -//line mysql_sql.y:8752 +//line mysql_sql.y:8790 { yyLOCAL = []tree.CreateOption{yyDollar[1].createOptionUnion()} } yyVAL.union = yyLOCAL - case 1308: + case 1312: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL []tree.CreateOption -//line mysql_sql.y:8756 +//line mysql_sql.y:8794 { yyLOCAL = append(yyDollar[1].createOptionsUnion(), yyDollar[2].createOptionUnion()) } yyVAL.union = yyLOCAL - case 1309: + case 1313: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.CreateOption -//line mysql_sql.y:8762 +//line mysql_sql.y:8800 { var IsDefault = yyDollar[1].defaultOptionalUnion() var Charset = yyDollar[4].str @@ -23071,10 +23243,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1310: + case 1314: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.CreateOption -//line mysql_sql.y:8771 +//line mysql_sql.y:8809 { var IsDefault = yyDollar[1].defaultOptionalUnion() var Collate = yyDollar[4].str @@ -23084,35 +23256,35 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1311: + case 1315: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.CreateOption -//line mysql_sql.y:8780 +//line mysql_sql.y:8818 { var Encrypt = yyDollar[4].str yyLOCAL = tree.NewCreateOptionEncryption(Encrypt) } yyVAL.union = yyLOCAL - case 1312: + case 1316: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:8786 +//line mysql_sql.y:8824 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 1313: + case 1317: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:8790 +//line mysql_sql.y:8828 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 1314: + case 1318: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8796 +//line mysql_sql.y:8834 { var TableName = yyDollar[4].tableNameUnion() var Options = yyDollar[7].connectorOptionsUnion() @@ -23122,18 +23294,18 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1315: + case 1319: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8807 +//line mysql_sql.y:8845 { yyLOCAL = &tree.ShowConnectors{} } yyVAL.union = yyLOCAL - case 1316: + case 1320: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8813 +//line mysql_sql.y:8851 { var taskID uint64 switch v := yyDollar[4].item.(type) { @@ -23150,10 +23322,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1317: + case 1321: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8831 +//line mysql_sql.y:8869 { var taskID uint64 switch v := yyDollar[4].item.(type) { @@ -23170,10 +23342,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1318: + case 1322: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8849 +//line mysql_sql.y:8887 { var taskID uint64 switch v := yyDollar[4].item.(type) { @@ -23190,10 +23362,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1319: + case 1323: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8867 +//line mysql_sql.y:8905 { var Replace = yyDollar[2].sourceOptionalUnion() var IfNotExists = yyDollar[4].ifNotExistsUnion() @@ -23209,26 +23381,26 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1320: + case 1324: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:8883 +//line mysql_sql.y:8921 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 1321: + case 1325: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:8887 +//line mysql_sql.y:8925 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 1322: + case 1326: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8893 +//line mysql_sql.y:8931 { t := tree.NewDataBranchCreateTable() t.CreateTable.Table = *yyDollar[5].tableNameUnion() @@ -23239,10 +23411,10 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1323: + case 1327: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8903 +//line mysql_sql.y:8941 { t := tree.NewDataBranchCreateDatabase() t.DstDatabase = tree.Identifier(yyDollar[5].str) @@ -23252,30 +23424,30 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1324: + case 1328: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8912 +//line mysql_sql.y:8950 { t := tree.NewDataBranchDeleteTable() t.TableName = *yyDollar[5].tableNameUnion() yyLOCAL = t } yyVAL.union = yyLOCAL - case 1325: + case 1329: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8918 +//line mysql_sql.y:8956 { t := tree.NewDataBranchDeleteDatabase() t.DatabaseName = tree.Identifier(yyDollar[5].str) yyLOCAL = t } yyVAL.union = yyLOCAL - case 1326: + case 1330: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8924 +//line mysql_sql.y:8962 { t := tree.NewDataBranchDiff() t.TargetTable = *yyDollar[4].tableNameUnion() @@ -23285,10 +23457,10 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1327: + case 1331: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8933 +//line mysql_sql.y:8971 { t := tree.NewDataBranchMerge() t.SrcTable = *yyDollar[4].tableNameUnion() @@ -23297,10 +23469,10 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1328: + case 1332: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8941 +//line mysql_sql.y:8979 { t := tree.NewDataBranchPick() t.SrcTable = *yyDollar[4].tableNameUnion() @@ -23310,10 +23482,10 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1329: + case 1333: yyDollar = yyS[yypt-12 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8950 +//line mysql_sql.y:8988 { t := tree.NewDataBranchPick() t.SrcTable = *yyDollar[4].tableNameUnion() @@ -23324,10 +23496,10 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1330: + case 1334: yyDollar = yyS[yypt-12 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8960 +//line mysql_sql.y:8998 { t := tree.NewDataBranchPick() t.SrcTable = *yyDollar[4].tableNameUnion() @@ -23338,10 +23510,10 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1331: + case 1335: yyDollar = yyS[yypt-13 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8970 +//line mysql_sql.y:9008 { t := tree.NewDataBranchPick() t.SrcTable = *yyDollar[4].tableNameUnion() @@ -23353,10 +23525,10 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1332: + case 1336: yyDollar = yyS[yypt-13 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:8981 +//line mysql_sql.y:9019 { t := tree.NewDataBranchPick() t.SrcTable = *yyDollar[4].tableNameUnion() @@ -23368,54 +23540,54 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1333: + case 1337: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.IdentifierList -//line mysql_sql.y:8993 +//line mysql_sql.y:9031 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1334: + case 1338: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.IdentifierList -//line mysql_sql.y:8997 +//line mysql_sql.y:9035 { yyLOCAL = yyDollar[3].identifierListUnion() } yyVAL.union = yyLOCAL - case 1335: + case 1339: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.DiffOutputOpt -//line mysql_sql.y:9002 +//line mysql_sql.y:9040 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1336: + case 1340: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.DiffOutputOpt -//line mysql_sql.y:9006 +//line mysql_sql.y:9044 { yyLOCAL = &tree.DiffOutputOpt{ As: *yyDollar[3].tableNameUnion(), } } yyVAL.union = yyLOCAL - case 1337: + case 1341: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.DiffOutputOpt -//line mysql_sql.y:9012 +//line mysql_sql.y:9050 { yyLOCAL = &tree.DiffOutputOpt{ DirPath: yyDollar[3].str, } } yyVAL.union = yyLOCAL - case 1338: + case 1342: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.DiffOutputOpt -//line mysql_sql.y:9018 +//line mysql_sql.y:9056 { x := yyDollar[3].item.(int64) yyLOCAL = &tree.DiffOutputOpt{ @@ -23423,68 +23595,68 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1339: + case 1343: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.DiffOutputOpt -//line mysql_sql.y:9025 +//line mysql_sql.y:9063 { yyLOCAL = &tree.DiffOutputOpt{ Count: true, } } yyVAL.union = yyLOCAL - case 1340: + case 1344: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.DiffOutputOpt -//line mysql_sql.y:9031 +//line mysql_sql.y:9069 { yyLOCAL = &tree.DiffOutputOpt{ Summary: true, } } yyVAL.union = yyLOCAL - case 1341: + case 1345: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.ConflictOpt -//line mysql_sql.y:9039 +//line mysql_sql.y:9077 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1342: + case 1346: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.ConflictOpt -//line mysql_sql.y:9043 +//line mysql_sql.y:9081 { yyLOCAL = &tree.ConflictOpt{ Opt: tree.CONFLICT_FAIL, } } yyVAL.union = yyLOCAL - case 1343: + case 1347: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.ConflictOpt -//line mysql_sql.y:9049 +//line mysql_sql.y:9087 { yyLOCAL = &tree.ConflictOpt{ Opt: tree.CONFLICT_SKIP, } } yyVAL.union = yyLOCAL - case 1344: + case 1348: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.ConflictOpt -//line mysql_sql.y:9055 +//line mysql_sql.y:9093 { yyLOCAL = &tree.ConflictOpt{ Opt: tree.CONFLICT_ACCEPT, } } yyVAL.union = yyLOCAL - case 1345: + case 1349: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.PickKeys -//line mysql_sql.y:9063 +//line mysql_sql.y:9101 { yyLOCAL = &tree.PickKeys{ Type: tree.PickKeysValues, @@ -23492,10 +23664,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1346: + case 1350: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.PickKeys -//line mysql_sql.y:9070 +//line mysql_sql.y:9108 { yyLOCAL = &tree.PickKeys{ Type: tree.PickKeysSubquery, @@ -23503,44 +23675,44 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1347: + case 1351: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.ToAccountOpt -//line mysql_sql.y:9079 +//line mysql_sql.y:9117 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1348: + case 1352: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.ToAccountOpt -//line mysql_sql.y:9083 +//line mysql_sql.y:9121 { yyLOCAL = &tree.ToAccountOpt{ AccountName: tree.Identifier(yyDollar[3].cstrUnion().Compare()), } } yyVAL.union = yyLOCAL - case 1349: + case 1353: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:9091 +//line mysql_sql.y:9129 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 1350: + case 1354: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:9095 +//line mysql_sql.y:9133 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 1351: + case 1355: yyDollar = yyS[yypt-11 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:9101 +//line mysql_sql.y:9139 { t := tree.NewCreateTable() t.Temporary = yyDollar[2].boolValUnion() @@ -23553,10 +23725,10 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1352: + case 1356: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:9113 +//line mysql_sql.y:9151 { t := tree.NewCreateTable() t.IfNotExists = yyDollar[4].ifNotExistsUnion() @@ -23566,10 +23738,10 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1353: + case 1357: yyDollar = yyS[yypt-11 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:9122 +//line mysql_sql.y:9160 { t := tree.NewCreateTable() t.IsClusterTable = true @@ -23582,10 +23754,10 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1354: + case 1358: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:9134 +//line mysql_sql.y:9172 { t := tree.NewCreateTable() t.IsDynamicTable = true @@ -23596,10 +23768,10 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1355: + case 1359: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:9144 +//line mysql_sql.y:9182 { t := tree.NewCreateTable() t.IsAsSelect = true @@ -23610,10 +23782,10 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1356: + case 1360: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:9154 +//line mysql_sql.y:9192 { t := tree.NewCreateTable() t.IsAsSelect = true @@ -23625,10 +23797,10 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1357: + case 1361: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:9165 +//line mysql_sql.y:9203 { t := tree.NewCreateTable() t.IsAsSelect = true @@ -23639,10 +23811,10 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1358: + case 1362: yyDollar = yyS[yypt-10 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:9175 +//line mysql_sql.y:9213 { t := tree.NewCreateTable() t.IsAsSelect = true @@ -23654,10 +23826,10 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1359: + case 1363: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:9186 +//line mysql_sql.y:9224 { t := tree.NewCreateTable() t.IsAsLike = true @@ -23668,10 +23840,10 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1360: + case 1364: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:9196 +//line mysql_sql.y:9234 { t := tree.NewCreateTable() t.Temporary = yyDollar[2].boolValUnion() @@ -23681,10 +23853,10 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1361: + case 1365: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:9205 +//line mysql_sql.y:9243 { t := tree.NewCloneTable() t.CreateTable.Temporary = yyDollar[2].boolValUnion() @@ -23698,10 +23870,10 @@ yydefault: yyLOCAL = t } yyVAL.union = yyLOCAL - case 1362: + case 1366: yyDollar = yyS[yypt-11 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:9218 +//line mysql_sql.y:9256 { var TableName = yyDollar[5].tableNameUnion() var FromUri = yyDollar[7].str @@ -23725,19 +23897,19 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1363: + case 1367: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.ExternParam -//line mysql_sql.y:9243 +//line mysql_sql.y:9281 { yyLOCAL = yyDollar[1].loadParamUnion() yyLOCAL.Tail = yyDollar[2].tailParamUnion() } yyVAL.union = yyLOCAL - case 1364: + case 1368: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.ExternParam -//line mysql_sql.y:9250 +//line mysql_sql.y:9288 { yyLOCAL = &tree.ExternParam{ ExParamConst: tree.ExParamConst{ @@ -23748,10 +23920,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1365: + case 1369: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL *tree.ExternParam -//line mysql_sql.y:9260 +//line mysql_sql.y:9298 { yyLOCAL = &tree.ExternParam{ ExParamConst: tree.ExParamConst{ @@ -23765,10 +23937,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1366: + case 1370: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.ExternParam -//line mysql_sql.y:9273 +//line mysql_sql.y:9311 { yyLOCAL = &tree.ExternParam{ ExParamConst: tree.ExParamConst{ @@ -23777,10 +23949,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1367: + case 1371: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.ExternParam -//line mysql_sql.y:9281 +//line mysql_sql.y:9319 { yyLOCAL = &tree.ExternParam{ ExParamConst: tree.ExParamConst{ @@ -23790,10 +23962,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1368: + case 1372: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.ExternParam -//line mysql_sql.y:9290 +//line mysql_sql.y:9328 { yyLOCAL = &tree.ExternParam{ ExParamConst: tree.ExParamConst{ @@ -23802,55 +23974,55 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1369: + case 1373: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:9299 +//line mysql_sql.y:9337 { yyVAL.str = "" } - case 1370: + case 1374: yyDollar = yyS[yypt-4 : yypt+1] -//line mysql_sql.y:9303 +//line mysql_sql.y:9341 { yyVAL.str = yyDollar[4].str } - case 1371: + case 1375: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:9309 +//line mysql_sql.y:9347 { yyLOCAL = yyDollar[1].strsUnion() } yyVAL.union = yyLOCAL - case 1372: + case 1376: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:9313 +//line mysql_sql.y:9351 { yyLOCAL = append(yyDollar[1].strsUnion(), yyDollar[3].strsUnion()...) } yyVAL.union = yyLOCAL - case 1373: + case 1377: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:9318 +//line mysql_sql.y:9356 { yyLOCAL = []string{} } yyVAL.union = yyLOCAL - case 1374: + case 1378: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:9322 +//line mysql_sql.y:9360 { yyLOCAL = append(yyLOCAL, yyDollar[1].str) yyLOCAL = append(yyLOCAL, yyDollar[3].str) } yyVAL.union = yyLOCAL - case 1375: + case 1379: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.TailParameter -//line mysql_sql.y:9329 +//line mysql_sql.y:9367 { yyLOCAL = &tree.TailParameter{ Charset: yyDollar[1].str, @@ -23862,22 +24034,22 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1376: + case 1380: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:9341 +//line mysql_sql.y:9379 { yyVAL.str = "" } - case 1377: + case 1381: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:9345 +//line mysql_sql.y:9383 { yyVAL.str = yyDollar[2].str } - case 1378: + case 1382: yyDollar = yyS[yypt-10 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:9351 +//line mysql_sql.y:9389 { var Name = yyDollar[4].tableNameUnion() var Type = yyDollar[5].columnTypeUnion() @@ -23899,10 +24071,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1379: + case 1383: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:9372 +//line mysql_sql.y:9410 { locale := "" fstr := "bigint" @@ -23917,44 +24089,44 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1380: + case 1384: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:9386 +//line mysql_sql.y:9424 { yyLOCAL = yyDollar[2].columnTypeUnion() } yyVAL.union = yyLOCAL - case 1381: + case 1385: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.TypeOption -//line mysql_sql.y:9390 +//line mysql_sql.y:9428 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1382: + case 1386: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.TypeOption -//line mysql_sql.y:9394 +//line mysql_sql.y:9432 { yyLOCAL = &tree.TypeOption{ Type: yyDollar[2].columnTypeUnion(), } } yyVAL.union = yyLOCAL - case 1383: + case 1387: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.IncrementByOption -//line mysql_sql.y:9400 +//line mysql_sql.y:9438 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1384: + case 1388: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.IncrementByOption -//line mysql_sql.y:9404 +//line mysql_sql.y:9442 { yyLOCAL = &tree.IncrementByOption{ Minus: false, @@ -23962,10 +24134,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1385: + case 1389: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.IncrementByOption -//line mysql_sql.y:9411 +//line mysql_sql.y:9449 { yyLOCAL = &tree.IncrementByOption{ Minus: false, @@ -23973,10 +24145,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1386: + case 1390: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.IncrementByOption -//line mysql_sql.y:9418 +//line mysql_sql.y:9456 { yyLOCAL = &tree.IncrementByOption{ Minus: true, @@ -23984,10 +24156,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1387: + case 1391: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.IncrementByOption -//line mysql_sql.y:9425 +//line mysql_sql.y:9463 { yyLOCAL = &tree.IncrementByOption{ Minus: true, @@ -23995,42 +24167,42 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1388: + case 1392: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:9432 +//line mysql_sql.y:9470 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 1389: + case 1393: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:9436 +//line mysql_sql.y:9474 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 1390: + case 1394: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:9440 +//line mysql_sql.y:9478 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 1391: + case 1395: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.MinValueOption -//line mysql_sql.y:9444 +//line mysql_sql.y:9482 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1392: + case 1396: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.MinValueOption -//line mysql_sql.y:9448 +//line mysql_sql.y:9486 { yyLOCAL = &tree.MinValueOption{ Minus: false, @@ -24038,10 +24210,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1393: + case 1397: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.MinValueOption -//line mysql_sql.y:9455 +//line mysql_sql.y:9493 { yyLOCAL = &tree.MinValueOption{ Minus: true, @@ -24049,18 +24221,18 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1394: + case 1398: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.MaxValueOption -//line mysql_sql.y:9462 +//line mysql_sql.y:9500 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1395: + case 1399: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.MaxValueOption -//line mysql_sql.y:9466 +//line mysql_sql.y:9504 { yyLOCAL = &tree.MaxValueOption{ Minus: false, @@ -24068,10 +24240,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1396: + case 1400: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.MaxValueOption -//line mysql_sql.y:9473 +//line mysql_sql.y:9511 { yyLOCAL = &tree.MaxValueOption{ Minus: true, @@ -24079,46 +24251,46 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1397: + case 1401: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.CycleOption -//line mysql_sql.y:9480 +//line mysql_sql.y:9518 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1398: + case 1402: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.CycleOption -//line mysql_sql.y:9484 +//line mysql_sql.y:9522 { yyLOCAL = &tree.CycleOption{ Cycle: false, } } yyVAL.union = yyLOCAL - case 1399: + case 1403: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.CycleOption -//line mysql_sql.y:9490 +//line mysql_sql.y:9528 { yyLOCAL = &tree.CycleOption{ Cycle: true, } } yyVAL.union = yyLOCAL - case 1400: + case 1404: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.StartWithOption -//line mysql_sql.y:9496 +//line mysql_sql.y:9534 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1401: + case 1405: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.StartWithOption -//line mysql_sql.y:9500 +//line mysql_sql.y:9538 { yyLOCAL = &tree.StartWithOption{ Minus: false, @@ -24126,10 +24298,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1402: + case 1406: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.StartWithOption -//line mysql_sql.y:9507 +//line mysql_sql.y:9545 { yyLOCAL = &tree.StartWithOption{ Minus: false, @@ -24137,10 +24309,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1403: + case 1407: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.StartWithOption -//line mysql_sql.y:9514 +//line mysql_sql.y:9552 { yyLOCAL = &tree.StartWithOption{ Minus: true, @@ -24148,10 +24320,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1404: + case 1408: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.StartWithOption -//line mysql_sql.y:9521 +//line mysql_sql.y:9559 { yyLOCAL = &tree.StartWithOption{ Minus: true, @@ -24159,58 +24331,58 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1405: + case 1409: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:9528 +//line mysql_sql.y:9566 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 1406: + case 1410: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:9532 +//line mysql_sql.y:9570 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 1407: + case 1411: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:9537 +//line mysql_sql.y:9575 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 1408: + case 1412: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:9541 +//line mysql_sql.y:9579 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 1409: + case 1413: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:9545 +//line mysql_sql.y:9583 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 1410: + case 1414: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.PartitionOption -//line mysql_sql.y:9550 +//line mysql_sql.y:9588 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1411: + case 1415: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.PartitionOption -//line mysql_sql.y:9554 +//line mysql_sql.y:9592 { yyDollar[3].partitionByUnion().Num = uint64(yyDollar[4].int64ValUnion()) var PartBy = yyDollar[3].partitionByUnion() @@ -24223,18 +24395,18 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1412: + case 1416: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.ClusterByOption -//line mysql_sql.y:9567 +//line mysql_sql.y:9605 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1413: + case 1417: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.ClusterByOption -//line mysql_sql.y:9571 +//line mysql_sql.y:9609 { var ColumnList = []*tree.UnresolvedName{yyDollar[3].unresolvedNameUnion()} yyLOCAL = tree.NewClusterByOption( @@ -24243,10 +24415,10 @@ yydefault: } yyVAL.union = yyLOCAL - case 1414: + case 1418: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.ClusterByOption -//line mysql_sql.y:9579 +//line mysql_sql.y:9617 { var ColumnList = yyDollar[4].unresolveNamesUnion() yyLOCAL = tree.NewClusterByOption( @@ -24254,18 +24426,18 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1415: + case 1419: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.PartitionBy -//line mysql_sql.y:9587 +//line mysql_sql.y:9625 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1416: + case 1420: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.PartitionBy -//line mysql_sql.y:9591 +//line mysql_sql.y:9629 { var IsSubPartition = true var PType = yyDollar[3].partitionByUnion().PType @@ -24279,42 +24451,42 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1417: + case 1421: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL []*tree.Partition -//line mysql_sql.y:9605 +//line mysql_sql.y:9643 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1418: + case 1422: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []*tree.Partition -//line mysql_sql.y:9609 +//line mysql_sql.y:9647 { yyLOCAL = yyDollar[2].partitionsUnion() } yyVAL.union = yyLOCAL - case 1419: + case 1423: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []*tree.Partition -//line mysql_sql.y:9615 +//line mysql_sql.y:9653 { yyLOCAL = []*tree.Partition{yyDollar[1].partitionUnion()} } yyVAL.union = yyLOCAL - case 1420: + case 1424: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []*tree.Partition -//line mysql_sql.y:9619 +//line mysql_sql.y:9657 { yyLOCAL = append(yyDollar[1].partitionsUnion(), yyDollar[3].partitionUnion()) } yyVAL.union = yyLOCAL - case 1421: + case 1425: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.Partition -//line mysql_sql.y:9625 +//line mysql_sql.y:9663 { var Name = tree.Identifier(yyDollar[2].cstrUnion().Compare()) var Values = yyDollar[3].valuesUnion() @@ -24328,10 +24500,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1422: + case 1426: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.Partition -//line mysql_sql.y:9638 +//line mysql_sql.y:9676 { var Name = tree.Identifier(yyDollar[2].cstrUnion().Compare()) var Values = yyDollar[3].valuesUnion() @@ -24345,42 +24517,42 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1423: + case 1427: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL []*tree.SubPartition -//line mysql_sql.y:9652 +//line mysql_sql.y:9690 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1424: + case 1428: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []*tree.SubPartition -//line mysql_sql.y:9656 +//line mysql_sql.y:9694 { yyLOCAL = yyDollar[2].subPartitionsUnion() } yyVAL.union = yyLOCAL - case 1425: + case 1429: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []*tree.SubPartition -//line mysql_sql.y:9662 +//line mysql_sql.y:9700 { yyLOCAL = []*tree.SubPartition{yyDollar[1].subPartitionUnion()} } yyVAL.union = yyLOCAL - case 1426: + case 1430: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []*tree.SubPartition -//line mysql_sql.y:9666 +//line mysql_sql.y:9704 { yyLOCAL = append(yyDollar[1].subPartitionsUnion(), yyDollar[3].subPartitionUnion()) } yyVAL.union = yyLOCAL - case 1427: + case 1431: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.SubPartition -//line mysql_sql.y:9672 +//line mysql_sql.y:9710 { var Name = tree.Identifier(yyDollar[2].cstrUnion().Compare()) var Options []tree.TableOption @@ -24390,10 +24562,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1428: + case 1432: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.SubPartition -//line mysql_sql.y:9681 +//line mysql_sql.y:9719 { var Name = tree.Identifier(yyDollar[2].cstrUnion().Compare()) var Options = yyDollar[3].tableOptionsUnion() @@ -24403,53 +24575,53 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1429: + case 1433: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []tree.TableOption -//line mysql_sql.y:9692 +//line mysql_sql.y:9730 { yyLOCAL = []tree.TableOption{yyDollar[1].tableOptionUnion()} } yyVAL.union = yyLOCAL - case 1430: + case 1434: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL []tree.TableOption -//line mysql_sql.y:9696 +//line mysql_sql.y:9734 { yyLOCAL = append(yyDollar[1].tableOptionsUnion(), yyDollar[2].tableOptionUnion()) } yyVAL.union = yyLOCAL - case 1431: + case 1435: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.Values -//line mysql_sql.y:9701 +//line mysql_sql.y:9739 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1432: + case 1436: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Values -//line mysql_sql.y:9705 +//line mysql_sql.y:9743 { expr := tree.NewMaxValue() var valueList = tree.Exprs{expr} yyLOCAL = tree.NewValuesLessThan(valueList) } yyVAL.union = yyLOCAL - case 1433: + case 1437: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL tree.Values -//line mysql_sql.y:9711 +//line mysql_sql.y:9749 { var valueList = yyDollar[5].exprsUnion() yyLOCAL = tree.NewValuesLessThan(valueList) } yyVAL.union = yyLOCAL - case 1434: + case 1438: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Values -//line mysql_sql.y:9716 +//line mysql_sql.y:9754 { var valueList = yyDollar[4].exprsUnion() yyLOCAL = tree.NewValuesIn( @@ -24457,18 +24629,18 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1435: + case 1439: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL int64 -//line mysql_sql.y:9724 +//line mysql_sql.y:9762 { yyLOCAL = 0 } yyVAL.union = yyLOCAL - case 1436: + case 1440: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL int64 -//line mysql_sql.y:9728 +//line mysql_sql.y:9766 { res := yyDollar[2].item.(int64) if res == 0 { @@ -24478,18 +24650,18 @@ yydefault: yyLOCAL = res } yyVAL.union = yyLOCAL - case 1437: + case 1441: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL int64 -//line mysql_sql.y:9738 +//line mysql_sql.y:9776 { yyLOCAL = 0 } yyVAL.union = yyLOCAL - case 1438: + case 1442: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL int64 -//line mysql_sql.y:9742 +//line mysql_sql.y:9780 { res := yyDollar[2].item.(int64) if res == 0 { @@ -24499,10 +24671,10 @@ yydefault: yyLOCAL = res } yyVAL.union = yyLOCAL - case 1439: + case 1443: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.PartitionBy -//line mysql_sql.y:9753 +//line mysql_sql.y:9791 { rangeTyp := tree.NewRangeType() rangeTyp.Expr = yyDollar[3].exprUnion() @@ -24511,10 +24683,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1440: + case 1444: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.PartitionBy -//line mysql_sql.y:9761 +//line mysql_sql.y:9799 { rangeTyp := tree.NewRangeType() rangeTyp.ColumnList = yyDollar[4].unresolveNamesUnion() @@ -24523,10 +24695,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1441: + case 1445: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.PartitionBy -//line mysql_sql.y:9769 +//line mysql_sql.y:9807 { listTyp := tree.NewListType() listTyp.Expr = yyDollar[3].exprUnion() @@ -24535,10 +24707,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1442: + case 1446: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.PartitionBy -//line mysql_sql.y:9777 +//line mysql_sql.y:9815 { listTyp := tree.NewListType() listTyp.ColumnList = yyDollar[4].unresolveNamesUnion() @@ -24547,10 +24719,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1444: + case 1448: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.PartitionBy -//line mysql_sql.y:9788 +//line mysql_sql.y:9826 { keyTyp := tree.NewKeyType() keyTyp.Linear = yyDollar[1].boolValUnion() @@ -24560,10 +24732,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1445: + case 1449: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.PartitionBy -//line mysql_sql.y:9797 +//line mysql_sql.y:9835 { keyTyp := tree.NewKeyType() keyTyp.Linear = yyDollar[1].boolValUnion() @@ -24574,10 +24746,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1446: + case 1450: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.PartitionBy -//line mysql_sql.y:9807 +//line mysql_sql.y:9845 { Linear := yyDollar[1].boolValUnion() Expr := yyDollar[4].exprUnion() @@ -24587,58 +24759,58 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1447: + case 1451: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL int64 -//line mysql_sql.y:9817 +//line mysql_sql.y:9855 { yyLOCAL = 2 } yyVAL.union = yyLOCAL - case 1448: + case 1452: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL int64 -//line mysql_sql.y:9821 +//line mysql_sql.y:9859 { yyLOCAL = yyDollar[3].item.(int64) } yyVAL.union = yyLOCAL - case 1449: + case 1453: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:9826 +//line mysql_sql.y:9864 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 1450: + case 1454: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:9830 +//line mysql_sql.y:9868 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 1451: + case 1455: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []*tree.ConnectorOption -//line mysql_sql.y:9836 +//line mysql_sql.y:9874 { yyLOCAL = []*tree.ConnectorOption{yyDollar[1].connectorOptionUnion()} } yyVAL.union = yyLOCAL - case 1452: + case 1456: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []*tree.ConnectorOption -//line mysql_sql.y:9840 +//line mysql_sql.y:9878 { yyLOCAL = append(yyDollar[1].connectorOptionsUnion(), yyDollar[3].connectorOptionUnion()) } yyVAL.union = yyLOCAL - case 1453: + case 1457: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.ConnectorOption -//line mysql_sql.y:9846 +//line mysql_sql.y:9884 { var Key = tree.Identifier(yyDollar[1].cstrUnion().Compare()) var Val = yyDollar[3].exprUnion() @@ -24648,10 +24820,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1454: + case 1458: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.ConnectorOption -//line mysql_sql.y:9855 +//line mysql_sql.y:9893 { var Key = tree.Identifier(yyDollar[1].str) var Val = yyDollar[3].exprUnion() @@ -24661,42 +24833,42 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1455: + case 1459: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL []tree.TableOption -//line mysql_sql.y:9865 +//line mysql_sql.y:9903 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1456: + case 1460: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL []tree.TableOption -//line mysql_sql.y:9869 +//line mysql_sql.y:9907 { yyLOCAL = yyDollar[3].tableOptionsUnion() } yyVAL.union = yyLOCAL - case 1457: + case 1461: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []tree.TableOption -//line mysql_sql.y:9875 +//line mysql_sql.y:9913 { yyLOCAL = []tree.TableOption{yyDollar[1].tableOptionUnion()} } yyVAL.union = yyLOCAL - case 1458: + case 1462: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []tree.TableOption -//line mysql_sql.y:9879 +//line mysql_sql.y:9917 { yyLOCAL = append(yyDollar[1].tableOptionsUnion(), yyDollar[3].tableOptionUnion()) } yyVAL.union = yyLOCAL - case 1459: + case 1463: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9885 +//line mysql_sql.y:9923 { var Key = tree.Identifier(yyDollar[1].cstrUnion().Compare()) var Val = yyDollar[3].exprUnion() @@ -24706,10 +24878,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1460: + case 1464: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9894 +//line mysql_sql.y:9932 { var Key = tree.Identifier(yyDollar[1].str) var Val = yyDollar[3].exprUnion() @@ -24719,364 +24891,364 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1461: + case 1465: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL []tree.TableOption -//line mysql_sql.y:9904 +//line mysql_sql.y:9942 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1462: + case 1466: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []tree.TableOption -//line mysql_sql.y:9908 +//line mysql_sql.y:9946 { yyLOCAL = yyDollar[1].tableOptionsUnion() } yyVAL.union = yyLOCAL - case 1463: + case 1467: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []tree.TableOption -//line mysql_sql.y:9914 +//line mysql_sql.y:9952 { yyLOCAL = []tree.TableOption{yyDollar[1].tableOptionUnion()} } yyVAL.union = yyLOCAL - case 1464: + case 1468: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []tree.TableOption -//line mysql_sql.y:9918 +//line mysql_sql.y:9956 { yyLOCAL = append(yyDollar[1].tableOptionsUnion(), yyDollar[3].tableOptionUnion()) } yyVAL.union = yyLOCAL - case 1465: + case 1469: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL []tree.TableOption -//line mysql_sql.y:9922 +//line mysql_sql.y:9960 { yyLOCAL = append(yyDollar[1].tableOptionsUnion(), yyDollar[2].tableOptionUnion()) } yyVAL.union = yyLOCAL - case 1466: + case 1470: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9928 +//line mysql_sql.y:9966 { yyLOCAL = tree.NewTableOptionAUTOEXTEND_SIZE(uint64(yyDollar[3].item.(int64))) } yyVAL.union = yyLOCAL - case 1467: + case 1471: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9932 +//line mysql_sql.y:9970 { yyLOCAL = tree.NewTableOptionAutoIncrement(uint64(yyDollar[3].item.(int64))) } yyVAL.union = yyLOCAL - case 1468: + case 1472: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9936 +//line mysql_sql.y:9974 { yyLOCAL = tree.NewTableOptionAvgRowLength(uint64(yyDollar[3].item.(int64))) } yyVAL.union = yyLOCAL - case 1469: + case 1473: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9940 +//line mysql_sql.y:9978 { yyLOCAL = tree.NewTableOptionCharset(yyDollar[4].str) } yyVAL.union = yyLOCAL - case 1470: + case 1474: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9944 +//line mysql_sql.y:9982 { yyLOCAL = tree.NewTableOptionCollate(yyDollar[4].str) } yyVAL.union = yyLOCAL - case 1471: + case 1475: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9948 +//line mysql_sql.y:9986 { yyLOCAL = tree.NewTableOptionChecksum(uint64(yyDollar[3].item.(int64))) } yyVAL.union = yyLOCAL - case 1472: + case 1476: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9952 +//line mysql_sql.y:9990 { str := util.DealCommentString(yyDollar[3].str) yyLOCAL = tree.NewTableOptionComment(str) } yyVAL.union = yyLOCAL - case 1473: + case 1477: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9957 +//line mysql_sql.y:9995 { yyLOCAL = tree.NewTableOptionCompression(yyDollar[3].str) } yyVAL.union = yyLOCAL - case 1474: + case 1478: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9961 +//line mysql_sql.y:9999 { yyLOCAL = tree.NewTableOptionConnection(yyDollar[3].str) } yyVAL.union = yyLOCAL - case 1475: + case 1479: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9965 +//line mysql_sql.y:10003 { yyLOCAL = tree.NewTableOptionDataDirectory(yyDollar[4].str) } yyVAL.union = yyLOCAL - case 1476: + case 1480: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9969 +//line mysql_sql.y:10007 { yyLOCAL = tree.NewTableOptionIndexDirectory(yyDollar[4].str) } yyVAL.union = yyLOCAL - case 1477: + case 1481: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9973 +//line mysql_sql.y:10011 { yyLOCAL = tree.NewTableOptionDelayKeyWrite(uint64(yyDollar[3].item.(int64))) } yyVAL.union = yyLOCAL - case 1478: + case 1482: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9977 +//line mysql_sql.y:10015 { yyLOCAL = tree.NewTableOptionEncryption(yyDollar[3].str) } yyVAL.union = yyLOCAL - case 1479: + case 1483: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9981 +//line mysql_sql.y:10019 { yyLOCAL = tree.NewTableOptionEngine(yyDollar[3].str) } yyVAL.union = yyLOCAL - case 1480: + case 1484: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9985 +//line mysql_sql.y:10023 { yyLOCAL = tree.NewTableOptionEngineAttr(yyDollar[3].str) } yyVAL.union = yyLOCAL - case 1481: + case 1485: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9989 +//line mysql_sql.y:10027 { yyLOCAL = tree.NewTableOptionInsertMethod(yyDollar[3].str) } yyVAL.union = yyLOCAL - case 1482: + case 1486: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9993 +//line mysql_sql.y:10031 { yyLOCAL = tree.NewTableOptionKeyBlockSize(uint64(yyDollar[3].item.(int64))) } yyVAL.union = yyLOCAL - case 1483: + case 1487: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:9997 +//line mysql_sql.y:10035 { yyLOCAL = tree.NewTableOptionMaxRows(uint64(yyDollar[3].item.(int64))) } yyVAL.union = yyLOCAL - case 1484: + case 1488: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:10001 +//line mysql_sql.y:10039 { yyLOCAL = tree.NewTableOptionMinRows(uint64(yyDollar[3].item.(int64))) } yyVAL.union = yyLOCAL - case 1485: + case 1489: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:10005 +//line mysql_sql.y:10043 { t := tree.NewTableOptionPackKeys() t.Value = yyDollar[3].item.(int64) yyLOCAL = t } yyVAL.union = yyLOCAL - case 1486: + case 1490: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:10011 +//line mysql_sql.y:10049 { t := tree.NewTableOptionPackKeys() t.Default = true yyLOCAL = t } yyVAL.union = yyLOCAL - case 1487: + case 1491: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:10017 +//line mysql_sql.y:10055 { yyLOCAL = tree.NewTableOptionPassword(yyDollar[3].str) } yyVAL.union = yyLOCAL - case 1488: + case 1492: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:10021 +//line mysql_sql.y:10059 { yyLOCAL = tree.NewTableOptionRowFormat(yyDollar[3].rowFormatTypeUnion()) } yyVAL.union = yyLOCAL - case 1489: + case 1493: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:10025 +//line mysql_sql.y:10063 { yyLOCAL = tree.NewTTableOptionStartTrans(true) } yyVAL.union = yyLOCAL - case 1490: + case 1494: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:10029 +//line mysql_sql.y:10067 { yyLOCAL = tree.NewTTableOptionSecondaryEngineAttr(yyDollar[3].str) } yyVAL.union = yyLOCAL - case 1491: + case 1495: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:10033 +//line mysql_sql.y:10071 { t := tree.NewTableOptionStatsAutoRecalc() t.Value = uint64(yyDollar[3].item.(int64)) yyLOCAL = t } yyVAL.union = yyLOCAL - case 1492: + case 1496: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:10039 +//line mysql_sql.y:10077 { t := tree.NewTableOptionStatsAutoRecalc() t.Default = true yyLOCAL = t } yyVAL.union = yyLOCAL - case 1493: + case 1497: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:10045 +//line mysql_sql.y:10083 { t := tree.NewTableOptionStatsPersistent() t.Value = uint64(yyDollar[3].item.(int64)) yyLOCAL = t } yyVAL.union = yyLOCAL - case 1494: + case 1498: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:10051 +//line mysql_sql.y:10089 { t := tree.NewTableOptionStatsPersistent() t.Default = true yyLOCAL = t } yyVAL.union = yyLOCAL - case 1495: + case 1499: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:10057 +//line mysql_sql.y:10095 { t := tree.NewTableOptionStatsSamplePages() t.Value = uint64(yyDollar[3].item.(int64)) yyLOCAL = t } yyVAL.union = yyLOCAL - case 1496: + case 1500: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:10063 +//line mysql_sql.y:10101 { t := tree.NewTableOptionStatsSamplePages() t.Default = true yyLOCAL = t } yyVAL.union = yyLOCAL - case 1497: + case 1501: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:10069 +//line mysql_sql.y:10107 { yyLOCAL = tree.NewTableOptionTablespace(yyDollar[3].cstrUnion().Compare(), "") } yyVAL.union = yyLOCAL - case 1498: + case 1502: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:10073 +//line mysql_sql.y:10111 { yyLOCAL = tree.NewTableOptionTablespace("", yyDollar[1].str) } yyVAL.union = yyLOCAL - case 1499: + case 1503: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:10077 +//line mysql_sql.y:10115 { yyLOCAL = tree.NewTableOptionUnion(yyDollar[4].tableNamesUnion()) } yyVAL.union = yyLOCAL - case 1500: + case 1504: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.TableOption -//line mysql_sql.y:10081 +//line mysql_sql.y:10119 { var Preperties = yyDollar[3].propertiesUnion() yyLOCAL = tree.NewTableOptionProperties(Preperties) } yyVAL.union = yyLOCAL - case 1501: + case 1505: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []tree.Property -//line mysql_sql.y:10088 +//line mysql_sql.y:10126 { yyLOCAL = []tree.Property{yyDollar[1].propertyUnion()} } yyVAL.union = yyLOCAL - case 1502: + case 1506: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []tree.Property -//line mysql_sql.y:10092 +//line mysql_sql.y:10130 { yyLOCAL = append(yyDollar[1].propertiesUnion(), yyDollar[3].propertyUnion()) } yyVAL.union = yyLOCAL - case 1503: + case 1507: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Property -//line mysql_sql.y:10098 +//line mysql_sql.y:10136 { var Key = yyDollar[1].str var Value = yyDollar[3].str @@ -25086,96 +25258,96 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1504: + case 1508: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:10109 +//line mysql_sql.y:10147 { yyVAL.str = " " + yyDollar[1].str + " " + yyDollar[2].str } - case 1505: + case 1509: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:10113 +//line mysql_sql.y:10151 { yyVAL.str = " " + yyDollar[1].str + " " + yyDollar[2].str } - case 1506: + case 1510: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.RowFormatType -//line mysql_sql.y:10119 +//line mysql_sql.y:10157 { yyLOCAL = tree.ROW_FORMAT_DEFAULT } yyVAL.union = yyLOCAL - case 1507: + case 1511: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.RowFormatType -//line mysql_sql.y:10123 +//line mysql_sql.y:10161 { yyLOCAL = tree.ROW_FORMAT_DYNAMIC } yyVAL.union = yyLOCAL - case 1508: + case 1512: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.RowFormatType -//line mysql_sql.y:10127 +//line mysql_sql.y:10165 { yyLOCAL = tree.ROW_FORMAT_FIXED } yyVAL.union = yyLOCAL - case 1509: + case 1513: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.RowFormatType -//line mysql_sql.y:10131 +//line mysql_sql.y:10169 { yyLOCAL = tree.ROW_FORMAT_COMPRESSED } yyVAL.union = yyLOCAL - case 1510: + case 1514: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.RowFormatType -//line mysql_sql.y:10135 +//line mysql_sql.y:10173 { yyLOCAL = tree.ROW_FORMAT_REDUNDANT } yyVAL.union = yyLOCAL - case 1511: + case 1515: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.RowFormatType -//line mysql_sql.y:10139 +//line mysql_sql.y:10177 { yyLOCAL = tree.ROW_FORMAT_COMPACT } yyVAL.union = yyLOCAL - case 1516: + case 1520: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.TableNames -//line mysql_sql.y:10153 +//line mysql_sql.y:10191 { yyLOCAL = tree.TableNames{yyDollar[1].tableNameUnion()} } yyVAL.union = yyLOCAL - case 1517: + case 1521: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableNames -//line mysql_sql.y:10157 +//line mysql_sql.y:10195 { yyLOCAL = append(yyDollar[1].tableNamesUnion(), yyDollar[3].tableNameUnion()) } yyVAL.union = yyLOCAL - case 1518: + case 1522: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.TableName -//line mysql_sql.y:10166 +//line mysql_sql.y:10204 { tblName := yylex.(*Lexer).GetDbOrTblName(yyDollar[1].cstrUnion().Origin()) prefix := tree.ObjectNamePrefix{ExplicitSchema: false} yyLOCAL = tree.NewTableName(tree.Identifier(tblName), prefix, yyDollar[2].atTimeStampUnion()) } yyVAL.union = yyLOCAL - case 1519: + case 1523: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.TableName -//line mysql_sql.y:10172 +//line mysql_sql.y:10210 { dbName := yylex.(*Lexer).GetDbOrTblName(yyDollar[1].cstrUnion().Origin()) tblName := yylex.(*Lexer).GetDbOrTblName(yyDollar[3].cstrUnion().Origin()) @@ -25183,18 +25355,18 @@ yydefault: yyLOCAL = tree.NewTableName(tree.Identifier(tblName), prefix, yyDollar[4].atTimeStampUnion()) } yyVAL.union = yyLOCAL - case 1520: + case 1524: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.AtTimeStamp -//line mysql_sql.y:10179 +//line mysql_sql.y:10217 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1521: + case 1525: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.AtTimeStamp -//line mysql_sql.y:10183 +//line mysql_sql.y:10221 { yyLOCAL = &tree.AtTimeStamp{ Type: tree.ATTIMESTAMPTIME, @@ -25202,10 +25374,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1522: + case 1526: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.AtTimeStamp -//line mysql_sql.y:10190 +//line mysql_sql.y:10228 { var str = yyDollar[4].cstrUnion().Compare() yyLOCAL = &tree.AtTimeStamp{ @@ -25215,10 +25387,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1523: + case 1527: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.AtTimeStamp -//line mysql_sql.y:10199 +//line mysql_sql.y:10237 { yyLOCAL = &tree.AtTimeStamp{ Type: tree.ATTIMESTAMPSNAPSHOT, @@ -25227,10 +25399,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1524: + case 1528: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.AtTimeStamp -//line mysql_sql.y:10207 +//line mysql_sql.y:10245 { yyLOCAL = &tree.AtTimeStamp{ Type: tree.ATMOTIMESTAMP, @@ -25238,10 +25410,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1525: + case 1529: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.AtTimeStamp -//line mysql_sql.y:10214 +//line mysql_sql.y:10252 { yyLOCAL = &tree.AtTimeStamp{ Type: tree.ASOFTIMESTAMP, @@ -25249,74 +25421,74 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1526: + case 1530: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.TableDefs -//line mysql_sql.y:10222 +//line mysql_sql.y:10260 { yyLOCAL = tree.TableDefs(nil) } yyVAL.union = yyLOCAL - case 1528: + case 1532: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.TableDefs -//line mysql_sql.y:10229 +//line mysql_sql.y:10267 { yyLOCAL = tree.TableDefs{yyDollar[1].tableDefUnion()} } yyVAL.union = yyLOCAL - case 1529: + case 1533: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.TableDefs -//line mysql_sql.y:10233 +//line mysql_sql.y:10271 { yyLOCAL = append(yyDollar[1].tableDefsUnion(), yyDollar[3].tableDefUnion()) } yyVAL.union = yyLOCAL - case 1530: + case 1534: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.TableDef -//line mysql_sql.y:10239 +//line mysql_sql.y:10277 { yyLOCAL = tree.TableDef(yyDollar[1].columnTableDefUnion()) } yyVAL.union = yyLOCAL - case 1531: + case 1535: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.TableDef -//line mysql_sql.y:10243 +//line mysql_sql.y:10281 { yyLOCAL = yyDollar[1].tableDefUnion() } yyVAL.union = yyLOCAL - case 1532: + case 1536: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.TableDef -//line mysql_sql.y:10247 +//line mysql_sql.y:10285 { yyLOCAL = yyDollar[1].tableDefUnion() } yyVAL.union = yyLOCAL - case 1533: + case 1537: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.TableDef -//line mysql_sql.y:10253 +//line mysql_sql.y:10291 { yyLOCAL = yyDollar[1].tableDefUnion() } yyVAL.union = yyLOCAL - case 1534: + case 1538: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.TableDef -//line mysql_sql.y:10257 +//line mysql_sql.y:10295 { yyLOCAL = yyDollar[1].tableDefUnion() } yyVAL.union = yyLOCAL - case 1535: + case 1539: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.TableDef -//line mysql_sql.y:10263 +//line mysql_sql.y:10301 { var KeyParts = yyDollar[5].keyPartsUnion() var Name = yyDollar[3].str @@ -25330,10 +25502,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1536: + case 1540: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL tree.TableDef -//line mysql_sql.y:10276 +//line mysql_sql.y:10314 { var KeyParts = yyDollar[5].keyPartsUnion() var Name = yyDollar[3].str @@ -25347,10 +25519,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1537: + case 1541: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.TableDef -//line mysql_sql.y:10289 +//line mysql_sql.y:10327 { keyTyp := tree.INDEX_TYPE_INVALID if yyDollar[3].strsUnion()[1] != "" { @@ -25376,6 +25548,8 @@ yydefault: keyTyp = tree.INDEX_TYPE_CAGRA case "ivfpq": keyTyp = tree.INDEX_TYPE_IVFPQ + case "bm25": + keyTyp = tree.INDEX_TYPE_BM25 default: yylex.Error("Invalid the type of index") goto ret1 @@ -25396,10 +25570,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1538: + case 1542: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL tree.TableDef -//line mysql_sql.y:10334 +//line mysql_sql.y:10374 { keyTyp := tree.INDEX_TYPE_INVALID if yyDollar[3].strsUnion()[1] != "" { @@ -25444,10 +25618,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1539: + case 1543: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.TableDef -//line mysql_sql.y:10380 +//line mysql_sql.y:10420 { if yyDollar[1].str != "" { switch v := yyDollar[2].tableDefUnion().(type) { @@ -25462,18 +25636,18 @@ yydefault: yyLOCAL = yyDollar[2].tableDefUnion() } yyVAL.union = yyLOCAL - case 1540: + case 1544: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.TableDef -//line mysql_sql.y:10394 +//line mysql_sql.y:10434 { yyLOCAL = yyDollar[1].tableDefUnion() } yyVAL.union = yyLOCAL - case 1541: + case 1545: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.TableDef -//line mysql_sql.y:10400 +//line mysql_sql.y:10440 { var KeyParts = yyDollar[5].keyPartsUnion() var Name = yyDollar[3].strsUnion()[0] @@ -25487,10 +25661,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1542: + case 1546: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL tree.TableDef -//line mysql_sql.y:10413 +//line mysql_sql.y:10453 { var KeyParts = yyDollar[5].keyPartsUnion() var Name = yyDollar[3].strsUnion()[0] @@ -25504,10 +25678,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1543: + case 1547: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.TableDef -//line mysql_sql.y:10426 +//line mysql_sql.y:10466 { var KeyParts = yyDollar[5].keyPartsUnion() var Name = yyDollar[3].strsUnion()[0] @@ -25521,10 +25695,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1544: + case 1548: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL tree.TableDef -//line mysql_sql.y:10439 +//line mysql_sql.y:10479 { var KeyParts = yyDollar[5].keyPartsUnion() var Name = yyDollar[3].strsUnion()[0] @@ -25538,10 +25712,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1545: + case 1549: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL tree.TableDef -//line mysql_sql.y:10452 +//line mysql_sql.y:10492 { var IfNotExists = yyDollar[3].ifNotExistsUnion() var KeyParts = yyDollar[6].keyPartsUnion() @@ -25557,10 +25731,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1546: + case 1550: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.TableDef -//line mysql_sql.y:10467 +//line mysql_sql.y:10507 { var Expr = yyDollar[3].exprUnion() var Enforced = yyDollar[5].boolValUnion() @@ -25570,327 +25744,327 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1547: + case 1551: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:10477 +//line mysql_sql.y:10517 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 1549: + case 1553: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:10483 +//line mysql_sql.y:10523 { yyVAL.str = "" } - case 1550: + case 1554: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:10487 +//line mysql_sql.y:10527 { yyVAL.str = yyDollar[1].str } - case 1553: + case 1557: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:10497 +//line mysql_sql.y:10537 { yyLOCAL = make([]string, 2) yyLOCAL[0] = yyDollar[1].str yyLOCAL[1] = "" } yyVAL.union = yyLOCAL - case 1554: + case 1558: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:10503 +//line mysql_sql.y:10543 { yyLOCAL = make([]string, 2) yyLOCAL[0] = yyDollar[1].str yyLOCAL[1] = yyDollar[3].str } yyVAL.union = yyLOCAL - case 1555: + case 1559: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:10509 +//line mysql_sql.y:10549 { yyLOCAL = make([]string, 2) yyLOCAL[0] = yyDollar[1].cstrUnion().Compare() yyLOCAL[1] = yyDollar[3].str } yyVAL.union = yyLOCAL - case 1569: + case 1574: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:10533 +//line mysql_sql.y:10574 { yyVAL.str = "" } - case 1570: + case 1575: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:10537 +//line mysql_sql.y:10578 { yyVAL.str = yyDollar[1].cstrUnion().Compare() } - case 1571: + case 1576: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.ColumnTableDef -//line mysql_sql.y:10543 +//line mysql_sql.y:10584 { yyLOCAL = tree.NewColumnTableDef(yyDollar[1].unresolvedNameUnion(), yyDollar[2].columnTypeUnion(), yyDollar[3].columnAttributesUnion()) } yyVAL.union = yyLOCAL - case 1572: + case 1577: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.UnresolvedName -//line mysql_sql.y:10549 +//line mysql_sql.y:10590 { yyLOCAL = tree.NewUnresolvedName(yyDollar[1].cstrUnion()) } yyVAL.union = yyLOCAL - case 1573: + case 1578: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.UnresolvedName -//line mysql_sql.y:10553 +//line mysql_sql.y:10594 { tblNameCStr := yylex.(*Lexer).GetDbOrTblNameCStr(yyDollar[1].cstrUnion().Origin()) yyLOCAL = tree.NewUnresolvedName(tblNameCStr, yyDollar[3].cstrUnion()) } yyVAL.union = yyLOCAL - case 1574: + case 1579: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.UnresolvedName -//line mysql_sql.y:10558 +//line mysql_sql.y:10599 { dbNameCStr := yylex.(*Lexer).GetDbOrTblNameCStr(yyDollar[1].cstrUnion().Origin()) tblNameCStr := yylex.(*Lexer).GetDbOrTblNameCStr(yyDollar[3].cstrUnion().Origin()) yyLOCAL = tree.NewUnresolvedName(dbNameCStr, tblNameCStr, yyDollar[5].cstrUnion()) } yyVAL.union = yyLOCAL - case 1575: + case 1580: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.CStr -//line mysql_sql.y:10566 +//line mysql_sql.y:10607 { yyLOCAL = tree.NewCStr(yyDollar[1].str, 1) } yyVAL.union = yyLOCAL - case 1576: + case 1581: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.CStr -//line mysql_sql.y:10570 +//line mysql_sql.y:10611 { yyLOCAL = tree.NewCStr(yyDollar[1].str, 1) } yyVAL.union = yyLOCAL - case 1577: + case 1582: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.CStr -//line mysql_sql.y:10574 +//line mysql_sql.y:10615 { yyLOCAL = tree.NewCStr(yyDollar[1].str, 1) } yyVAL.union = yyLOCAL - case 1578: + case 1583: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.CStr -//line mysql_sql.y:10578 +//line mysql_sql.y:10619 { yyLOCAL = tree.NewCStr(yyDollar[1].str, 1) } yyVAL.union = yyLOCAL - case 1579: + case 1584: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.CStr -//line mysql_sql.y:10584 +//line mysql_sql.y:10625 { yyLOCAL = yylex.(*Lexer).GetDbOrTblNameCStr(yyDollar[1].cstrUnion().Origin()) } yyVAL.union = yyLOCAL - case 1580: + case 1585: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.UnresolvedName -//line mysql_sql.y:10590 +//line mysql_sql.y:10631 { yyLOCAL = tree.NewUnresolvedName(yyDollar[1].cstrUnion()) } yyVAL.union = yyLOCAL - case 1581: + case 1586: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.UnresolvedName -//line mysql_sql.y:10594 +//line mysql_sql.y:10635 { tblNameCStr := yylex.(*Lexer).GetDbOrTblNameCStr(yyDollar[1].cstrUnion().Origin()) yyLOCAL = tree.NewUnresolvedName(tblNameCStr, yyDollar[3].cstrUnion()) } yyVAL.union = yyLOCAL - case 1582: + case 1587: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.UnresolvedName -//line mysql_sql.y:10599 +//line mysql_sql.y:10640 { dbNameCStr := yylex.(*Lexer).GetDbOrTblNameCStr(yyDollar[1].cstrUnion().Origin()) tblNameCStr := yylex.(*Lexer).GetDbOrTblNameCStr(yyDollar[3].cstrUnion().Origin()) yyLOCAL = tree.NewUnresolvedName(dbNameCStr, tblNameCStr, yyDollar[5].cstrUnion()) } yyVAL.union = yyLOCAL - case 1583: + case 1588: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL []tree.ColumnAttribute -//line mysql_sql.y:10606 +//line mysql_sql.y:10647 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1584: + case 1589: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []tree.ColumnAttribute -//line mysql_sql.y:10610 +//line mysql_sql.y:10651 { yyLOCAL = yyDollar[1].columnAttributesUnion() } yyVAL.union = yyLOCAL - case 1585: + case 1590: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []tree.ColumnAttribute -//line mysql_sql.y:10616 +//line mysql_sql.y:10657 { yyLOCAL = []tree.ColumnAttribute{yyDollar[1].columnAttributeUnion()} } yyVAL.union = yyLOCAL - case 1586: + case 1591: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL []tree.ColumnAttribute -//line mysql_sql.y:10620 +//line mysql_sql.y:10661 { yyLOCAL = append(yyDollar[1].columnAttributesUnion(), yyDollar[2].columnAttributeUnion()) } yyVAL.union = yyLOCAL - case 1587: + case 1592: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10626 +//line mysql_sql.y:10667 { yyLOCAL = tree.NewAttributeNull(true) } yyVAL.union = yyLOCAL - case 1588: + case 1593: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10630 +//line mysql_sql.y:10671 { yyLOCAL = tree.NewAttributeNull(false) } yyVAL.union = yyLOCAL - case 1589: + case 1594: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10634 +//line mysql_sql.y:10675 { yyLOCAL = tree.NewAttributeDefault(yyDollar[2].exprUnion()) } yyVAL.union = yyLOCAL - case 1590: + case 1595: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10638 +//line mysql_sql.y:10679 { yyLOCAL = tree.NewAttributeAutoIncrement() } yyVAL.union = yyLOCAL - case 1591: + case 1596: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10642 +//line mysql_sql.y:10683 { yyLOCAL = yyDollar[1].columnAttributeUnion() } yyVAL.union = yyLOCAL - case 1592: + case 1597: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10646 +//line mysql_sql.y:10687 { str := util.DealCommentString(yyDollar[2].str) yyLOCAL = tree.NewAttributeComment(tree.NewNumVal(str, str, false, tree.P_char)) } yyVAL.union = yyLOCAL - case 1593: + case 1598: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10651 +//line mysql_sql.y:10692 { yyLOCAL = tree.NewAttributeCollate(yyDollar[2].str) } yyVAL.union = yyLOCAL - case 1594: + case 1599: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10655 +//line mysql_sql.y:10696 { yyLOCAL = tree.NewAttributeColumnFormat(yyDollar[2].str) } yyVAL.union = yyLOCAL - case 1595: + case 1600: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10659 +//line mysql_sql.y:10700 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1596: + case 1601: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10663 +//line mysql_sql.y:10704 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1597: + case 1602: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10667 +//line mysql_sql.y:10708 { yyLOCAL = tree.NewAttributeStorage(yyDollar[2].str) } yyVAL.union = yyLOCAL - case 1598: + case 1603: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10671 +//line mysql_sql.y:10712 { yyLOCAL = tree.NewAttributeAutoRandom(int(yyDollar[2].int64ValUnion())) } yyVAL.union = yyLOCAL - case 1599: + case 1604: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10675 +//line mysql_sql.y:10716 { yyLOCAL = yyDollar[1].attributeReferenceUnion() } yyVAL.union = yyLOCAL - case 1600: + case 1605: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10679 +//line mysql_sql.y:10720 { yyLOCAL = tree.NewAttributeCheckConstraint(yyDollar[4].exprUnion(), false, yyDollar[1].str) } yyVAL.union = yyLOCAL - case 1601: + case 1606: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10683 +//line mysql_sql.y:10724 { yyLOCAL = tree.NewAttributeCheckConstraint(yyDollar[4].exprUnion(), yyDollar[6].boolValUnion(), yyDollar[1].str) } yyVAL.union = yyLOCAL - case 1602: + case 1607: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10687 +//line mysql_sql.y:10728 { name := tree.NewUnresolvedColName(yyDollar[3].str) var es tree.Exprs = nil @@ -25905,10 +26079,10 @@ yydefault: yyLOCAL = tree.NewAttributeOnUpdate(expr) } yyVAL.union = yyLOCAL - case 1603: + case 1608: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10701 +//line mysql_sql.y:10742 { v, errStr := util.GetInt64(yyDollar[2].item) if errStr != "" { @@ -25922,138 +26096,138 @@ yydefault: yyLOCAL = tree.NewAttributeSRID(uint32(v)) } yyVAL.union = yyLOCAL - case 1604: + case 1609: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10714 +//line mysql_sql.y:10755 { yyLOCAL = tree.NewAttributeLowCardinality() } yyVAL.union = yyLOCAL - case 1605: + case 1610: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10718 +//line mysql_sql.y:10759 { yyLOCAL = tree.NewAttributeVisable(true) } yyVAL.union = yyLOCAL - case 1606: + case 1611: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10722 +//line mysql_sql.y:10763 { yyLOCAL = tree.NewAttributeVisable(false) } yyVAL.union = yyLOCAL - case 1607: + case 1612: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10726 +//line mysql_sql.y:10767 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1608: + case 1613: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10730 +//line mysql_sql.y:10771 { yyLOCAL = tree.NewAttributeHeader(yyDollar[3].str) } yyVAL.union = yyLOCAL - case 1609: + case 1614: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10734 +//line mysql_sql.y:10775 { yyLOCAL = tree.NewAttributeHeaders() } yyVAL.union = yyLOCAL - case 1610: + case 1615: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10738 +//line mysql_sql.y:10779 { yyLOCAL = tree.NewAttributeGeneratedAlways(yyDollar[5].exprUnion(), yyDollar[7].boolValUnion()) } yyVAL.union = yyLOCAL - case 1611: + case 1616: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:10742 +//line mysql_sql.y:10783 { yyLOCAL = tree.NewAttributeGeneratedAlways(yyDollar[3].exprUnion(), yyDollar[5].boolValUnion()) } yyVAL.union = yyLOCAL - case 1612: + case 1617: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:10747 +//line mysql_sql.y:10788 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 1613: + case 1618: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:10751 +//line mysql_sql.y:10792 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 1614: + case 1619: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:10755 +//line mysql_sql.y:10796 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 1615: + case 1620: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:10761 +//line mysql_sql.y:10802 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 1616: + case 1621: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:10765 +//line mysql_sql.y:10806 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 1617: + case 1622: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:10770 +//line mysql_sql.y:10811 { yyVAL.str = "" } - case 1618: + case 1623: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:10774 +//line mysql_sql.y:10815 { yyVAL.str = yyDollar[1].str } - case 1619: + case 1624: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:10780 +//line mysql_sql.y:10821 { yyVAL.str = "" } - case 1620: + case 1625: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:10784 +//line mysql_sql.y:10825 { yyVAL.str = yyDollar[2].cstrUnion().Compare() } - case 1621: + case 1626: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.AttributeReference -//line mysql_sql.y:10790 +//line mysql_sql.y:10831 { var TableName = yyDollar[2].tableNameUnion() var KeyParts = yyDollar[3].keyPartsUnion() @@ -26069,10 +26243,10 @@ yydefault: ) } yyVAL.union = yyLOCAL - case 1622: + case 1627: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.ReferenceOnRecord -//line mysql_sql.y:10807 +//line mysql_sql.y:10848 { yyLOCAL = &tree.ReferenceOnRecord{ OnDelete: tree.REFERENCE_OPTION_INVALID, @@ -26080,10 +26254,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1623: + case 1628: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.ReferenceOnRecord -//line mysql_sql.y:10814 +//line mysql_sql.y:10855 { yyLOCAL = &tree.ReferenceOnRecord{ OnDelete: yyDollar[1].referenceOptionTypeUnion(), @@ -26091,10 +26265,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1624: + case 1629: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.ReferenceOnRecord -//line mysql_sql.y:10821 +//line mysql_sql.y:10862 { yyLOCAL = &tree.ReferenceOnRecord{ OnDelete: tree.REFERENCE_OPTION_INVALID, @@ -26102,10 +26276,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1625: + case 1630: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.ReferenceOnRecord -//line mysql_sql.y:10828 +//line mysql_sql.y:10869 { yyLOCAL = &tree.ReferenceOnRecord{ OnDelete: yyDollar[1].referenceOptionTypeUnion(), @@ -26113,10 +26287,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1626: + case 1631: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.ReferenceOnRecord -//line mysql_sql.y:10835 +//line mysql_sql.y:10876 { yyLOCAL = &tree.ReferenceOnRecord{ OnDelete: yyDollar[2].referenceOptionTypeUnion(), @@ -26124,274 +26298,274 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1627: + case 1632: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.ReferenceOptionType -//line mysql_sql.y:10844 +//line mysql_sql.y:10885 { yyLOCAL = yyDollar[3].referenceOptionTypeUnion() } yyVAL.union = yyLOCAL - case 1628: + case 1633: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.ReferenceOptionType -//line mysql_sql.y:10850 +//line mysql_sql.y:10891 { yyLOCAL = yyDollar[3].referenceOptionTypeUnion() } yyVAL.union = yyLOCAL - case 1629: + case 1634: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ReferenceOptionType -//line mysql_sql.y:10856 +//line mysql_sql.y:10897 { yyLOCAL = tree.REFERENCE_OPTION_RESTRICT } yyVAL.union = yyLOCAL - case 1630: + case 1635: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ReferenceOptionType -//line mysql_sql.y:10860 +//line mysql_sql.y:10901 { yyLOCAL = tree.REFERENCE_OPTION_CASCADE } yyVAL.union = yyLOCAL - case 1631: + case 1636: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.ReferenceOptionType -//line mysql_sql.y:10864 +//line mysql_sql.y:10905 { yyLOCAL = tree.REFERENCE_OPTION_SET_NULL } yyVAL.union = yyLOCAL - case 1632: + case 1637: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.ReferenceOptionType -//line mysql_sql.y:10868 +//line mysql_sql.y:10909 { yyLOCAL = tree.REFERENCE_OPTION_NO_ACTION } yyVAL.union = yyLOCAL - case 1633: + case 1638: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.ReferenceOptionType -//line mysql_sql.y:10872 +//line mysql_sql.y:10913 { yyLOCAL = tree.REFERENCE_OPTION_SET_DEFAULT } yyVAL.union = yyLOCAL - case 1634: + case 1639: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.MatchType -//line mysql_sql.y:10877 +//line mysql_sql.y:10918 { yyLOCAL = tree.MATCH_INVALID } yyVAL.union = yyLOCAL - case 1636: + case 1641: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.MatchType -//line mysql_sql.y:10884 +//line mysql_sql.y:10925 { yyLOCAL = tree.MATCH_FULL } yyVAL.union = yyLOCAL - case 1637: + case 1642: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.MatchType -//line mysql_sql.y:10888 +//line mysql_sql.y:10929 { yyLOCAL = tree.MATCH_PARTIAL } yyVAL.union = yyLOCAL - case 1638: + case 1643: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.MatchType -//line mysql_sql.y:10892 +//line mysql_sql.y:10933 { yyLOCAL = tree.MATCH_SIMPLE } yyVAL.union = yyLOCAL - case 1639: + case 1644: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.FullTextSearchType -//line mysql_sql.y:10897 +//line mysql_sql.y:10938 { yyLOCAL = tree.FULLTEXT_DEFAULT } yyVAL.union = yyLOCAL - case 1640: + case 1645: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.FullTextSearchType -//line mysql_sql.y:10901 +//line mysql_sql.y:10942 { yyLOCAL = tree.FULLTEXT_NL } yyVAL.union = yyLOCAL - case 1641: + case 1646: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.FullTextSearchType -//line mysql_sql.y:10905 +//line mysql_sql.y:10946 { yyLOCAL = tree.FULLTEXT_NL_QUERY_EXPANSION } yyVAL.union = yyLOCAL - case 1642: + case 1647: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.FullTextSearchType -//line mysql_sql.y:10909 +//line mysql_sql.y:10950 { yyLOCAL = tree.FULLTEXT_BOOLEAN } yyVAL.union = yyLOCAL - case 1643: + case 1648: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.FullTextSearchType -//line mysql_sql.y:10913 +//line mysql_sql.y:10954 { yyLOCAL = tree.FULLTEXT_QUERY_EXPANSION } yyVAL.union = yyLOCAL - case 1644: + case 1649: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL []*tree.KeyPart -//line mysql_sql.y:10918 +//line mysql_sql.y:10959 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1645: + case 1650: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []*tree.KeyPart -//line mysql_sql.y:10922 +//line mysql_sql.y:10963 { yyLOCAL = yyDollar[2].keyPartsUnion() } yyVAL.union = yyLOCAL - case 1646: + case 1651: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL int64 -//line mysql_sql.y:10927 +//line mysql_sql.y:10968 { yyLOCAL = -1 } yyVAL.union = yyLOCAL - case 1647: + case 1652: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL int64 -//line mysql_sql.y:10931 +//line mysql_sql.y:10972 { yyLOCAL = yyDollar[2].item.(int64) } yyVAL.union = yyLOCAL - case 1654: + case 1659: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.Subquery -//line mysql_sql.y:10947 +//line mysql_sql.y:10988 { yyLOCAL = &tree.Subquery{Select: yyDollar[1].selectStatementUnion(), Exists: false} } yyVAL.union = yyLOCAL - case 1655: + case 1660: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:10953 +//line mysql_sql.y:10994 { yyLOCAL = tree.NewBinaryExpr(tree.BIT_AND, yyDollar[1].exprUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1656: + case 1661: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:10957 +//line mysql_sql.y:10998 { yyLOCAL = tree.NewBinaryExpr(tree.BIT_OR, yyDollar[1].exprUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1657: + case 1662: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:10961 +//line mysql_sql.y:11002 { yyLOCAL = tree.NewBinaryExpr(tree.BIT_XOR, yyDollar[1].exprUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1658: + case 1663: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:10965 +//line mysql_sql.y:11006 { yyLOCAL = tree.NewBinaryExpr(tree.PLUS, yyDollar[1].exprUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1659: + case 1664: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:10969 +//line mysql_sql.y:11010 { yyLOCAL = tree.NewBinaryExpr(tree.MINUS, yyDollar[1].exprUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1660: + case 1665: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:10973 +//line mysql_sql.y:11014 { yyLOCAL = tree.NewBinaryExpr(tree.MULTI, yyDollar[1].exprUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1661: + case 1666: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:10977 +//line mysql_sql.y:11018 { yyLOCAL = tree.NewBinaryExpr(tree.DIV, yyDollar[1].exprUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1662: + case 1667: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:10981 +//line mysql_sql.y:11022 { yyLOCAL = tree.NewBinaryExpr(tree.INTEGER_DIV, yyDollar[1].exprUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1663: + case 1668: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:10985 +//line mysql_sql.y:11026 { yyLOCAL = tree.NewBinaryExpr(tree.MOD, yyDollar[1].exprUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1664: + case 1669: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:10989 +//line mysql_sql.y:11030 { yyLOCAL = tree.NewBinaryExpr(tree.MOD, yyDollar[1].exprUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1665: + case 1670: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:10993 +//line mysql_sql.y:11034 { yyLOCAL = tree.NewBinaryExpr(tree.LEFT_SHIFT, yyDollar[1].exprUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1666: + case 1671: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:10997 +//line mysql_sql.y:11038 { yyLOCAL = tree.NewBinaryExpr(tree.RIGHT_SHIFT, yyDollar[1].exprUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1667: + case 1672: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11001 +//line mysql_sql.y:11042 { name := tree.NewUnresolvedColName("json_extract") yyLOCAL = &tree.FuncExpr{ @@ -26401,10 +26575,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1668: + case 1673: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11010 +//line mysql_sql.y:11051 { extractName := tree.NewUnresolvedColName("json_extract") inner := &tree.FuncExpr{ @@ -26420,90 +26594,90 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1669: + case 1674: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11025 +//line mysql_sql.y:11066 { yyLOCAL = yyDollar[1].exprUnion() } yyVAL.union = yyLOCAL - case 1670: + case 1675: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11031 +//line mysql_sql.y:11072 { yyLOCAL = yyDollar[1].unresolvedNameUnion() } yyVAL.union = yyLOCAL - case 1671: + case 1676: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11035 +//line mysql_sql.y:11076 { yyLOCAL = yyDollar[1].varExprUnion() } yyVAL.union = yyLOCAL - case 1672: + case 1677: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11039 +//line mysql_sql.y:11080 { yyLOCAL = yyDollar[1].exprUnion() } yyVAL.union = yyLOCAL - case 1673: + case 1678: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11043 +//line mysql_sql.y:11084 { yyLOCAL = tree.NewParentExpr(yyDollar[2].exprUnion()) } yyVAL.union = yyLOCAL - case 1674: + case 1679: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11047 +//line mysql_sql.y:11088 { yyLOCAL = tree.NewTuple(append(yyDollar[2].exprsUnion(), yyDollar[4].exprUnion())) } yyVAL.union = yyLOCAL - case 1675: + case 1680: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11051 +//line mysql_sql.y:11092 { yyLOCAL = tree.NewUnaryExpr(tree.UNARY_PLUS, yyDollar[2].exprUnion()) } yyVAL.union = yyLOCAL - case 1676: + case 1681: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11055 +//line mysql_sql.y:11096 { yyLOCAL = tree.NewUnaryExpr(tree.UNARY_MINUS, yyDollar[2].exprUnion()) } yyVAL.union = yyLOCAL - case 1677: + case 1682: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11059 +//line mysql_sql.y:11100 { yyLOCAL = tree.NewUnaryExpr(tree.UNARY_TILDE, yyDollar[2].exprUnion()) } yyVAL.union = yyLOCAL - case 1678: + case 1683: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11063 +//line mysql_sql.y:11104 { yyLOCAL = tree.NewUnaryExpr(tree.UNARY_MARK, yyDollar[2].exprUnion()) } yyVAL.union = yyLOCAL - case 1679: + case 1684: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11067 +//line mysql_sql.y:11108 { hint := strings.ToLower(yyDollar[2].cstrUnion().Compare()) switch hint { @@ -26546,35 +26720,35 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1680: + case 1685: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11109 +//line mysql_sql.y:11150 { yyLOCAL = yyDollar[1].exprUnion() } yyVAL.union = yyLOCAL - case 1681: + case 1686: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11113 +//line mysql_sql.y:11154 { yyLOCAL = yyDollar[1].subqueryUnion() } yyVAL.union = yyLOCAL - case 1682: + case 1687: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11117 +//line mysql_sql.y:11158 { yyDollar[2].subqueryUnion().Exists = true yyLOCAL = yyDollar[2].subqueryUnion() } yyVAL.union = yyLOCAL - case 1683: + case 1688: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11122 +//line mysql_sql.y:11163 { yyLOCAL = &tree.CaseExpr{ Expr: yyDollar[2].exprUnion(), @@ -26583,50 +26757,50 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1684: + case 1689: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11130 +//line mysql_sql.y:11171 { yyLOCAL = tree.NewCastExpr(yyDollar[3].exprUnion(), yyDollar[5].columnTypeUnion()) } yyVAL.union = yyLOCAL - case 1685: + case 1690: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11134 +//line mysql_sql.y:11175 { yyLOCAL = tree.NewSerialExtractExpr(yyDollar[3].exprUnion(), yyDollar[5].exprUnion(), yyDollar[7].columnTypeUnion()) } yyVAL.union = yyLOCAL - case 1686: + case 1691: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11138 +//line mysql_sql.y:11179 { yyLOCAL = tree.NewBitCastExpr(yyDollar[3].exprUnion(), yyDollar[5].columnTypeUnion()) } yyVAL.union = yyLOCAL - case 1687: + case 1692: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11142 +//line mysql_sql.y:11183 { yyLOCAL = tree.NewCastExpr(yyDollar[1].exprUnion(), yyDollar[3].columnTypeUnion()) } yyVAL.union = yyLOCAL - case 1688: + case 1693: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11146 +//line mysql_sql.y:11187 { yyLOCAL = tree.NewCastExpr(yyDollar[3].exprUnion(), yyDollar[5].columnTypeUnion()) } yyVAL.union = yyLOCAL - case 1689: + case 1694: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11150 +//line mysql_sql.y:11191 { name := tree.NewUnresolvedColName(yyDollar[1].str) es := tree.NewNumVal(yyDollar[5].str, yyDollar[5].str, false, tree.P_char) @@ -26637,66 +26811,66 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1690: + case 1695: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11160 +//line mysql_sql.y:11201 { yyLOCAL = yyDollar[1].funcExprUnion() } yyVAL.union = yyLOCAL - case 1691: + case 1696: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11164 +//line mysql_sql.y:11205 { yyLOCAL = yyDollar[1].funcExprUnion() } yyVAL.union = yyLOCAL - case 1692: + case 1697: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11168 +//line mysql_sql.y:11209 { yyLOCAL = yyDollar[1].funcExprUnion() } yyVAL.union = yyLOCAL - case 1693: + case 1698: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11172 +//line mysql_sql.y:11213 { yyLOCAL = yyDollar[1].funcExprUnion() } yyVAL.union = yyLOCAL - case 1694: + case 1699: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11176 +//line mysql_sql.y:11217 { yyLOCAL = yyDollar[1].funcExprUnion() } yyVAL.union = yyLOCAL - case 1695: + case 1700: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11180 +//line mysql_sql.y:11221 { yyLOCAL = yyDollar[1].exprUnion() } yyVAL.union = yyLOCAL - case 1696: + case 1701: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11184 +//line mysql_sql.y:11225 { yyLOCAL = yyDollar[1].exprUnion() } yyVAL.union = yyLOCAL - case 1697: + case 1702: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11188 +//line mysql_sql.y:11229 { val, err := tree.NewFullTextMatchFuncExpression(yyDollar[3].keyPartsUnion(), yyDollar[7].str, yyDollar[8].fullTextSearchTypeUnion()) if err != nil { @@ -26706,16 +26880,29 @@ yydefault: yyLOCAL = val } yyVAL.union = yyLOCAL - case 1698: + case 1703: + yyDollar = yyS[yypt-8 : yypt+1] + var yyLOCAL tree.Expr +//line mysql_sql.y:11238 + { + val, err := tree.NewBm25MatchFuncExpression(yyDollar[3].keyPartsUnion(), yyDollar[7].str) + if err != nil { + yylex.Error(err.Error()) + goto ret1 + } + yyLOCAL = val + } + yyVAL.union = yyLOCAL + case 1704: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:11199 +//line mysql_sql.y:11249 { yyVAL.str = yyDollar[1].str } - case 1699: + case 1705: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11205 +//line mysql_sql.y:11255 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -26725,10 +26912,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1700: + case 1706: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11214 +//line mysql_sql.y:11264 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -26738,10 +26925,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1701: + case 1707: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11223 +//line mysql_sql.y:11273 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -26751,10 +26938,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1702: + case 1708: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11232 +//line mysql_sql.y:11282 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -26764,10 +26951,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1703: + case 1709: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11241 +//line mysql_sql.y:11291 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -26778,10 +26965,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1704: + case 1710: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11251 +//line mysql_sql.y:11301 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -26791,10 +26978,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1705: + case 1711: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11260 +//line mysql_sql.y:11310 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -26805,10 +26992,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1706: + case 1712: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11270 +//line mysql_sql.y:11320 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -26819,10 +27006,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1707: + case 1713: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11280 +//line mysql_sql.y:11330 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -26833,10 +27020,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1708: + case 1714: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11290 +//line mysql_sql.y:11340 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -26847,10 +27034,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1709: + case 1715: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11300 +//line mysql_sql.y:11350 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -26861,10 +27048,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1710: + case 1716: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11310 +//line mysql_sql.y:11360 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -26875,10 +27062,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1711: + case 1717: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11320 +//line mysql_sql.y:11370 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -26889,10 +27076,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1712: + case 1718: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11330 +//line mysql_sql.y:11380 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -26903,10 +27090,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1713: + case 1719: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11340 +//line mysql_sql.y:11390 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -26917,10 +27104,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1714: + case 1720: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11352 +//line mysql_sql.y:11402 { v := int(yyDollar[5].item.(int64)) val, err := tree.NewSampleRowsFuncExpression(v, true, nil, "block") @@ -26931,10 +27118,10 @@ yydefault: yyLOCAL = val } yyVAL.union = yyLOCAL - case 1715: + case 1721: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11362 +//line mysql_sql.y:11412 { v := int(yyDollar[5].item.(int64)) val, err := tree.NewSampleRowsFuncExpression(v, true, nil, yyDollar[8].str) @@ -26945,10 +27132,10 @@ yydefault: yyLOCAL = val } yyVAL.union = yyLOCAL - case 1716: + case 1722: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11372 +//line mysql_sql.y:11422 { val, err := tree.NewSamplePercentFuncExpression1(yyDollar[5].item.(int64), true, nil) if err != nil { @@ -26958,10 +27145,10 @@ yydefault: yyLOCAL = val } yyVAL.union = yyLOCAL - case 1717: + case 1723: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11381 +//line mysql_sql.y:11431 { val, err := tree.NewSamplePercentFuncExpression2(yyDollar[5].item.(float64), true, nil) if err != nil { @@ -26971,10 +27158,10 @@ yydefault: yyLOCAL = val } yyVAL.union = yyLOCAL - case 1718: + case 1724: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11391 +//line mysql_sql.y:11441 { v := int(yyDollar[5].item.(int64)) val, err := tree.NewSampleRowsFuncExpression(v, false, yyDollar[3].exprsUnion(), "block") @@ -26985,10 +27172,10 @@ yydefault: yyLOCAL = val } yyVAL.union = yyLOCAL - case 1719: + case 1725: yyDollar = yyS[yypt-9 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11401 +//line mysql_sql.y:11451 { v := int(yyDollar[5].item.(int64)) val, err := tree.NewSampleRowsFuncExpression(v, false, yyDollar[3].exprsUnion(), yyDollar[8].str) @@ -26999,10 +27186,10 @@ yydefault: yyLOCAL = val } yyVAL.union = yyLOCAL - case 1720: + case 1726: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11411 +//line mysql_sql.y:11461 { val, err := tree.NewSamplePercentFuncExpression1(yyDollar[5].item.(int64), false, yyDollar[3].exprsUnion()) if err != nil { @@ -27012,10 +27199,10 @@ yydefault: yyLOCAL = val } yyVAL.union = yyLOCAL - case 1721: + case 1727: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11420 +//line mysql_sql.y:11470 { val, err := tree.NewSamplePercentFuncExpression2(yyDollar[5].item.(float64), false, yyDollar[3].exprsUnion()) if err != nil { @@ -27025,58 +27212,58 @@ yydefault: yyLOCAL = val } yyVAL.union = yyLOCAL - case 1722: + case 1728: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11430 +//line mysql_sql.y:11480 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1723: + case 1729: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11434 +//line mysql_sql.y:11484 { yyLOCAL = yyDollar[2].exprUnion() } yyVAL.union = yyLOCAL - case 1724: + case 1730: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11439 +//line mysql_sql.y:11489 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1725: + case 1731: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:11443 +//line mysql_sql.y:11493 { yyLOCAL = yyDollar[1].exprUnion() } yyVAL.union = yyLOCAL - case 1726: + case 1732: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []*tree.When -//line mysql_sql.y:11449 +//line mysql_sql.y:11499 { yyLOCAL = []*tree.When{yyDollar[1].whenClauseUnion()} } yyVAL.union = yyLOCAL - case 1727: + case 1733: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL []*tree.When -//line mysql_sql.y:11453 +//line mysql_sql.y:11503 { yyLOCAL = append(yyDollar[1].whenClauseListUnion(), yyDollar[2].whenClauseUnion()) } yyVAL.union = yyLOCAL - case 1728: + case 1734: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.When -//line mysql_sql.y:11459 +//line mysql_sql.y:11509 { yyLOCAL = &tree.When{ Cond: yyDollar[2].exprUnion(), @@ -27084,9 +27271,9 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1729: + case 1735: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:11468 +//line mysql_sql.y:11518 { t := yyVAL.columnTypeUnion() str := strings.ToLower(t.InternalType.FamilyString) @@ -27099,10 +27286,10 @@ yydefault: } } } - case 1730: + case 1736: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:11480 +//line mysql_sql.y:11530 { name := yyDollar[1].str if yyDollar[2].str != "" { @@ -27120,10 +27307,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1731: + case 1737: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:11497 +//line mysql_sql.y:11547 { locale := "" yyLOCAL = &tree.T{ @@ -27138,10 +27325,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1733: + case 1739: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:11514 +//line mysql_sql.y:11564 { locale := "" yyLOCAL = &tree.T{ @@ -27156,10 +27343,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1734: + case 1740: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:11528 +//line mysql_sql.y:11578 { locale := "" oid := uint32(defines.MYSQL_TYPE_STRING) @@ -27179,10 +27366,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1735: + case 1741: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:11547 +//line mysql_sql.y:11597 { locale := "" yyLOCAL = &tree.T{ @@ -27195,10 +27382,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1736: + case 1742: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:11559 +//line mysql_sql.y:11609 { locale := "" yyLOCAL = &tree.T{ @@ -27213,10 +27400,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1737: + case 1743: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:11573 +//line mysql_sql.y:11623 { locale := "" yyLOCAL = &tree.T{ @@ -27232,10 +27419,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1738: + case 1744: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:11588 +//line mysql_sql.y:11638 { locale := "" yyLOCAL = &tree.T{ @@ -27251,10 +27438,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1739: + case 1745: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:11603 +//line mysql_sql.y:11653 { name := yyDollar[1].str if yyDollar[2].str != "" { @@ -27272,10 +27459,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1740: + case 1746: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:11620 +//line mysql_sql.y:11670 { locale := "" yyLOCAL = &tree.T{ @@ -27290,96 +27477,96 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1741: + case 1747: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:11636 +//line mysql_sql.y:11686 { yyVAL.str = "" } - case 1745: + case 1751: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.FrameBound -//line mysql_sql.y:11645 +//line mysql_sql.y:11695 { yyLOCAL = &tree.FrameBound{Type: tree.Following, UnBounded: true} } yyVAL.union = yyLOCAL - case 1746: + case 1752: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.FrameBound -//line mysql_sql.y:11649 +//line mysql_sql.y:11699 { yyLOCAL = &tree.FrameBound{Type: tree.Following, Expr: yyDollar[1].exprUnion()} } yyVAL.union = yyLOCAL - case 1747: + case 1753: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.FrameBound -//line mysql_sql.y:11653 +//line mysql_sql.y:11703 { yyLOCAL = &tree.FrameBound{Type: tree.Following, Expr: yyDollar[1].exprUnion()} } yyVAL.union = yyLOCAL - case 1748: + case 1754: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.FrameBound -//line mysql_sql.y:11659 +//line mysql_sql.y:11709 { yyLOCAL = &tree.FrameBound{Type: tree.CurrentRow} } yyVAL.union = yyLOCAL - case 1749: + case 1755: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.FrameBound -//line mysql_sql.y:11663 +//line mysql_sql.y:11713 { yyLOCAL = &tree.FrameBound{Type: tree.Preceding, UnBounded: true} } yyVAL.union = yyLOCAL - case 1750: + case 1756: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.FrameBound -//line mysql_sql.y:11667 +//line mysql_sql.y:11717 { yyLOCAL = &tree.FrameBound{Type: tree.Preceding, Expr: yyDollar[1].exprUnion()} } yyVAL.union = yyLOCAL - case 1751: + case 1757: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.FrameBound -//line mysql_sql.y:11671 +//line mysql_sql.y:11721 { yyLOCAL = &tree.FrameBound{Type: tree.Preceding, Expr: yyDollar[1].exprUnion()} } yyVAL.union = yyLOCAL - case 1752: + case 1758: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.FrameType -//line mysql_sql.y:11677 +//line mysql_sql.y:11727 { yyLOCAL = tree.Rows } yyVAL.union = yyLOCAL - case 1753: + case 1759: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.FrameType -//line mysql_sql.y:11681 +//line mysql_sql.y:11731 { yyLOCAL = tree.Range } yyVAL.union = yyLOCAL - case 1754: + case 1760: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.FrameType -//line mysql_sql.y:11685 +//line mysql_sql.y:11735 { yyLOCAL = tree.Groups } yyVAL.union = yyLOCAL - case 1755: + case 1761: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.FrameClause -//line mysql_sql.y:11691 +//line mysql_sql.y:11741 { yyLOCAL = &tree.FrameClause{ Type: yyDollar[1].frameTypeUnion(), @@ -27388,10 +27575,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1756: + case 1762: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.FrameClause -//line mysql_sql.y:11699 +//line mysql_sql.y:11749 { yyLOCAL = &tree.FrameClause{ Type: yyDollar[1].frameTypeUnion(), @@ -27401,82 +27588,82 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1757: + case 1763: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.FrameClause -//line mysql_sql.y:11709 +//line mysql_sql.y:11759 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1758: + case 1764: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.FrameClause -//line mysql_sql.y:11713 +//line mysql_sql.y:11763 { yyLOCAL = yyDollar[1].frameClauseUnion() } yyVAL.union = yyLOCAL - case 1759: + case 1765: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Exprs -//line mysql_sql.y:11719 +//line mysql_sql.y:11769 { yyLOCAL = yyDollar[3].exprsUnion() } yyVAL.union = yyLOCAL - case 1760: + case 1766: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.Exprs -//line mysql_sql.y:11724 +//line mysql_sql.y:11774 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1761: + case 1767: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Exprs -//line mysql_sql.y:11728 +//line mysql_sql.y:11778 { yyLOCAL = yyDollar[1].exprsUnion() } yyVAL.union = yyLOCAL - case 1762: + case 1768: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:11733 +//line mysql_sql.y:11783 { yyVAL.str = "," } - case 1763: + case 1769: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:11737 +//line mysql_sql.y:11787 { yyVAL.str = yyDollar[2].str } - case 1764: + case 1770: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:11742 +//line mysql_sql.y:11792 { yyVAL.str = "1,vector_l2_ops,random,false" } - case 1765: + case 1771: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:11746 +//line mysql_sql.y:11796 { yyVAL.str = yyDollar[2].str } - case 1766: + case 1772: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL *tree.WindowSpec -//line mysql_sql.y:11751 +//line mysql_sql.y:11801 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1768: + case 1774: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.WindowSpec -//line mysql_sql.y:11758 +//line mysql_sql.y:11808 { hasFrame := true var f *tree.FrameClause @@ -27501,10 +27688,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1769: + case 1775: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11784 +//line mysql_sql.y:11834 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27517,10 +27704,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1770: + case 1776: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11796 +//line mysql_sql.y:11846 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27533,10 +27720,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1771: + case 1777: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11808 +//line mysql_sql.y:11858 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27548,10 +27735,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1772: + case 1778: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11819 +//line mysql_sql.y:11869 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27563,10 +27750,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1773: + case 1779: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11830 +//line mysql_sql.y:11880 { name := tree.NewUnresolvedColName(yyDollar[1].str) es := tree.NewNumVal("*", "*", false, tree.P_star) @@ -27578,10 +27765,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1774: + case 1780: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11841 +//line mysql_sql.y:11891 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27592,10 +27779,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1775: + case 1781: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11851 +//line mysql_sql.y:11901 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27606,10 +27793,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1776: + case 1782: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11861 +//line mysql_sql.y:11911 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27621,10 +27808,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1777: + case 1783: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11872 +//line mysql_sql.y:11922 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27636,10 +27823,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1778: + case 1784: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11883 +//line mysql_sql.y:11933 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27651,10 +27838,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1779: + case 1785: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11894 +//line mysql_sql.y:11944 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27666,10 +27853,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1780: + case 1786: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11905 +//line mysql_sql.y:11955 { name := tree.NewUnresolvedColName(yyDollar[1].str) es := tree.NewNumVal("*", "*", false, tree.P_star) @@ -27681,10 +27868,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1781: + case 1787: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11916 +//line mysql_sql.y:11966 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27696,10 +27883,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1782: + case 1788: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11927 +//line mysql_sql.y:11977 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27711,10 +27898,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1783: + case 1789: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11938 +//line mysql_sql.y:11988 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27726,10 +27913,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1784: + case 1790: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11949 +//line mysql_sql.y:11999 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27741,10 +27928,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1785: + case 1791: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11960 +//line mysql_sql.y:12010 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27756,10 +27943,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1786: + case 1792: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11971 +//line mysql_sql.y:12021 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27771,10 +27958,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1787: + case 1793: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11982 +//line mysql_sql.y:12032 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27786,10 +27973,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1788: + case 1794: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:11993 +//line mysql_sql.y:12043 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27801,10 +27988,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1789: + case 1795: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12004 +//line mysql_sql.y:12054 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27816,10 +28003,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1790: + case 1796: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12015 +//line mysql_sql.y:12065 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27831,10 +28018,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1791: + case 1797: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12026 +//line mysql_sql.y:12076 { name := tree.NewUnresolvedColName(yyDollar[1].str) var columnList tree.Exprs @@ -27852,10 +28039,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1792: + case 1798: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12043 +//line mysql_sql.y:12093 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27867,10 +28054,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1793: + case 1799: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12054 +//line mysql_sql.y:12104 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27882,10 +28069,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1797: + case 1803: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12072 +//line mysql_sql.y:12122 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27895,10 +28082,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1798: + case 1804: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12081 +//line mysql_sql.y:12131 { name := tree.NewUnresolvedColName(yyDollar[1].str) exprs := tree.Exprs{yyDollar[3].exprUnion()} @@ -27910,10 +28097,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1799: + case 1805: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12092 +//line mysql_sql.y:12142 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27923,10 +28110,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1800: + case 1806: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12101 +//line mysql_sql.y:12151 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27936,10 +28123,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1801: + case 1807: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12110 +//line mysql_sql.y:12160 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27949,10 +28136,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1802: + case 1808: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12119 +//line mysql_sql.y:12169 { name := tree.NewUnresolvedColName(yyDollar[1].str) str := strings.ToLower(yyDollar[3].str) @@ -27964,10 +28151,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1803: + case 1809: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12130 +//line mysql_sql.y:12180 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27977,10 +28164,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1804: + case 1810: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12139 +//line mysql_sql.y:12189 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -27990,10 +28177,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1805: + case 1811: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12148 +//line mysql_sql.y:12198 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -28004,10 +28191,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1806: + case 1812: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12158 +//line mysql_sql.y:12208 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -28017,10 +28204,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1807: + case 1813: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12167 +//line mysql_sql.y:12217 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -28030,10 +28217,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1808: + case 1814: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12176 +//line mysql_sql.y:12226 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -28043,10 +28230,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1809: + case 1815: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12185 +//line mysql_sql.y:12235 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -28056,10 +28243,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1810: + case 1816: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12194 +//line mysql_sql.y:12244 { name := tree.NewUnresolvedColName(yyDollar[1].str) arg0 := tree.NewNumVal(int64(0), "0", false, tree.P_int64) @@ -28072,10 +28259,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1811: + case 1817: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12206 +//line mysql_sql.y:12256 { name := tree.NewUnresolvedColName(yyDollar[1].str) arg0 := tree.NewNumVal(int64(1), "1", false, tree.P_int64) @@ -28087,10 +28274,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1812: + case 1818: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12217 +//line mysql_sql.y:12267 { name := tree.NewUnresolvedColName(yyDollar[1].str) arg0 := tree.NewNumVal(int64(2), "2", false, tree.P_int64) @@ -28104,10 +28291,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1813: + case 1819: yyDollar = yyS[yypt-7 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12230 +//line mysql_sql.y:12280 { name := tree.NewUnresolvedColName(yyDollar[1].str) arg0 := tree.NewNumVal(int64(3), "3", false, tree.P_int64) @@ -28120,10 +28307,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1814: + case 1820: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12242 +//line mysql_sql.y:12292 { column := tree.NewUnresolvedColName(yyDollar[3].str) name := tree.NewUnresolvedColName(yyDollar[1].str) @@ -28134,16 +28321,16 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1821: + case 1827: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:12264 +//line mysql_sql.y:12314 { yyVAL.str = yyDollar[1].str } - case 1854: + case 1860: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12306 +//line mysql_sql.y:12356 { name := tree.NewUnresolvedColName(yyDollar[1].str) var es tree.Exprs = nil @@ -28157,10 +28344,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1855: + case 1861: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12319 +//line mysql_sql.y:12369 { name := tree.NewUnresolvedColName(yyDollar[1].str) var es tree.Exprs = nil @@ -28174,10 +28361,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1856: + case 1862: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12332 +//line mysql_sql.y:12382 { name := tree.NewUnresolvedColName(yyDollar[1].str) str := strings.ToLower(yyDollar[3].str) @@ -28189,10 +28376,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1857: + case 1863: yyDollar = yyS[yypt-8 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12343 +//line mysql_sql.y:12393 { name := tree.NewUnresolvedColName(yyDollar[1].str) str := strings.ToLower(yyDollar[3].str) @@ -28204,10 +28391,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1858: + case 1864: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12354 +//line mysql_sql.y:12404 { name := tree.NewUnresolvedColName(yyDollar[1].str) str := strings.ToUpper(yyDollar[3].str) @@ -28219,10 +28406,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1859: + case 1865: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12366 +//line mysql_sql.y:12416 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -28232,10 +28419,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1860: + case 1866: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12375 +//line mysql_sql.y:12425 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -28244,10 +28431,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1861: + case 1867: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12383 +//line mysql_sql.y:12433 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -28256,10 +28443,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1862: + case 1868: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12391 +//line mysql_sql.y:12441 { name := tree.NewUnresolvedColName(yyDollar[1].str) var es tree.Exprs = nil @@ -28273,10 +28460,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1863: + case 1869: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12404 +//line mysql_sql.y:12454 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -28286,10 +28473,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1864: + case 1870: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12413 +//line mysql_sql.y:12463 { name := tree.NewUnresolvedColName(yyDollar[1].str) exprs := make([]tree.Expr, 1) @@ -28301,10 +28488,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1865: + case 1871: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12424 +//line mysql_sql.y:12474 { name := tree.NewUnresolvedColName(yyDollar[1].str) exprs := make([]tree.Expr, 1) @@ -28316,10 +28503,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1866: + case 1872: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12435 +//line mysql_sql.y:12485 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -28329,10 +28516,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1867: + case 1873: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12444 +//line mysql_sql.y:12494 { cn := tree.NewNumVal(yyDollar[5].str, yyDollar[5].str, false, tree.P_char) es := yyDollar[3].exprsUnion() @@ -28345,10 +28532,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1868: + case 1874: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12456 +//line mysql_sql.y:12506 { val := tree.NewNumVal(yyDollar[2].str, yyDollar[2].str, false, tree.P_char) name := tree.NewUnresolvedColName(yyDollar[1].str) @@ -28359,10 +28546,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1869: + case 1875: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12466 +//line mysql_sql.y:12516 { val := tree.NewNumVal(yyDollar[2].str, yyDollar[2].str, false, tree.P_char) name := tree.NewUnresolvedColName(yyDollar[1].str) @@ -28373,10 +28560,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1870: + case 1876: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12476 +//line mysql_sql.y:12526 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -28386,10 +28573,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1871: + case 1877: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12485 +//line mysql_sql.y:12535 { es := tree.Exprs{yyDollar[3].exprUnion()} es = append(es, yyDollar[5].exprUnion()) @@ -28401,10 +28588,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1872: + case 1878: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12496 +//line mysql_sql.y:12546 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -28414,10 +28601,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1873: + case 1879: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12505 +//line mysql_sql.y:12555 { val := tree.NewNumVal(yyDollar[2].str, yyDollar[2].str, false, tree.P_char) name := tree.NewUnresolvedColName(yyDollar[1].str) @@ -28428,10 +28615,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1874: + case 1880: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12515 +//line mysql_sql.y:12565 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -28441,10 +28628,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1875: + case 1881: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12524 +//line mysql_sql.y:12574 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -28454,10 +28641,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1876: + case 1882: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.FuncExpr -//line mysql_sql.y:12533 +//line mysql_sql.y:12583 { name := tree.NewUnresolvedColName(yyDollar[1].str) yyLOCAL = &tree.FuncExpr{ @@ -28467,34 +28654,34 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1877: + case 1883: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12543 +//line mysql_sql.y:12593 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1878: + case 1884: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12547 +//line mysql_sql.y:12597 { yyLOCAL = yyDollar[1].exprUnion() } yyVAL.union = yyLOCAL - case 1879: + case 1885: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12553 +//line mysql_sql.y:12603 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1880: + case 1886: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12557 +//line mysql_sql.y:12607 { ival, errStr := util.GetInt64(yyDollar[2].item) if errStr != "" { @@ -28505,20 +28692,20 @@ yydefault: yyLOCAL = tree.NewNumVal(ival, str, false, tree.P_int64) } yyVAL.union = yyLOCAL - case 1887: + case 1893: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:12576 +//line mysql_sql.y:12626 { } - case 1888: + case 1894: yyDollar = yyS[yypt-2 : yypt+1] -//line mysql_sql.y:12578 +//line mysql_sql.y:12628 { } - case 1922: + case 1928: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12619 +//line mysql_sql.y:12669 { name := tree.NewUnresolvedColName(yyDollar[1].str) str := strings.ToLower(yyDollar[3].str) @@ -28530,106 +28717,106 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1923: + case 1929: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.FuncType -//line mysql_sql.y:12631 +//line mysql_sql.y:12681 { yyLOCAL = tree.FUNC_TYPE_DEFAULT } yyVAL.union = yyLOCAL - case 1924: + case 1930: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.FuncType -//line mysql_sql.y:12635 +//line mysql_sql.y:12685 { yyLOCAL = tree.FUNC_TYPE_DISTINCT } yyVAL.union = yyLOCAL - case 1925: + case 1931: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.FuncType -//line mysql_sql.y:12639 +//line mysql_sql.y:12689 { yyLOCAL = tree.FUNC_TYPE_ALL } yyVAL.union = yyLOCAL - case 1926: + case 1932: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.Tuple -//line mysql_sql.y:12645 +//line mysql_sql.y:12695 { yyLOCAL = tree.NewTuple(yyDollar[2].exprsUnion()) } yyVAL.union = yyLOCAL - case 1927: + case 1933: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.Exprs -//line mysql_sql.y:12650 +//line mysql_sql.y:12700 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1928: + case 1934: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Exprs -//line mysql_sql.y:12654 +//line mysql_sql.y:12704 { yyLOCAL = yyDollar[1].exprsUnion() } yyVAL.union = yyLOCAL - case 1929: + case 1935: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Exprs -//line mysql_sql.y:12660 +//line mysql_sql.y:12710 { yyLOCAL = tree.Exprs{yyDollar[1].exprUnion()} } yyVAL.union = yyLOCAL - case 1930: + case 1936: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Exprs -//line mysql_sql.y:12664 +//line mysql_sql.y:12714 { yyLOCAL = append(yyDollar[1].exprsUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1931: + case 1937: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Exprs -//line mysql_sql.y:12670 +//line mysql_sql.y:12720 { yyLOCAL = tree.Exprs{yyDollar[1].exprUnion()} } yyVAL.union = yyLOCAL - case 1932: + case 1938: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Exprs -//line mysql_sql.y:12674 +//line mysql_sql.y:12724 { yyLOCAL = append(yyDollar[1].exprsUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1933: + case 1939: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12681 +//line mysql_sql.y:12731 { yyLOCAL = tree.NewAndExpr(yyDollar[1].exprUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1934: + case 1940: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12685 +//line mysql_sql.y:12735 { yyLOCAL = tree.NewOrExpr(yyDollar[1].exprUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1935: + case 1941: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12689 +//line mysql_sql.y:12739 { name := tree.NewUnresolvedColName("concat") yyLOCAL = &tree.FuncExpr{ @@ -28639,355 +28826,355 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1936: + case 1942: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12698 +//line mysql_sql.y:12748 { yyLOCAL = tree.NewXorExpr(yyDollar[1].exprUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1937: + case 1943: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12702 +//line mysql_sql.y:12752 { yyLOCAL = tree.NewNotExpr(yyDollar[2].exprUnion()) } yyVAL.union = yyLOCAL - case 1938: + case 1944: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12706 +//line mysql_sql.y:12756 { yyLOCAL = yyDollar[1].exprUnion() } yyVAL.union = yyLOCAL - case 1939: + case 1945: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12711 +//line mysql_sql.y:12761 { yyLOCAL = yyDollar[1].exprUnion() } yyVAL.union = yyLOCAL - case 1940: + case 1946: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12715 +//line mysql_sql.y:12765 { yyLOCAL = tree.NewMaxValue() } yyVAL.union = yyLOCAL - case 1941: + case 1947: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12721 +//line mysql_sql.y:12771 { yyLOCAL = tree.NewIsNullExpr(yyDollar[1].exprUnion()) } yyVAL.union = yyLOCAL - case 1942: + case 1948: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12725 +//line mysql_sql.y:12775 { yyLOCAL = tree.NewIsNotNullExpr(yyDollar[1].exprUnion()) } yyVAL.union = yyLOCAL - case 1943: + case 1949: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12729 +//line mysql_sql.y:12779 { yyLOCAL = tree.NewIsUnknownExpr(yyDollar[1].exprUnion()) } yyVAL.union = yyLOCAL - case 1944: + case 1950: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12733 +//line mysql_sql.y:12783 { yyLOCAL = tree.NewIsNotUnknownExpr(yyDollar[1].exprUnion()) } yyVAL.union = yyLOCAL - case 1945: + case 1951: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12737 +//line mysql_sql.y:12787 { yyLOCAL = tree.NewIsTrueExpr(yyDollar[1].exprUnion()) } yyVAL.union = yyLOCAL - case 1946: + case 1952: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12741 +//line mysql_sql.y:12791 { yyLOCAL = tree.NewIsNotTrueExpr(yyDollar[1].exprUnion()) } yyVAL.union = yyLOCAL - case 1947: + case 1953: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12745 +//line mysql_sql.y:12795 { yyLOCAL = tree.NewIsFalseExpr(yyDollar[1].exprUnion()) } yyVAL.union = yyLOCAL - case 1948: + case 1954: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12749 +//line mysql_sql.y:12799 { yyLOCAL = tree.NewIsNotFalseExpr(yyDollar[1].exprUnion()) } yyVAL.union = yyLOCAL - case 1949: + case 1955: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12753 +//line mysql_sql.y:12803 { yyLOCAL = tree.NewComparisonExpr(yyDollar[2].comparisonOpUnion(), yyDollar[1].exprUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1950: + case 1956: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12757 +//line mysql_sql.y:12807 { yyLOCAL = tree.NewSubqueryComparisonExpr(yyDollar[2].comparisonOpUnion(), yyDollar[3].comparisonOpUnion(), yyDollar[1].exprUnion(), yyDollar[4].subqueryUnion()) yyLOCAL = tree.NewSubqueryComparisonExpr(yyDollar[2].comparisonOpUnion(), yyDollar[3].comparisonOpUnion(), yyDollar[1].exprUnion(), yyDollar[4].subqueryUnion()) } yyVAL.union = yyLOCAL - case 1952: + case 1958: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12765 +//line mysql_sql.y:12815 { yyLOCAL = tree.NewComparisonExpr(tree.IN, yyDollar[1].exprUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1953: + case 1959: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12769 +//line mysql_sql.y:12819 { yyLOCAL = tree.NewComparisonExpr(tree.NOT_IN, yyDollar[1].exprUnion(), yyDollar[4].exprUnion()) } yyVAL.union = yyLOCAL - case 1954: + case 1960: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12773 +//line mysql_sql.y:12823 { yyLOCAL = tree.NewComparisonExprWithEscape(tree.LIKE, yyDollar[1].exprUnion(), yyDollar[3].exprUnion(), yyDollar[4].exprUnion()) } yyVAL.union = yyLOCAL - case 1955: + case 1961: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12777 +//line mysql_sql.y:12827 { yyLOCAL = tree.NewComparisonExprWithEscape(tree.NOT_LIKE, yyDollar[1].exprUnion(), yyDollar[4].exprUnion(), yyDollar[5].exprUnion()) } yyVAL.union = yyLOCAL - case 1956: + case 1962: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12781 +//line mysql_sql.y:12831 { yyLOCAL = tree.NewComparisonExprWithEscape(tree.ILIKE, yyDollar[1].exprUnion(), yyDollar[3].exprUnion(), yyDollar[4].exprUnion()) } yyVAL.union = yyLOCAL - case 1957: + case 1963: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12785 +//line mysql_sql.y:12835 { yyLOCAL = tree.NewComparisonExprWithEscape(tree.NOT_ILIKE, yyDollar[1].exprUnion(), yyDollar[4].exprUnion(), yyDollar[5].exprUnion()) } yyVAL.union = yyLOCAL - case 1958: + case 1964: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12789 +//line mysql_sql.y:12839 { yyLOCAL = tree.NewComparisonExpr(tree.REG_MATCH, yyDollar[1].exprUnion(), yyDollar[3].exprUnion()) } yyVAL.union = yyLOCAL - case 1959: + case 1965: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12793 +//line mysql_sql.y:12843 { yyLOCAL = tree.NewComparisonExpr(tree.NOT_REG_MATCH, yyDollar[1].exprUnion(), yyDollar[4].exprUnion()) } yyVAL.union = yyLOCAL - case 1960: + case 1966: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12797 +//line mysql_sql.y:12847 { yyLOCAL = tree.NewRangeCond(false, yyDollar[1].exprUnion(), yyDollar[3].exprUnion(), yyDollar[5].exprUnion()) } yyVAL.union = yyLOCAL - case 1961: + case 1967: yyDollar = yyS[yypt-6 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12801 +//line mysql_sql.y:12851 { yyLOCAL = tree.NewRangeCond(true, yyDollar[1].exprUnion(), yyDollar[4].exprUnion(), yyDollar[6].exprUnion()) } yyVAL.union = yyLOCAL - case 1963: + case 1969: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12807 +//line mysql_sql.y:12857 { yyLOCAL = nil } yyVAL.union = yyLOCAL - case 1964: + case 1970: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12811 +//line mysql_sql.y:12861 { yyLOCAL = yyDollar[2].exprUnion() } yyVAL.union = yyLOCAL - case 1965: + case 1971: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12817 +//line mysql_sql.y:12867 { yyLOCAL = yyDollar[1].tupleUnion() } yyVAL.union = yyLOCAL - case 1966: + case 1972: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12821 +//line mysql_sql.y:12871 { yyLOCAL = yyDollar[1].subqueryUnion() } yyVAL.union = yyLOCAL - case 1967: + case 1973: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ComparisonOp -//line mysql_sql.y:12828 +//line mysql_sql.y:12878 { yyLOCAL = tree.ALL } yyVAL.union = yyLOCAL - case 1968: + case 1974: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ComparisonOp -//line mysql_sql.y:12832 +//line mysql_sql.y:12882 { yyLOCAL = tree.ANY } yyVAL.union = yyLOCAL - case 1969: + case 1975: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ComparisonOp -//line mysql_sql.y:12836 +//line mysql_sql.y:12886 { yyLOCAL = tree.SOME } yyVAL.union = yyLOCAL - case 1970: + case 1976: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ComparisonOp -//line mysql_sql.y:12842 +//line mysql_sql.y:12892 { yyLOCAL = tree.EQUAL } yyVAL.union = yyLOCAL - case 1971: + case 1977: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ComparisonOp -//line mysql_sql.y:12846 +//line mysql_sql.y:12896 { yyLOCAL = tree.LESS_THAN } yyVAL.union = yyLOCAL - case 1972: + case 1978: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ComparisonOp -//line mysql_sql.y:12850 +//line mysql_sql.y:12900 { yyLOCAL = tree.GREAT_THAN } yyVAL.union = yyLOCAL - case 1973: + case 1979: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ComparisonOp -//line mysql_sql.y:12854 +//line mysql_sql.y:12904 { yyLOCAL = tree.LESS_THAN_EQUAL } yyVAL.union = yyLOCAL - case 1974: + case 1980: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ComparisonOp -//line mysql_sql.y:12858 +//line mysql_sql.y:12908 { yyLOCAL = tree.GREAT_THAN_EQUAL } yyVAL.union = yyLOCAL - case 1975: + case 1981: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ComparisonOp -//line mysql_sql.y:12862 +//line mysql_sql.y:12912 { yyLOCAL = tree.NOT_EQUAL } yyVAL.union = yyLOCAL - case 1976: + case 1982: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ComparisonOp -//line mysql_sql.y:12866 +//line mysql_sql.y:12916 { yyLOCAL = tree.NULL_SAFE_EQUAL } yyVAL.union = yyLOCAL - case 1977: + case 1983: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:12872 +//line mysql_sql.y:12922 { yyLOCAL = tree.NewAttributePrimaryKey() } yyVAL.union = yyLOCAL - case 1978: + case 1984: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:12876 +//line mysql_sql.y:12926 { yyLOCAL = tree.NewAttributeUniqueKey() } yyVAL.union = yyLOCAL - case 1979: + case 1985: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:12880 +//line mysql_sql.y:12930 { yyLOCAL = tree.NewAttributeUnique() } yyVAL.union = yyLOCAL - case 1980: + case 1986: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.ColumnAttribute -//line mysql_sql.y:12884 +//line mysql_sql.y:12934 { yyLOCAL = tree.NewAttributeKey() } yyVAL.union = yyLOCAL - case 1981: + case 1987: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12890 +//line mysql_sql.y:12940 { str := fmt.Sprintf("%v", yyDollar[1].item) switch v := yyDollar[1].item.(type) { @@ -29001,35 +29188,35 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1982: + case 1988: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12903 +//line mysql_sql.y:12953 { fval := yyDollar[1].item.(float64) yyLOCAL = tree.NewNumVal(fval, yylex.(*Lexer).scanner.LastToken, false, tree.P_float64) } yyVAL.union = yyLOCAL - case 1983: + case 1989: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12908 +//line mysql_sql.y:12958 { yyLOCAL = tree.NewNumVal(yyDollar[1].str, yyDollar[1].str, false, tree.P_decimal) } yyVAL.union = yyLOCAL - case 1984: + case 1990: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12914 +//line mysql_sql.y:12964 { yyLOCAL = tree.NewNumVal(yyDollar[1].str, yyDollar[1].str, false, tree.P_char) } yyVAL.union = yyLOCAL - case 1985: + case 1991: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12918 +//line mysql_sql.y:12968 { str := fmt.Sprintf("%v", yyDollar[1].item) switch v := yyDollar[1].item.(type) { @@ -29043,101 +29230,101 @@ yydefault: } } yyVAL.union = yyLOCAL - case 1986: + case 1992: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12931 +//line mysql_sql.y:12981 { fval := yyDollar[1].item.(float64) yyLOCAL = tree.NewNumVal(fval, yylex.(*Lexer).scanner.LastToken, false, tree.P_float64) } yyVAL.union = yyLOCAL - case 1987: + case 1993: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12936 +//line mysql_sql.y:12986 { yyLOCAL = tree.NewNumVal(true, "true", false, tree.P_bool) } yyVAL.union = yyLOCAL - case 1988: + case 1994: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12940 +//line mysql_sql.y:12990 { yyLOCAL = tree.NewNumVal(false, "false", false, tree.P_bool) } yyVAL.union = yyLOCAL - case 1989: + case 1995: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12944 +//line mysql_sql.y:12994 { yyLOCAL = tree.NewNumVal("null", "null", false, tree.P_null) } yyVAL.union = yyLOCAL - case 1990: + case 1996: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12948 +//line mysql_sql.y:12998 { yyLOCAL = tree.NewNumVal(yyDollar[1].str, yyDollar[1].str, false, tree.P_hexnum) } yyVAL.union = yyLOCAL - case 1991: + case 1997: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12952 +//line mysql_sql.y:13002 { yyLOCAL = tree.NewNumVal(yyDollar[2].str, yyDollar[2].str, false, tree.P_ScoreBinaryHexnum) } yyVAL.union = yyLOCAL - case 1992: + case 1998: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12956 +//line mysql_sql.y:13006 { yyLOCAL = tree.NewNumVal(yyDollar[1].str, yyDollar[1].str, false, tree.P_decimal) } yyVAL.union = yyLOCAL - case 1993: + case 1999: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12960 +//line mysql_sql.y:13010 { yyLOCAL = tree.NewNumVal(yyDollar[1].str, yyDollar[1].str, false, tree.P_bit) } yyVAL.union = yyLOCAL - case 1994: + case 2000: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12964 +//line mysql_sql.y:13014 { yyLOCAL = tree.NewParamExpr(yylex.(*Lexer).GetParamIndex()) } yyVAL.union = yyLOCAL - case 1995: + case 2001: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Expr -//line mysql_sql.y:12968 +//line mysql_sql.y:13018 { yyLOCAL = tree.NewNumVal(yyDollar[2].str, yyDollar[2].str, false, tree.P_ScoreBinary) } yyVAL.union = yyLOCAL - case 1996: + case 2002: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:12974 +//line mysql_sql.y:13024 { yyLOCAL = yyDollar[1].columnTypeUnion() yyLOCAL.InternalType.Unsigned = yyDollar[2].unsignedOptUnion() yyLOCAL.InternalType.Zerofill = yyDollar[3].zeroFillOptUnion() } yyVAL.union = yyLOCAL - case 2000: + case 2006: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:12983 +//line mysql_sql.y:13033 { locale := "" yyLOCAL = &tree.T{ @@ -29151,27 +29338,27 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2001: + case 2007: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:12998 +//line mysql_sql.y:13048 { yyLOCAL = yyDollar[1].columnTypeUnion() yyLOCAL.InternalType.DisplayWith = yyDollar[2].lengthOptUnion() } yyVAL.union = yyLOCAL - case 2002: + case 2008: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13003 +//line mysql_sql.y:13053 { yyLOCAL = yyDollar[1].columnTypeUnion() } yyVAL.union = yyLOCAL - case 2003: + case 2009: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13009 +//line mysql_sql.y:13059 { locale := "" yyLOCAL = &tree.T{ @@ -29184,10 +29371,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2004: + case 2010: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13021 +//line mysql_sql.y:13071 { locale := "" yyLOCAL = &tree.T{ @@ -29200,10 +29387,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2005: + case 2011: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13033 +//line mysql_sql.y:13083 { locale := "" yyLOCAL = &tree.T{ @@ -29216,10 +29403,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2006: + case 2012: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13045 +//line mysql_sql.y:13095 { locale := "" yyLOCAL = &tree.T{ @@ -29233,10 +29420,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2007: + case 2013: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13058 +//line mysql_sql.y:13108 { locale := "" yyLOCAL = &tree.T{ @@ -29250,10 +29437,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2008: + case 2014: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13071 +//line mysql_sql.y:13121 { locale := "" yyLOCAL = &tree.T{ @@ -29267,10 +29454,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2009: + case 2015: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13084 +//line mysql_sql.y:13134 { locale := "" yyLOCAL = &tree.T{ @@ -29284,10 +29471,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2010: + case 2016: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13097 +//line mysql_sql.y:13147 { locale := "" yyLOCAL = &tree.T{ @@ -29301,10 +29488,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2011: + case 2017: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13110 +//line mysql_sql.y:13160 { locale := "" yyLOCAL = &tree.T{ @@ -29318,10 +29505,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2012: + case 2018: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13123 +//line mysql_sql.y:13173 { locale := "" yyLOCAL = &tree.T{ @@ -29335,10 +29522,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2013: + case 2019: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13136 +//line mysql_sql.y:13186 { locale := "" yyLOCAL = &tree.T{ @@ -29352,10 +29539,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2014: + case 2020: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13149 +//line mysql_sql.y:13199 { locale := "" yyLOCAL = &tree.T{ @@ -29369,10 +29556,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2015: + case 2021: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13162 +//line mysql_sql.y:13212 { locale := "" yyLOCAL = &tree.T{ @@ -29386,10 +29573,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2016: + case 2022: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13175 +//line mysql_sql.y:13225 { locale := "" yyLOCAL = &tree.T{ @@ -29403,10 +29590,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2017: + case 2023: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13190 +//line mysql_sql.y:13240 { locale := "" if yyDollar[2].lengthScaleOptUnion().DisplayWith > 255 { @@ -29434,10 +29621,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2018: + case 2024: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13217 +//line mysql_sql.y:13267 { // DOUBLE PRECISION is the SQL-standard synonym for DOUBLE (float64). locale := "" @@ -29466,10 +29653,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2019: + case 2025: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13245 +//line mysql_sql.y:13295 { locale := "" if yyDollar[2].lengthScaleOptUnion().DisplayWith > 255 { @@ -29511,10 +29698,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2020: + case 2026: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13287 +//line mysql_sql.y:13337 { locale := "" if yyDollar[2].lengthScaleOptUnion().Scale != tree.NotDefineDec && yyDollar[2].lengthScaleOptUnion().Scale > yyDollar[2].lengthScaleOptUnion().DisplayWith { @@ -29563,10 +29750,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2021: + case 2027: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13335 +//line mysql_sql.y:13385 { locale := "" if yyDollar[2].lengthScaleOptUnion().Scale != tree.NotDefineDec && yyDollar[2].lengthScaleOptUnion().Scale > yyDollar[2].lengthScaleOptUnion().DisplayWith { @@ -29615,10 +29802,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2022: + case 2028: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13383 +//line mysql_sql.y:13433 { locale := "" yyLOCAL = &tree.T{ @@ -29634,10 +29821,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2023: + case 2029: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13400 +//line mysql_sql.y:13450 { locale := "" yyLOCAL = &tree.T{ @@ -29650,10 +29837,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2024: + case 2030: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13412 +//line mysql_sql.y:13462 { locale := "" if yyDollar[2].lengthOptUnion() < 0 || yyDollar[2].lengthOptUnion() > 6 { @@ -29674,10 +29861,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2025: + case 2031: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13432 +//line mysql_sql.y:13482 { locale := "" if yyDollar[2].lengthOptUnion() < 0 || yyDollar[2].lengthOptUnion() > 6 { @@ -29698,10 +29885,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2026: + case 2032: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13452 +//line mysql_sql.y:13502 { locale := "" if yyDollar[2].lengthOptUnion() < 0 || yyDollar[2].lengthOptUnion() > 6 { @@ -29722,10 +29909,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2027: + case 2033: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13472 +//line mysql_sql.y:13522 { locale := "" yyLOCAL = &tree.T{ @@ -29740,10 +29927,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2028: + case 2034: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13488 +//line mysql_sql.y:13538 { locale := "" yyLOCAL = &tree.T{ @@ -29757,10 +29944,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2029: + case 2035: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13501 +//line mysql_sql.y:13551 { locale := "" yyLOCAL = &tree.T{ @@ -29774,10 +29961,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2030: + case 2036: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13514 +//line mysql_sql.y:13564 { locale := "" yyLOCAL = &tree.T{ @@ -29791,10 +29978,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2031: + case 2037: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13527 +//line mysql_sql.y:13577 { locale := "" yyLOCAL = &tree.T{ @@ -29808,10 +29995,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2032: + case 2038: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13540 +//line mysql_sql.y:13590 { locale := "" yyLOCAL = &tree.T{ @@ -29824,10 +30011,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2033: + case 2039: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13552 +//line mysql_sql.y:13602 { locale := "" yyLOCAL = &tree.T{ @@ -29840,10 +30027,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2034: + case 2040: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13564 +//line mysql_sql.y:13614 { locale := "" yyLOCAL = &tree.T{ @@ -29856,10 +30043,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2035: + case 2041: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13576 +//line mysql_sql.y:13626 { locale := "" yyLOCAL = &tree.T{ @@ -29872,10 +30059,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2036: + case 2042: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13588 +//line mysql_sql.y:13638 { locale := "" yyLOCAL = &tree.T{ @@ -29888,10 +30075,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2037: + case 2043: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13600 +//line mysql_sql.y:13650 { locale := "" yyLOCAL = &tree.T{ @@ -29904,10 +30091,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2038: + case 2044: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13612 +//line mysql_sql.y:13662 { locale := "" yyLOCAL = &tree.T{ @@ -29920,10 +30107,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2039: + case 2045: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13624 +//line mysql_sql.y:13674 { locale := "" yyLOCAL = &tree.T{ @@ -29936,10 +30123,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2040: + case 2046: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13636 +//line mysql_sql.y:13686 { locale := "" yyLOCAL = &tree.T{ @@ -29952,10 +30139,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2041: + case 2047: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13648 +//line mysql_sql.y:13698 { locale := "" yyLOCAL = &tree.T{ @@ -29968,10 +30155,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2042: + case 2048: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13660 +//line mysql_sql.y:13710 { locale := "" yyLOCAL = &tree.T{ @@ -29985,10 +30172,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2043: + case 2049: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13673 +//line mysql_sql.y:13723 { locale := "" yyLOCAL = &tree.T{ @@ -30002,10 +30189,78 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2044: + case 2050: + yyDollar = yyS[yypt-2 : yypt+1] + var yyLOCAL *tree.T +//line mysql_sql.y:13736 + { + locale := "" + yyLOCAL = &tree.T{ + InternalType: tree.InternalType{ + Family: tree.ArrayFamily, + Locale: &locale, + FamilyString: yyDollar[1].str, + DisplayWith: yyDollar[2].lengthOptUnion(), + Oid: uint32(defines.MYSQL_TYPE_VARCHAR), + }, + } + } + yyVAL.union = yyLOCAL + case 2051: + yyDollar = yyS[yypt-2 : yypt+1] + var yyLOCAL *tree.T +//line mysql_sql.y:13749 + { + locale := "" + yyLOCAL = &tree.T{ + InternalType: tree.InternalType{ + Family: tree.ArrayFamily, + Locale: &locale, + FamilyString: yyDollar[1].str, + DisplayWith: yyDollar[2].lengthOptUnion(), + Oid: uint32(defines.MYSQL_TYPE_VARCHAR), + }, + } + } + yyVAL.union = yyLOCAL + case 2052: + yyDollar = yyS[yypt-2 : yypt+1] + var yyLOCAL *tree.T +//line mysql_sql.y:13762 + { + locale := "" + yyLOCAL = &tree.T{ + InternalType: tree.InternalType{ + Family: tree.ArrayFamily, + Locale: &locale, + FamilyString: yyDollar[1].str, + DisplayWith: yyDollar[2].lengthOptUnion(), + Oid: uint32(defines.MYSQL_TYPE_VARCHAR), + }, + } + } + yyVAL.union = yyLOCAL + case 2053: + yyDollar = yyS[yypt-2 : yypt+1] + var yyLOCAL *tree.T +//line mysql_sql.y:13775 + { + locale := "" + yyLOCAL = &tree.T{ + InternalType: tree.InternalType{ + Family: tree.ArrayFamily, + Locale: &locale, + FamilyString: yyDollar[1].str, + DisplayWith: yyDollar[2].lengthOptUnion(), + Oid: uint32(defines.MYSQL_TYPE_VARCHAR), + }, + } + } + yyVAL.union = yyLOCAL + case 2054: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13686 +//line mysql_sql.y:13788 { locale := "" yyLOCAL = &tree.T{ @@ -30019,10 +30274,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2045: + case 2055: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13699 +//line mysql_sql.y:13801 { locale := "" yyLOCAL = &tree.T{ @@ -30036,10 +30291,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2046: + case 2056: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13712 +//line mysql_sql.y:13814 { locale := "" yyLOCAL = &tree.T{ @@ -30053,20 +30308,20 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2047: + case 2057: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:13727 +//line mysql_sql.y:13829 { yyLOCAL = &tree.Do{ Exprs: yyDollar[2].exprsUnion(), } } yyVAL.union = yyLOCAL - case 2048: + case 2058: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:13735 +//line mysql_sql.y:13837 { yyLOCAL = &tree.Declare{ Variables: yyDollar[2].strsUnion(), @@ -30075,10 +30330,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2049: + case 2059: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:13744 +//line mysql_sql.y:13846 { yyLOCAL = &tree.Declare{ Variables: yyDollar[2].strsUnion(), @@ -30087,83 +30342,83 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2050: + case 2060: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13754 +//line mysql_sql.y:13856 { yyLOCAL = tree.NewSpatialType(yyDollar[1].str) } yyVAL.union = yyLOCAL - case 2069: + case 2079: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:13782 +//line mysql_sql.y:13884 { yyLOCAL = make([]string, 0, 4) yyLOCAL = append(yyLOCAL, yyDollar[1].str) } yyVAL.union = yyLOCAL - case 2070: + case 2080: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:13787 +//line mysql_sql.y:13889 { yyLOCAL = append(yyDollar[1].strsUnion(), yyDollar[3].str) } yyVAL.union = yyLOCAL - case 2071: + case 2081: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL int32 -//line mysql_sql.y:13793 +//line mysql_sql.y:13895 { yyLOCAL = 0 } yyVAL.union = yyLOCAL - case 2073: + case 2083: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL int32 -//line mysql_sql.y:13800 +//line mysql_sql.y:13902 { yyLOCAL = 0 } yyVAL.union = yyLOCAL - case 2074: + case 2084: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL int32 -//line mysql_sql.y:13804 +//line mysql_sql.y:13906 { yyLOCAL = int32(yyDollar[2].item.(int64)) } yyVAL.union = yyLOCAL - case 2075: + case 2085: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL int32 -//line mysql_sql.y:13809 +//line mysql_sql.y:13911 { yyLOCAL = int32(-1) } yyVAL.union = yyLOCAL - case 2076: + case 2086: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL int32 -//line mysql_sql.y:13813 +//line mysql_sql.y:13915 { yyLOCAL = int32(yyDollar[2].item.(int64)) } yyVAL.union = yyLOCAL - case 2077: + case 2087: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL int32 -//line mysql_sql.y:13819 +//line mysql_sql.y:13921 { yyLOCAL = tree.GetDisplayWith(int32(yyDollar[2].item.(int64))) } yyVAL.union = yyLOCAL - case 2078: + case 2088: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.LengthScaleOpt -//line mysql_sql.y:13825 +//line mysql_sql.y:13927 { yyLOCAL = tree.LengthScaleOpt{ DisplayWith: tree.NotDefineDisplayWidth, @@ -30171,10 +30426,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2079: + case 2089: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.LengthScaleOpt -//line mysql_sql.y:13832 +//line mysql_sql.y:13934 { yyLOCAL = tree.LengthScaleOpt{ DisplayWith: tree.GetDisplayWith(int32(yyDollar[2].item.(int64))), @@ -30182,10 +30437,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2080: + case 2090: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.LengthScaleOpt -//line mysql_sql.y:13839 +//line mysql_sql.y:13941 { yyLOCAL = tree.LengthScaleOpt{ DisplayWith: tree.GetDisplayWith(int32(yyDollar[2].item.(int64))), @@ -30193,10 +30448,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2081: + case 2091: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.LengthScaleOpt -//line mysql_sql.y:13848 +//line mysql_sql.y:13950 { yyLOCAL = tree.LengthScaleOpt{ DisplayWith: 38, // this is the default precision for decimal @@ -30204,10 +30459,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2082: + case 2092: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.LengthScaleOpt -//line mysql_sql.y:13855 +//line mysql_sql.y:13957 { yyLOCAL = tree.LengthScaleOpt{ DisplayWith: tree.GetDisplayWith(int32(yyDollar[2].item.(int64))), @@ -30215,10 +30470,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2083: + case 2093: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.LengthScaleOpt -//line mysql_sql.y:13862 +//line mysql_sql.y:13964 { yyLOCAL = tree.LengthScaleOpt{ DisplayWith: tree.GetDisplayWith(int32(yyDollar[2].item.(int64))), @@ -30226,52 +30481,52 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2084: + case 2094: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:13871 +//line mysql_sql.y:13973 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 2085: + case 2095: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:13875 +//line mysql_sql.y:13977 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 2086: + case 2096: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:13879 +//line mysql_sql.y:13981 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 2087: + case 2097: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:13885 +//line mysql_sql.y:13987 { } - case 2088: + case 2098: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:13887 +//line mysql_sql.y:13989 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 2092: + case 2102: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:13897 +//line mysql_sql.y:13999 { yyVAL.str = "" } - case 2093: + case 2103: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:13901 +//line mysql_sql.y:14003 { yyVAL.str = string(yyDollar[1].str) } diff --git a/pkg/sql/parsers/dialect/mysql/mysql_sql.y b/pkg/sql/parsers/dialect/mysql/mysql_sql.y index 995e82b0a332a..59952980fab21 100644 --- a/pkg/sql/parsers/dialect/mysql/mysql_sql.y +++ b/pkg/sql/parsers/dialect/mysql/mysql_sql.y @@ -390,7 +390,7 @@ func sqlTaskInt64(v any) int64 { %token TIME TIMESTAMP DATETIME YEAR %token CHAR VARCHAR BOOL CHARACTER VARBINARY NCHAR %token TEXT TINYTEXT MEDIUMTEXT LONGTEXT DATALINK -%token BLOB TINYBLOB MEDIUMBLOB LONGBLOB JSON ENUM UUID VECF32 VECF64 +%token BLOB TINYBLOB MEDIUMBLOB LONGBLOB JSON ENUM UUID VECF32 VECF64 VECBF16 VECF16 VECINT8 VECUINT8 %token GEOMETRY POINT LINESTRING POLYGON GEOMETRYCOLLECTION MULTIPOINT MULTILINESTRING MULTIPOLYGON %token GEOMETRY32 GEOGRAPHY GEOGRAPHY32 POINT32 LINESTRING32 POLYGON32 GEOMETRYCOLLECTION32 MULTIPOINT32 MULTILINESTRING32 MULTIPOLYGON32 %token INT1 INT2 INT3 INT4 INT8 S3OPTION STAGEOPTION @@ -426,7 +426,7 @@ func sqlTaskInt64(v any) int64 { %token PROPERTIES // Secondary Index -%token PARSER VISIBLE INVISIBLE BTREE HASH RTREE BSI IVFFLAT MASTER HNSW CAGRA IVFPQ +%token PARSER VISIBLE INVISIBLE BTREE HASH RTREE BSI IVFFLAT MASTER HNSW CAGRA IVFPQ BM25 %token ZONEMAP LEADING BOTH TRAILING UNKNOWN LISTS OP_TYPE REINDEX EF_SEARCH EF_CONSTRUCTION M ASYNC FORCE_SYNC AUTO_UPDATE INTERMEDIATE_GRAPH_DEGREE GRAPH_DEGREE QUANTIZATION BITS_PER_CODE DISTRIBUTION_MODE ITOPK_SIZE INCLUDE KMEANS_TRAIN_PERCENT KMEANS_MAX_ITERATION MAX_INDEX_CAPACITY // Alter @@ -4179,6 +4179,19 @@ alter_table_alter: var name = tree.Identifier($2.Compare()) $$ = tree.NewAlterOptionAlterReIndex(name, io) } +| REINDEX ident BM25 index_option_list + { + var io *tree.IndexOption = nil + if $4 == nil { + io = tree.NewIndexOption() + io.IType = tree.INDEX_TYPE_BM25 + } else { + io = $4 + io.IType = tree.INDEX_TYPE_BM25 + } + var name = tree.Identifier($2.Compare()) + $$ = tree.NewAlterOptionAlterReIndex(name, io) + } | CHECK ident enforce { var checkType = $1 @@ -8325,12 +8338,16 @@ index_option_list: opt1.Async = opt2.Async } else if opt2.ForceSync { opt1.ForceSync = opt2.ForceSync + } else if opt2.Merge { + opt1.Merge = opt2.Merge } else if opt2.AutoUpdate { opt1.AutoUpdate = opt2.AutoUpdate } else if opt2.Day > 0 { opt1.Day = opt2.Day } else if opt2.Hour > 0 { opt1.Hour = opt2.Hour + } else if opt2.Second > 0 { + opt1.Second = opt2.Second } else if opt2.IntermediateGraphDegree > 0 { opt1.IntermediateGraphDegree = opt2.IntermediateGraphDegree } else if opt2.GraphDegree > 0 { @@ -8544,7 +8561,13 @@ index_option: | FORCE_SYNC { io := tree.NewIndexOption() - io.ForceSync = true + io.ForceSync = true + $$ = io + } +| MERGE + { + io := tree.NewIndexOption() + io.Merge = true $$ = io } | AUTO_UPDATE '=' TRUE @@ -8581,6 +8604,17 @@ index_option: io.Hour = val $$ = io } +| SECOND equal_opt INTEGRAL + { + val := int64($3.(int64)) + if val < 0 { + yylex.Error("SECOND should be greater than or equal to 0") + return 1 + } + io := tree.NewIndexOption() + io.Second = val + $$ = io + } index_column_list: @@ -8646,6 +8680,10 @@ using_opt: { $$ = tree.INDEX_TYPE_CAGRA } +| USING BM25 + { + $$ = tree.INDEX_TYPE_BM25 + } | USING MASTER { $$ = tree.INDEX_TYPE_MASTER @@ -10311,6 +10349,8 @@ index_def: keyTyp = tree.INDEX_TYPE_CAGRA case "ivfpq": keyTyp = tree.INDEX_TYPE_IVFPQ + case "bm25": + keyTyp = tree.INDEX_TYPE_BM25 default: yylex.Error("Invalid the type of index") goto ret1 @@ -10523,6 +10563,7 @@ index_type: | HNSW | CAGRA | IVFPQ +| BM25 insert_method_options: NO @@ -11191,7 +11232,16 @@ simple_expr: yylex.Error(err.Error()) goto ret1 } - $$ = val + $$ = val + } +| BM25 '(' index_column_list ')' AGAINST '(' search_pattern ')' + { + val, err := tree.NewBm25MatchFuncExpression($3, $7) + if err != nil { + yylex.Error(err.Error()) + goto ret1 + } + $$ = val } search_pattern: @@ -13683,6 +13733,58 @@ char_type: }, } } +| VECBF16 length_option_opt + { + locale := "" + $$ = &tree.T{ + InternalType: tree.InternalType{ + Family: tree.ArrayFamily, + Locale: &locale, + FamilyString: $1, + DisplayWith: $2, + Oid:uint32(defines.MYSQL_TYPE_VARCHAR), + }, + } + } +| VECF16 length_option_opt + { + locale := "" + $$ = &tree.T{ + InternalType: tree.InternalType{ + Family: tree.ArrayFamily, + Locale: &locale, + FamilyString: $1, + DisplayWith: $2, + Oid:uint32(defines.MYSQL_TYPE_VARCHAR), + }, + } + } +| VECINT8 length_option_opt + { + locale := "" + $$ = &tree.T{ + InternalType: tree.InternalType{ + Family: tree.ArrayFamily, + Locale: &locale, + FamilyString: $1, + DisplayWith: $2, + Oid:uint32(defines.MYSQL_TYPE_VARCHAR), + }, + } + } +| VECUINT8 length_option_opt + { + locale := "" + $$ = &tree.T{ + InternalType: tree.InternalType{ + Family: tree.ArrayFamily, + Locale: &locale, + FamilyString: $1, + DisplayWith: $2, + Oid:uint32(defines.MYSQL_TYPE_VARCHAR), + }, + } + } | ENUM '(' enum_values ')' { locale := "" @@ -14189,6 +14291,7 @@ non_reserved_keyword: | HNSW | CAGRA | IVFPQ +| BM25 | PERSIST | GRANT | INCLUDE @@ -14201,6 +14304,10 @@ non_reserved_keyword: | JSON | VECF32 | VECF64 +| VECBF16 +| VECF16 +| VECINT8 +| VECUINT8 | KEY_BLOCK_SIZE | LISTS | OP_TYPE diff --git a/pkg/sql/parsers/dialect/mysql/mysql_sql_test.go b/pkg/sql/parsers/dialect/mysql/mysql_sql_test.go index 12f488f798cc8..ce2093a5fc554 100644 --- a/pkg/sql/parsers/dialect/mysql/mysql_sql_test.go +++ b/pkg/sql/parsers/dialect/mysql/mysql_sql_test.go @@ -3437,6 +3437,50 @@ var ( input: "create table t1 (id bigint primary key, embedding vecf32(3), payload json, tags array(varchar(20)))", output: "create table t1 (id bigint primary key, embedding vecf32(3), payload json, tags array(varchar(20)))", }, + { + input: "create table t1(a vecbf16(3), b vecf16(3), c vecint8(3))", + output: "create table t1 (a vecbf16(3), b vecf16(3), c vecint8(3))", + }, + { + input: "create table t1(a vecbf16(128), b vecf16(65535), c vecint8(1))", + output: "create table t1 (a vecbf16(128), b vecf16(65535), c vecint8(1))", + }, + { + input: "create table t1(a vecuint8(3))", + output: "create table t1 (a vecuint8(3))", + }, + { + input: "create table t1(a vecuint8(128), b vecuint8(65535), c vecuint8(1))", + output: "create table t1 (a vecuint8(128), b vecuint8(65535), c vecuint8(1))", + }, + { + input: "select cast('[1,2,3]' as vecbf16(3))", + output: "select cast([1,2,3] as vecbf16(3))", + }, + { + input: "select cast('[1,2,3]' as vecf16(3))", + output: "select cast([1,2,3] as vecf16(3))", + }, + { + input: "select cast('[1,2,3]' as vecint8(3))", + output: "select cast([1,2,3] as vecint8(3))", + }, + { + input: "select cast(b as vecint8(3)) from t1", + output: "select cast(b as vecint8(3)) from t1", + }, + { + input: "select cast('[1,2,3]' as vecuint8(3))", + output: "select cast([1,2,3] as vecuint8(3))", + }, + { + input: "select cast(b as vecuint8(3)) from t1", + output: "select cast(b as vecuint8(3)) from t1", + }, + { + input: "select l2_distance(a, b) from t1", + output: "select l2_distance(a, b) from t1", + }, { input: "alter table tbl1 drop constraint fk_name", output: "alter table tbl1 drop foreign key fk_name", @@ -3593,6 +3637,15 @@ var ( input: "select * from t1 where MATCH (body, title) AGAINST ('abc dfc ghc')", output: "select * from t1 where MATCH (body, title) AGAINST (abc dfc ghc)", }, + { + // BM25() must deparse back to BM25() (not MATCH), and carries no mode. + input: "select * from t1 where BM25 (body) AGAINST ('abc dfc')", + output: "select * from t1 where BM25 (body) AGAINST (abc dfc)", + }, + { + input: "select BM25 (body) AGAINST ('abc') as score from t1", + output: "select BM25 (body) AGAINST (abc) as score from t1", + }, { input: "select * from t1 where MATCH (body, title) AGAINST ('abc- +abc' IN BOOLEAN MODE)", output: "select * from t1 where MATCH (body, title) AGAINST (abc- +abc IN BOOLEAN MODE)", diff --git a/pkg/sql/parsers/tree/alter.go b/pkg/sql/parsers/tree/alter.go index 48467574d1d31..3459663b3270e 100644 --- a/pkg/sql/parsers/tree/alter.go +++ b/pkg/sql/parsers/tree/alter.go @@ -915,6 +915,7 @@ type AlterOptionAlterReIndex struct { BitsPerCode int64 Async bool ForceSync bool + Merge bool AutoUpdate bool Day int64 Hour int64 @@ -947,6 +948,7 @@ func NewAlterOptionAlterReIndex(name Identifier, option *IndexOption) *AlterOpti a.BitsPerCode = option.BitsPerCode a.Async = option.Async a.ForceSync = option.ForceSync + a.Merge = option.Merge a.AutoUpdate = option.AutoUpdate a.Day = option.Day a.Hour = option.Hour @@ -1018,6 +1020,9 @@ func (node *AlterOptionAlterReIndex) Format(ctx *FmtCtx) { if node.ForceSync { ctx.WriteString(" force_sync") } + if node.Merge { + ctx.WriteString(" merge") + } } func (node AlterOptionAlterReIndex) TypeName() string { return "tree.AlterOptionAlterReIndex" } diff --git a/pkg/sql/parsers/tree/create.go b/pkg/sql/parsers/tree/create.go index 8ecf0c0484b50..748ab8ed0f21a 100644 --- a/pkg/sql/parsers/tree/create.go +++ b/pkg/sql/parsers/tree/create.go @@ -2067,6 +2067,8 @@ func (it IndexType) ToString() string { return "cagra" case INDEX_TYPE_IVFPQ: return "ivfpq" + case INDEX_TYPE_BM25: + return "bm25" case INDEX_TYPE_INVALID: return "" default: @@ -2087,6 +2089,7 @@ const ( INDEX_TYPE_HNSW INDEX_TYPE_CAGRA INDEX_TYPE_IVFPQ + INDEX_TYPE_BM25 ) type VisibleType int @@ -2125,9 +2128,11 @@ type IndexOption struct { BitsPerCode int64 Async bool ForceSync bool + Merge bool AutoUpdate bool Day int64 Hour int64 + Second int64 IntermediateGraphDegree int64 GraphDegree int64 Quantization string @@ -2146,7 +2151,7 @@ func (node *IndexOption) Format(ctx *FmtCtx) { node.AlgoParamList != 0 || node.AlgoParamVectorOpType != "" || node.AlgoParamM != 0 || node.HnswEfConstruction != 0 || node.HnswEfSearch != 0 || node.AutoUpdate || node.Day != 0 || - node.Hour != 0 || + node.Hour != 0 || node.Second != 0 || node.IntermediateGraphDegree != 0 || node.GraphDegree != 0 || node.Quantization != "" || node.DistributionMode != "" || node.BitsPerCode != 0 || node.ITopkSize != 0 || @@ -2204,6 +2209,9 @@ func (node *IndexOption) Format(ctx *FmtCtx) { if node.ForceSync { ctx.WriteString("FORCE_SYNC ") } + if node.Merge { + ctx.WriteString("MERGE ") + } if node.AutoUpdate { ctx.WriteString("AUTO_UPDATE=TRUE ") } @@ -2217,6 +2225,11 @@ func (node *IndexOption) Format(ctx *FmtCtx) { ctx.WriteString(strconv.FormatInt(node.Hour, 10)) ctx.WriteByte(' ') } + if node.Second != 0 { + ctx.WriteString("SECOND ") + ctx.WriteString(strconv.FormatInt(node.Second, 10)) + ctx.WriteByte(' ') + } if node.IntermediateGraphDegree != 0 { ctx.WriteString("INTERMEDIATE_GRAPH_DEGREE ") ctx.WriteString(strconv.FormatInt(node.IntermediateGraphDegree, 10)) diff --git a/pkg/sql/parsers/tree/expr.go b/pkg/sql/parsers/tree/expr.go index 84aa5c6b2e5bd..f01a1bc87c668 100644 --- a/pkg/sql/parsers/tree/expr.go +++ b/pkg/sql/parsers/tree/expr.go @@ -1873,6 +1873,12 @@ type FullTextMatchExpr struct { Pattern string Mode FullTextSearchType + + // IsBm25 marks a BM25(col) AGAINST('query') expression — the distinct + // ranked-retrieval surface of the bm25 index. It binds to the bm25_match + // function (not fulltext_match) so the planner routes it to bm25_search, and + // it carries no mode (bm25 is always ranked bag-of-words; Mode stays DEFAULT). + IsBm25 bool } func (node *FullTextSearchType) ToString() string { @@ -1917,8 +1923,27 @@ func NewFullTextMatchFuncExpression(columns []*KeyPart, pattern string, mode Ful return e, nil } +// NewBm25MatchFuncExpression builds a BM25(col) AGAINST('query') expression — the +// bm25 index's ranked-retrieval surface. It carries no mode (always DEFAULT ranked +// bag-of-words) and binds to the bm25_match function. +func NewBm25MatchFuncExpression(columns []*KeyPart, pattern string) (*FullTextMatchExpr, error) { + + e := &FullTextMatchExpr{KeyParts: columns, Pattern: pattern, Mode: FULLTEXT_DEFAULT, IsBm25: true} + if err := e.Valid(); err != nil { + return nil, err + } + return e, nil +} + func (node *FullTextMatchExpr) Format(ctx *FmtCtx) { - ctx.WriteString("MATCH (") + // A BM25() expression must deparse back to BM25(), not MATCH() — otherwise a + // deparse+reparse (e.g. CTAS's CreateAsSelectSql) would lose IsBm25 and re-bind + // to the classic fulltext index. BM25 carries no mode (always ranked). + if node.IsBm25 { + ctx.WriteString("BM25 (") + } else { + ctx.WriteString("MATCH (") + } for i, k := range node.KeyParts { if i > 0 { ctx.WriteString(", ") @@ -1929,7 +1954,7 @@ func (node *FullTextMatchExpr) Format(ctx *FmtCtx) { ctx.WriteString("AGAINST (") ctx.WriteString(node.Pattern) - if node.Mode != FULLTEXT_DEFAULT { + if !node.IsBm25 && node.Mode != FULLTEXT_DEFAULT { ctx.WriteString(" ") ctx.WriteString(node.Mode.ToString()) } diff --git a/pkg/sql/parsers/tree/fulltext_match_test.go b/pkg/sql/parsers/tree/fulltext_match_test.go new file mode 100644 index 0000000000000..edb600fe567f2 --- /dev/null +++ b/pkg/sql/parsers/tree/fulltext_match_test.go @@ -0,0 +1,72 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tree + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/matrixorigin/matrixone/pkg/sql/parsers/dialect" +) + +func TestBm25MatchFuncExpression(t *testing.T) { + cols := []*KeyPart{{ColName: NewUnresolvedColName("body")}} + + // BM25(): IsBm25 set, mode fixed to DEFAULT, deparses as BM25 with no mode. + bm, err := NewBm25MatchFuncExpression(cols, "apple banana") + require.NoError(t, err) + require.True(t, bm.IsBm25) + require.Equal(t, FULLTEXT_DEFAULT, bm.Mode) + + ctx := NewFmtCtx(dialect.MYSQL, WithQuoteString(true)) + bm.Format(ctx) + require.Equal(t, "BM25 (body) AGAINST (apple banana)", ctx.String()) +} + +func TestFullTextMatchFuncExpressionFormat(t *testing.T) { + cols := []*KeyPart{{ColName: NewUnresolvedColName("body")}} + + // classic MATCH: IsBm25 false, deparses as MATCH and carries its mode. + ft, err := NewFullTextMatchFuncExpression(cols, "apple", FULLTEXT_BOOLEAN) + require.NoError(t, err) + require.False(t, ft.IsBm25) + + ctx := NewFmtCtx(dialect.MYSQL, WithQuoteString(true)) + ft.Format(ctx) + require.Equal(t, "MATCH (body) AGAINST (apple IN BOOLEAN MODE)", ctx.String()) + + // default mode omits the mode clause. + def, _ := NewFullTextMatchFuncExpression(cols, "apple", FULLTEXT_DEFAULT) + ctx2 := NewFmtCtx(dialect.MYSQL, WithQuoteString(true)) + def.Format(ctx2) + require.Equal(t, "MATCH (body) AGAINST (apple)", ctx2.String()) +} + +func TestFullTextMatchExprValid(t *testing.T) { + cols := []*KeyPart{{ColName: NewUnresolvedColName("body")}} + + // empty column list is rejected + _, err := NewBm25MatchFuncExpression(nil, "apple") + require.Error(t, err) + + // empty pattern is rejected + _, err = NewBm25MatchFuncExpression(cols, "") + require.Error(t, err) + + // same for the classic constructor + _, err = NewFullTextMatchFuncExpression(nil, "apple", FULLTEXT_DEFAULT) + require.Error(t, err) +} diff --git a/pkg/sql/parsers/tree/types.go b/pkg/sql/parsers/tree/types.go index 8ee2d778c36c3..837e0eb777a55 100644 --- a/pkg/sql/parsers/tree/types.go +++ b/pkg/sql/parsers/tree/types.go @@ -218,7 +218,7 @@ func (node *InternalType) Format(ctx *FmtCtx) { ctx.WriteString(strconv.FormatInt(int64(node.DisplayWith), 10)) ctx.WriteByte(')') } - case "vecf32", "vecf64": + case "vecf32", "vecf64", "vecbf16", "vecf16", "vecint8", "vecuint8": if node.DisplayWith >= 0 { // Prints 'vecf32(4)' ctx.WriteByte('(') diff --git a/pkg/sql/plan/apply_indices.go b/pkg/sql/plan/apply_indices.go index 4c0197b35fe7e..5f549634cd72b 100644 --- a/pkg/sql/plan/apply_indices.go +++ b/pkg/sql/plan/apply_indices.go @@ -610,23 +610,23 @@ func (builder *QueryBuilder) applyIndicesForProject(nodeID int32, projNode *plan } if aggNode != nil { - // agg node and scan node present - // get the list of filter that is fulltext_match func + // agg node and scan node present. + // get the list of filter that is a match func (fulltext_match / bm25_match) filterids, filter_ftidxs := builder.getFullTextMatchFiltersFromScanNode(scanNode) - // apply fulltext indices when fulltext_match exists + // apply the match indices (one unified pass handles a mix of MATCH + BM25) if len(filterids) > 0 { return builder.applyIndicesForAggUsingFullTextIndex(nodeID, projNode, aggNode, scanNode, filterids, filter_ftidxs, colRefCnt, idxColMap) } } else { - // get the list of project that is fulltext_match func + // get the list of project that is a match func (fulltext_match / bm25_match) projids, proj_ftidxs := builder.getFullTextMatchFromProject(projNode, scanNode) - // get the list of filter that is fulltext_match func + // get the list of filter that is a match func (fulltext_match / bm25_match) filterids, filter_ftidxs := builder.getFullTextMatchFiltersFromScanNode(scanNode) - // apply fulltext indices when fulltext_match exists + // apply the match indices (one unified pass handles a mix of MATCH + BM25) if len(filterids) > 0 || len(projids) > 0 { return builder.applyIndicesForProjectionUsingFullTextIndex(nodeID, projNode, sortNode, scanNode, filterids, filter_ftidxs, projids, proj_ftidxs, colRefCnt, idxColMap) diff --git a/pkg/sql/plan/apply_indices_bm25.go b/pkg/sql/plan/apply_indices_bm25.go new file mode 100644 index 0000000000000..b281e93f2eba3 --- /dev/null +++ b/pkg/sql/plan/apply_indices_bm25.go @@ -0,0 +1,176 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file holds the bm25-specific pieces of the MATCH/BM25 query rewrite. bm25 has +// its OWN query verb — BM25(col) AGAINST('query') — which binds to the distinct +// bm25_match function (NOT fulltext_match), so the planner disambiguates it from a +// classic MATCH by function name. The shared join/sort/limit-pushdown machinery lives +// in apply_indices_fulltext.go and drives BOTH: its collectors resolve bm25_match via +// findMatchBm25Index (here) alongside fulltext_match, and its one join loop dispatches +// the per-match TVF by index algo — buildBm25SearchTableFunc (here) for bm25, +// fulltext_index_scan for classic — into a single join with one combined score sort. +// So a query mixing BM25() and MATCH() is served by one pass. This file therefore only +// contains what is genuinely bm25-specific: the bm25_search TVF builder, its hidden- +// table resolver, and the bm25 index finder. +package plan + +import ( + "encoding/json" + "strings" + + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" +) + +const bm25_search_func_name = "bm25_search" + +// buildBm25SearchTableFunc builds the bm25_search TVF AST for a BM25(...) match +// resolved to a bm25 ranked-retrieval index. Args: [param="", cfg{db,index,metadata} +// JSON, pattern]. The TVF tokenizes the pattern as bag-of-words and answers a BM25 +// top-K walk, emitting (doc_id, score) — the same shape fulltext_index_scan emits, so +// the downstream join/projection/limit is unchanged. +func (builder *QueryBuilder) buildBm25SearchTableFunc(scanNode *plan.Node, idxdef *plan.IndexDef, pattern string, aliasName string) (*tree.AliasedTableExpr, error) { + // bm25 is a position-free bag-of-words BM25 index with a single, mode-free query + // surface: BM25(col) AGAINST('query') always answers ranked bag-of-words top-K. + // (Boolean / phrase / query-expansion — which need term positions — are simply + // not expressible via BM25(); use a classic fulltext MATCH index for those.) + storeTbl, metaTbl, ok := builder.findBm25IndexTables(scanNode, idxdef) + if !ok { + return nil, moerr.NewInternalErrorf(builder.GetContext(), + "bm25 index %q: storage/metadata tables not found (index may be partially materialized); reindex required", + idxdef.IndexName) + } + + // json.Marshal so a schema name containing a double-quote/backslash is escaped + // (the search side sonic.Unmarshal's it). + cfgBytes, err := json.Marshal(map[string]string{ + "db": scanNode.ObjRef.SchemaName, + "index": storeTbl, + "metadata": metaTbl, + }) + if err != nil { + return nil, err + } + cfg := string(cfgBytes) + + bm25Func := tree.NewCStr(bm25_search_func_name, 1) + var exprs tree.Exprs + exprs = append(exprs, tree.NewNumVal[string]("", "", false, tree.P_char)) + exprs = append(exprs, tree.NewNumVal[string](cfg, cfg, false, tree.P_char)) + exprs = append(exprs, tree.NewNumVal[string](pattern, pattern, false, tree.P_char)) + name := tree.NewUnresolvedName(bm25Func) + + return &tree.AliasedTableExpr{ + Expr: &tree.TableFunction{ + Func: &tree.FuncExpr{ + Func: tree.FuncName2ResolvableFunctionReference(name), + FuncName: bm25Func, + Exprs: exprs, + Type: tree.FUNC_TYPE_TABLE, + }, + }, + As: tree.AliasClause{Alias: tree.Identifier(aliasName)}, + }, nil +} + +// findBm25IndexTables resolves the storage + metadata hidden tables of a bm25 +// index — the two sibling defs sharing the storage def's IndexName. ok is false +// if either is missing (partial/restored catalog). +func (builder *QueryBuilder) findBm25IndexTables(scanNode *plan.Node, idxdef *plan.IndexDef) (storeTbl string, metaTbl string, ok bool) { + if scanNode == nil || scanNode.TableDef == nil || idxdef == nil { + return "", "", false + } + for _, idx := range scanNode.TableDef.Indexes { + if idx == nil || idx.IndexName != idxdef.IndexName { + continue + } + switch idx.IndexAlgoTableType { + case catalog.Bm25Index_TblType_Storage: + storeTbl = idx.IndexTableName + case catalog.Bm25Index_TblType_Metadata: + metaTbl = idx.IndexTableName + } + } + return storeTbl, metaTbl, storeTbl != "" && metaTbl != "" +} + +// findMatchBm25Index resolves a bm25_match(...) function — the bound form of +// BM25(col) AGAINST('q') — to a bm25 storage index def on the scan's columns. +// Because BM25() binds to its own function (never fulltext_match), the function name +// alone disambiguates it from a classic MATCH; no mode gating is needed (BM25 has no +// modes) and a coexisting classic fulltext index on the same column is irrelevant here. +func (builder *QueryBuilder) findMatchBm25Index(fn *plan.Function, scanNode *plan.Node) *plan.IndexDef { + if fn == nil || scanNode == nil || scanNode.TableDef == nil || len(scanNode.BindingTags) == 0 { + return nil + } + if len(fn.Args) < 3 || fn.Args[0].GetLit() == nil || fn.Args[1].GetLit() == nil { + return nil + } + if scanNode.TableDef.Pkey == nil || scanNode.TableDef.Pkey.PkeyColName == "" { + return nil + } + + scanTag := scanNode.BindingTags[0] + argColNames := make([]string, 0, len(fn.Args)-2) + for j := 2; j < len(fn.Args); j++ { + col := fn.Args[j].GetCol() + if col == nil || col.RelPos != scanTag { + return nil + } + + colName := col.Name + if colName == "" { + if col.ColPos < 0 || int(col.ColPos) >= len(scanNode.TableDef.Cols) { + return nil + } + colName = scanNode.TableDef.Cols[col.ColPos].Name + } + argColNames = append(argColNames, colName) + } + + nargs := len(fn.Args) - 2 + for _, idx := range scanNode.TableDef.Indexes { + if idx == nil || !idx.TableExist { + continue + } + // Match the bm25 storage def as the single representative — the metadata + // sibling is resolved later in buildBm25SearchTableFunc. + if idx.GetIndexAlgo() != catalog.MoIndexBm25Algo.ToString() || + idx.IndexAlgoTableType != catalog.Bm25Index_TblType_Storage { + continue + } + if len(idx.Parts) != nargs { + continue + } + + nfound := 0 + for _, p := range idx.Parts { + partName := catalog.ResolveAlias(p) + for _, colName := range argColNames { + if strings.EqualFold(partName, colName) || strings.EqualFold(p, colName) { + // found + nfound++ + break + } + } + } + + if nfound == nargs && nfound == len(idx.Parts) { + return idx + } + } + return nil +} diff --git a/pkg/sql/plan/apply_indices_bm25_test.go b/pkg/sql/plan/apply_indices_bm25_test.go new file mode 100644 index 0000000000000..7aef0d2c6e112 --- /dev/null +++ b/pkg/sql/plan/apply_indices_bm25_test.go @@ -0,0 +1,211 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plan + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/container/types" + planpb "github.com/matrixorigin/matrixone/pkg/pb/plan" +) + +// makeBm25TestTableDef builds a table with an int64 pk, a text column, and a bm25 +// index (its storage + metadata sibling defs on the text column). +func makeBm25TestTableDef(name string, withBm25 bool) *planpb.TableDef { + td := &planpb.TableDef{ + Name: name, + Cols: []*planpb.ColDef{ + {Name: "id", Typ: planpb.Type{Id: int32(types.T_int64)}}, + {Name: "body", Typ: planpb.Type{Id: int32(types.T_text)}}, + }, + Name2ColIndex: map[string]int32{"id": 0, "body": 1}, + Pkey: &planpb.PrimaryKeyDef{PkeyColName: "id", Names: []string{"id"}}, + } + if withBm25 { + td.Indexes = []*planpb.IndexDef{ + { + IndexName: "bm25idx", + IndexAlgo: catalog.MoIndexBm25Algo.ToString(), + IndexAlgoTableType: catalog.Bm25Index_TblType_Storage, + IndexTableName: "__mo_bm25_store_" + name, + Parts: []string{"body"}, + TableExist: true, + }, + { + IndexName: "bm25idx", + IndexAlgo: catalog.MoIndexBm25Algo.ToString(), + IndexAlgoTableType: catalog.Bm25Index_TblType_Metadata, + IndexTableName: "__mo_bm25_meta_" + name, + Parts: []string{"body"}, + TableExist: true, + }, + } + } + return td +} + +// makeBm25MatchExpr builds a bound bm25_match(pattern, mode=0, cols...) function expr +// (the form BM25(col) AGAINST('pattern') binds to). +func makeBm25MatchExpr(pattern string, tableDef *planpb.TableDef, tag int32, colPositions []int32) *planpb.Expr { + args := []*planpb.Expr{ + makePlan2StringConstExprWithType(pattern, false), + makePlan2Int64ConstExprWithType(0), + } + for _, pos := range colPositions { + args = append(args, ftjColExpr(tableDef, tag, pos)) + } + return &planpb.Expr{ + Typ: planpb.Type{Id: int32(types.T_float32)}, + Expr: &planpb.Expr_F{F: &planpb.Function{ + Func: &planpb.ObjectRef{ObjName: "bm25_match"}, + Args: args, + }}, + } +} + +func TestFindMatchBm25Index(t *testing.T) { + builder := NewQueryBuilder(planpb.Query_SELECT, NewMockCompilerContext(true), false, true) + td := makeBm25TestTableDef("t", true) + tag := builder.genNewBindTag() + scan := makeFullTextJoinTestScan(td, tag, nil) + + // matched -> the storage index def + idx := builder.findMatchBm25Index(makeBm25MatchExpr("apple", td, tag, []int32{1}).GetF(), scan) + require.NotNil(t, idx) + require.Equal(t, catalog.Bm25Index_TblType_Storage, idx.IndexAlgoTableType) + + // no bm25 index on the table -> nil + tdNo := makeBm25TestTableDef("t", false) + scanNo := makeFullTextJoinTestScan(tdNo, tag, nil) + require.Nil(t, builder.findMatchBm25Index(makeBm25MatchExpr("apple", tdNo, tag, []int32{1}).GetF(), scanNo)) + + // column referencing a different binding tag (cross-table) -> nil + crossTag := builder.genNewBindTag() + require.Nil(t, builder.findMatchBm25Index(makeBm25MatchExpr("apple", td, crossTag, []int32{1}).GetF(), scan)) + + // dynamic (non-literal) pattern -> nil + dyn := makeBm25MatchExpr("apple", td, tag, []int32{1}) + textTyp := types.T_text.ToType() + dyn.GetF().Args[0] = &planpb.Expr{ + Typ: makePlan2Type(&textTyp), + Expr: &planpb.Expr_P{P: &planpb.ParamRef{Pos: 0}}, + } + require.Nil(t, builder.findMatchBm25Index(dyn.GetF(), scan)) + + // too few args -> nil + short := makeBm25MatchExpr("apple", td, tag, nil) + require.Nil(t, builder.findMatchBm25Index(short.GetF(), scan)) +} + +func TestFindBm25IndexTables(t *testing.T) { + builder := NewQueryBuilder(planpb.Query_SELECT, NewMockCompilerContext(true), false, true) + td := makeBm25TestTableDef("t", true) + tag := builder.genNewBindTag() + scan := makeFullTextJoinTestScan(td, tag, nil) + storeDef := td.Indexes[0] + + store, meta, ok := builder.findBm25IndexTables(scan, storeDef) + require.True(t, ok) + require.Equal(t, "__mo_bm25_store_t", store) + require.Equal(t, "__mo_bm25_meta_t", meta) + + // drop the metadata sibling -> not ok + tdPartial := makeBm25TestTableDef("t", true) + tdPartial.Indexes = tdPartial.Indexes[:1] // storage only + scanPartial := makeFullTextJoinTestScan(tdPartial, tag, nil) + _, _, ok = builder.findBm25IndexTables(scanPartial, tdPartial.Indexes[0]) + require.False(t, ok) + + // nil inputs + _, _, ok = builder.findBm25IndexTables(nil, storeDef) + require.False(t, ok) +} + +func TestBuildBm25SearchTableFunc(t *testing.T) { + builder := NewQueryBuilder(planpb.Query_SELECT, NewMockCompilerContext(true), false, true) + td := makeBm25TestTableDef("t", true) + tag := builder.genNewBindTag() + scan := makeFullTextJoinTestScan(td, tag, nil) + storeDef := td.Indexes[0] + + tf, err := builder.buildBm25SearchTableFunc(scan, storeDef, "apple", "mo_bm25_alias_0") + require.NoError(t, err) + require.NotNil(t, tf) + require.Equal(t, "mo_bm25_alias_0", string(tf.As.Alias)) + + // missing sibling tables -> error + tdPartial := makeBm25TestTableDef("t", true) + tdPartial.Indexes = tdPartial.Indexes[:1] + scanPartial := makeFullTextJoinTestScan(tdPartial, tag, nil) + _, err = builder.buildBm25SearchTableFunc(scanPartial, tdPartial.Indexes[0], "apple", "a") + require.Error(t, err) +} + +// --- apply_indices_match.go shared helpers --- + +func TestEqualsMatchFunc(t *testing.T) { + builder := NewQueryBuilder(planpb.Query_SELECT, NewMockCompilerContext(true), false, true) + td := makeBm25TestTableDef("t", true) + tag := builder.genNewBindTag() + + a := makeBm25MatchExpr("apple", td, tag, []int32{1}).GetF() + b := makeBm25MatchExpr("apple", td, tag, []int32{1}).GetF() + require.True(t, builder.equalsMatchFunc(a, b)) + + // different pattern + c := makeBm25MatchExpr("banana", td, tag, []int32{1}).GetF() + require.False(t, builder.equalsMatchFunc(a, c)) + + // different function name (bm25_match vs fulltext_match) must NOT be equal + ft := makeFullTextMatchExpr("apple", 0, td, tag, []int32{1}).GetF() + require.False(t, builder.equalsMatchFunc(a, ft)) + + // different arg count + d := makeBm25MatchExpr("apple", td, tag, nil).GetF() + require.False(t, builder.equalsMatchFunc(a, d)) +} + +func TestFindEqualMatchFunc(t *testing.T) { + builder := NewQueryBuilder(planpb.Query_SELECT, NewMockCompilerContext(true), false, true) + td := makeBm25TestTableDef("t", true) + tag := builder.genNewBindTag() + + projNode := &planpb.Node{ProjectList: []*planpb.Expr{makeBm25MatchExpr("apple", td, tag, []int32{1})}} + scanNode := &planpb.Node{FilterList: []*planpb.Expr{makeBm25MatchExpr("apple", td, tag, []int32{1})}} + + eqmap := builder.findEqualMatchFunc(projNode, scanNode, []int32{0}, []int32{0}) + require.Equal(t, map[int32]int32{0: 0}, eqmap) + + // non-equal (different pattern) -> empty map + scanNode2 := &planpb.Node{FilterList: []*planpb.Expr{makeBm25MatchExpr("banana", td, tag, []int32{1})}} + require.Empty(t, builder.findEqualMatchFunc(projNode, scanNode2, []int32{0}, []int32{0})) +} + +func TestMatchRewriteContextNodeID(t *testing.T) { + builder := NewQueryBuilder(planpb.Query_SELECT, NewMockCompilerContext(true), false, true) + ctx := NewBindContext(builder, nil) + td := makeBm25TestTableDef("t", true) + tag := builder.genNewBindTag() + scanID := builder.appendNode(makeFullTextJoinTestScan(td, tag, nil), ctx) + scan := builder.qry.Nodes[scanID] + + // preferred id valid -> returned as-is + require.Equal(t, scanID, builder.matchRewriteContextNodeID(scanID, scan)) + // preferred id invalid (-1) -> falls back to the scan node's id + require.Equal(t, scanID, builder.matchRewriteContextNodeID(-1, scan)) +} diff --git a/pkg/sql/plan/apply_indices_cagra.go b/pkg/sql/plan/apply_indices_cagra.go index eb83eaa63edb8..f227027b49a57 100644 --- a/pkg/sql/plan/apply_indices_cagra.go +++ b/pkg/sql/plan/apply_indices_cagra.go @@ -34,6 +34,7 @@ type cagraIndexContext struct { vecLitArg *plan.Expr origFuncName string partPos int32 + partType plan.Type pkPos int32 pkType plan.Type params string @@ -84,6 +85,7 @@ func (builder *QueryBuilder) prepareCagraIndexContext(vecCtx *vectorSortContext, keyPart := idxDef.Parts[0] partPos := vecCtx.scanNode.TableDef.Name2ColIndex[keyPart] + partType := vecCtx.scanNode.TableDef.Cols[partPos].Typ _, vecLitArg, found := builder.getArgsFromDistFn(vecCtx.distFnExpr, partPos) if !found { return nil, nil @@ -114,6 +116,7 @@ func (builder *QueryBuilder) prepareCagraIndexContext(vecCtx *vectorSortContext, vecLitArg: vecLitArg, origFuncName: origFuncName, partPos: partPos, + partType: partType, pkPos: pkPos, pkType: pkType, params: idxDef.IndexAlgoParams, @@ -141,7 +144,7 @@ func (builder *QueryBuilder) applyIndicesForSortUsingCagra(nodeID int32, vecCtx return nodeID, err } - tblCfgStr := fmt.Sprintf(`{"db": "%s", "src": "%s", "metadata":"%s", "index":"%s", "threads_search": %d, "orig_func_name": "%s", "batch_window": %d, "gpu_multi_simulation": %d}`, + tblCfgStr := fmt.Sprintf(`{"db": "%s", "src": "%s", "metadata":"%s", "index":"%s", "threads_search": %d, "orig_func_name": "%s", "batch_window": %d, "gpu_multi_simulation": %d, "parttype": %d}`, scanNode.ObjRef.SchemaName, scanNode.TableDef.Name, cagraCtx.metaDef.IndexTableName, @@ -149,7 +152,8 @@ func (builder *QueryBuilder) applyIndicesForSortUsingCagra(nodeID int32, vecCtx cagraCtx.nThread, cagraCtx.origFuncName, cagraCtx.batchWindow, - cagraCtx.gpuMultiSim) + cagraCtx.gpuMultiSim, + cagraCtx.partType.Id) // Predicate pushdown on INCLUDE columns and the primary key: peel // filters that reference only INCLUDE columns (or the PK, routed to diff --git a/pkg/sql/plan/apply_indices_fulltext.go b/pkg/sql/plan/apply_indices_fulltext.go index 95609677406ca..7ce06efe5d3ec 100644 --- a/pkg/sql/plan/apply_indices_fulltext.go +++ b/pkg/sql/plan/apply_indices_fulltext.go @@ -60,7 +60,7 @@ func (builder *QueryBuilder) applyIndicesForProjectionUsingFullTextIndex(nodeID ctx := builder.ctxByNode[nodeID] // check equal fulltext_match func and only compute once for equal function() - eqmap := builder.findEqualFullTextMatchFunc(projNode, scanNode, projids, filterids) + eqmap := builder.findEqualMatchFunc(projNode, scanNode, projids, filterids) var paginationLimit, paginationOffset *plan.Expr if projNode.Limit != nil || projNode.Offset != nil { paginationLimit, paginationOffset = projNode.Limit, projNode.Offset @@ -270,41 +270,54 @@ func (builder *QueryBuilder) applyJoinFullTextIndices(nodeID int32, projNode *pl for i := 0; i < len(ft_filters); i++ { ftidxscan := ft_filters[i] idxdef := indexDefs[i] - idxtblname := fmt.Sprintf("`%s`.`%s`", scanNode.ObjRef.SchemaName, idxdef.IndexTableName) - srctblname := fmt.Sprintf("`%s`.`%s`", scanNode.ObjRef.SchemaName, scanNode.TableDef.Name) fn := ftidxscan.GetF() pattern := fn.Args[0].GetLit().GetSval() mode := fn.Args[1].GetLit().GetI64Val() - fulltext_func := tree.NewCStr(fulltext_index_scan_func_name, 1) alias_name := fmt.Sprintf("mo_fulltext_alias_%d", i) if projNode == nil { alias_name = fmt.Sprintf("mo_fulltext_alias_%d_%d", scanNode.NodeId, i) } - params := idxdef.IndexAlgoParams - - var exprs tree.Exprs - exprs = append(exprs, tree.NewNumVal[string](params, params, false, tree.P_char)) - exprs = append(exprs, tree.NewNumVal[string](srctblname, srctblname, false, tree.P_char)) - exprs = append(exprs, tree.NewNumVal[string](idxtblname, idxtblname, false, tree.P_char)) - exprs = append(exprs, tree.NewNumVal[string](pattern, pattern, false, tree.P_char)) - exprs = append(exprs, tree.NewNumVal[int64](mode, strconv.FormatInt(mode, 10), false, tree.P_int64)) - - name := tree.NewUnresolvedName(fulltext_func) - - // TableFuncion AST - tmpTableFunc := &tree.AliasedTableExpr{ - Expr: &tree.TableFunction{ - Func: &tree.FuncExpr{ - Func: tree.FuncName2ResolvableFunctionReference(name), - FuncName: fulltext_func, - Exprs: exprs, - Type: tree.FUNC_TYPE_TABLE, + + // One unified loop over ALL match predicates on this scan, so a mix of + // BM25(...) (bm25_match -> bm25_search) and MATCH(...) (fulltext_match -> + // fulltext_index_scan) on the same query chains into ONE join structure with + // ONE combined score sort. Dispatch the per-match TVF by the resolved index's + // algo; both TVFs emit the same (doc_id, score) shape so the rest is identical. + var tmpTableFunc *tree.AliasedTableExpr + if idxdef.IndexAlgo == catalog.MoIndexBm25Algo.ToString() { + var berr error + tmpTableFunc, berr = builder.buildBm25SearchTableFunc(scanNode, idxdef, pattern, alias_name) + if berr != nil { + return -1, nil, nil, berr + } + } else { + idxtblname := fmt.Sprintf("`%s`.`%s`", scanNode.ObjRef.SchemaName, idxdef.IndexTableName) + srctblname := fmt.Sprintf("`%s`.`%s`", scanNode.ObjRef.SchemaName, scanNode.TableDef.Name) + fulltext_func := tree.NewCStr(fulltext_index_scan_func_name, 1) + params := idxdef.IndexAlgoParams + + var exprs tree.Exprs + exprs = append(exprs, tree.NewNumVal[string](params, params, false, tree.P_char)) + exprs = append(exprs, tree.NewNumVal[string](srctblname, srctblname, false, tree.P_char)) + exprs = append(exprs, tree.NewNumVal[string](idxtblname, idxtblname, false, tree.P_char)) + exprs = append(exprs, tree.NewNumVal[string](pattern, pattern, false, tree.P_char)) + exprs = append(exprs, tree.NewNumVal[int64](mode, strconv.FormatInt(mode, 10), false, tree.P_int64)) + + name := tree.NewUnresolvedName(fulltext_func) + tmpTableFunc = &tree.AliasedTableExpr{ + Expr: &tree.TableFunction{ + Func: &tree.FuncExpr{ + Func: tree.FuncName2ResolvableFunctionReference(name), + FuncName: fulltext_func, + Exprs: exprs, + Type: tree.FUNC_TYPE_TABLE, + }, }, - }, - As: tree.AliasClause{ - Alias: tree.Identifier(alias_name), - }, + As: tree.AliasClause{ + Alias: tree.Identifier(alias_name), + }, + } } curr_ftnode_id, err := builder.buildTable(tmpTableFunc, ctx, -1, nil) @@ -669,7 +682,7 @@ func (builder *QueryBuilder) applyFullTextFiltersForScanInJoin(nodeID int32, sca return scanNode.NodeId, false, nil } - ctxNodeID := builder.fullTextRewriteContextNodeID(nodeID, scanNode) + ctxNodeID := builder.matchRewriteContextNodeID(nodeID, scanNode) newNodeID, _, _, err := builder.applyJoinFullTextIndices( ctxNodeID, nil, @@ -696,68 +709,6 @@ func (builder *QueryBuilder) applyFullTextFiltersForScanInJoin(nodeID int32, sca return newNodeID, true, nil } -func (builder *QueryBuilder) fullTextRewriteContextNodeID(preferredNodeID int32, scanNode *plan.Node) int32 { - if preferredNodeID >= 0 && int(preferredNodeID) < len(builder.ctxByNode) && builder.ctxByNode[preferredNodeID] != nil { - return preferredNodeID - } - if scanNode != nil && scanNode.NodeId >= 0 && int(scanNode.NodeId) < len(builder.ctxByNode) && builder.ctxByNode[scanNode.NodeId] != nil { - return scanNode.NodeId - } - return preferredNodeID -} - -func (builder *QueryBuilder) equalsFullTextMatchFunc(fn1 *plan.Function, fn2 *plan.Function) bool { - - nargs1 := len(fn1.Args) - nargs2 := len(fn2.Args) - - if nargs1 != nargs2 { - return false - } - - // check search pattern and mode - var pattern1, pattern2 string - var mode1, mode2 int64 - - pattern1 = fn1.Args[0].GetLit().GetSval() - mode1 = fn1.Args[1].GetLit().GetI64Val() - - pattern2 = fn2.Args[0].GetLit().GetSval() - mode2 = fn2.Args[1].GetLit().GetI64Val() - - if pattern1 != pattern2 || mode1 != mode2 { - return false - } - - // check index parts - for i := 2; i < nargs1; i++ { - if !strings.EqualFold(fn1.Args[i].GetCol().GetName(), fn2.Args[i].GetCol().GetName()) { - return false - } - } - - return true -} - -// return map[projid]fiter_id -- position of the projids and filterids but NOT position of ProjectList and FilterList -func (builder *QueryBuilder) findEqualFullTextMatchFunc(projNode *plan.Node, scanNode *plan.Node, projids, filterids []int32) map[int32]int32 { - - eqmap := make(map[int32]int32) - - for i, projid := range projids { - prexpr := projNode.ProjectList[projid] - for j, fid := range filterids { - fexpr := scanNode.FilterList[fid] - eq := builder.equalsFullTextMatchFunc(prexpr.GetF(), fexpr.GetF()) - if eq { - eqmap[int32(i)] = int32(j) - } - } - } - - return eqmap -} - func (builder *QueryBuilder) findMatchFullTextIndex(fn *plan.Function, scanNode *plan.Node) *plan.IndexDef { if fn == nil || scanNode == nil || scanNode.TableDef == nil || len(scanNode.BindingTags) == 0 { return nil @@ -815,7 +766,9 @@ func (builder *QueryBuilder) findMatchFullTextIndex(fn *plan.Function, scanNode return nil } -// Get the filters that are fulltext_match() in ScanNode +// Get the match filters in ScanNode — both fulltext_match (classic MATCH) and +// bm25_match (BM25). Collecting both here lets applyJoinFullTextIndices chain a mix +// of the two into one join structure with one combined score sort. func (builder *QueryBuilder) getFullTextMatchFiltersFromScanNode(node *plan.Node) ([]int32, []*plan.IndexDef) { filterids := make([]int32, 0) @@ -827,22 +780,27 @@ func (builder *QueryBuilder) getFullTextMatchFiltersFromScanNode(node *plan.Node continue } + var idx *plan.IndexDef switch fn.Func.ObjName { case "fulltext_match": - - idx := builder.findMatchFullTextIndex(fn, node) - if idx != nil { - ftidxs = append(ftidxs, idx) - filterids = append(filterids, int32(i)) - } + idx = builder.findMatchFullTextIndex(fn, node) + case "bm25_match": + idx = builder.findMatchBm25Index(fn, node) default: } + if idx != nil { + ftidxs = append(ftidxs, idx) + filterids = append(filterids, int32(i)) + } } return filterids, ftidxs } -// Get the projection that are fulltext_match() in ProjectList +// Get the match projections in ProjectList — both fulltext_match (classic MATCH) and +// bm25_match (BM25). An unmatched classic fulltext_match is rewritten to +// fulltext_match_score (float32); an unmatched bm25_match is left as-is (BM25 requires +// a bm25 index — it errors at execution rather than silently scoring). func (builder *QueryBuilder) getFullTextMatchFromProject(projNode *plan.Node, scanNode *plan.Node) ([]int32, []*plan.IndexDef) { projids := make([]int32, 0) ftidxs := make([]*plan.IndexDef, 0) @@ -865,6 +823,13 @@ func (builder *QueryBuilder) getFullTextMatchFromProject(projNode *plan.Node, sc // which has return type float32 instead of bool projNode.ProjectList[i] = builder.getFullTextMatchScoreExpr(expr) } + case "bm25_match": + + idx := builder.findMatchBm25Index(fn, scanNode) + if idx != nil { + ftidxs = append(ftidxs, idx) + projids = append(projids, int32(i)) + } default: } } diff --git a/pkg/sql/plan/apply_indices_hnsw.go b/pkg/sql/plan/apply_indices_hnsw.go index 7212c40f60be9..1d6860b24df79 100644 --- a/pkg/sql/plan/apply_indices_hnsw.go +++ b/pkg/sql/plan/apply_indices_hnsw.go @@ -305,7 +305,10 @@ func (builder *QueryBuilder) getArgsFromDistFn(distFnExpr *plan.Function, partPo } distFnArgs := distFnExpr.Args - if distFnArgs[0].Typ.GetId() != int32(types.T_array_float32) && distFnArgs[0].Typ.GetId() != int32(types.T_array_float64) { + // Accept any vector element type (f32/f64 and the narrow bf16/f16/int8/uint8), + // so a direct ivf index on a narrow-base column also pushes down rather than + // brute-forcing. + if !types.T(distFnArgs[0].Typ.GetId()).IsArrayRelate() { return } diff --git a/pkg/sql/plan/apply_indices_ivfpq.go b/pkg/sql/plan/apply_indices_ivfpq.go index 217f9fa2824fd..0739bf4142e24 100644 --- a/pkg/sql/plan/apply_indices_ivfpq.go +++ b/pkg/sql/plan/apply_indices_ivfpq.go @@ -34,6 +34,7 @@ type ivfpqIndexContext struct { vecLitArg *plan.Expr origFuncName string partPos int32 + partType plan.Type pkPos int32 pkType plan.Type params string @@ -82,6 +83,7 @@ func (builder *QueryBuilder) prepareIvfpqIndexContext(vecCtx *vectorSortContext, keyPart := idxDef.Parts[0] partPos := vecCtx.scanNode.TableDef.Name2ColIndex[keyPart] + partType := vecCtx.scanNode.TableDef.Cols[partPos].Typ _, vecLitArg, found := builder.getArgsFromDistFn(vecCtx.distFnExpr, partPos) if !found { return nil, nil @@ -119,6 +121,7 @@ func (builder *QueryBuilder) prepareIvfpqIndexContext(vecCtx *vectorSortContext, vecLitArg: vecLitArg, origFuncName: origFuncName, partPos: partPos, + partType: partType, pkPos: pkPos, pkType: pkType, params: idxDef.IndexAlgoParams, @@ -147,7 +150,7 @@ func (builder *QueryBuilder) applyIndicesForSortUsingIvfpq(nodeID int32, vecCtx return nodeID, err } - tblCfgStr := fmt.Sprintf(`{"db": "%s", "src": "%s", "metadata":"%s", "index":"%s", "threads_search": %d, "orig_func_name": "%s", "batch_window": %d, "nprobe": %d, "gpu_multi_simulation": %d}`, + tblCfgStr := fmt.Sprintf(`{"db": "%s", "src": "%s", "metadata":"%s", "index":"%s", "threads_search": %d, "orig_func_name": "%s", "batch_window": %d, "nprobe": %d, "gpu_multi_simulation": %d, "parttype": %d}`, scanNode.ObjRef.SchemaName, scanNode.TableDef.Name, ivfpqCtx.metaDef.IndexTableName, @@ -156,7 +159,8 @@ func (builder *QueryBuilder) applyIndicesForSortUsingIvfpq(nodeID int32, vecCtx ivfpqCtx.origFuncName, ivfpqCtx.batchWindow, ivfpqCtx.nProbe, - ivfpqCtx.gpuMultiSim) + ivfpqCtx.gpuMultiSim, + ivfpqCtx.partType.Id) // Predicate pushdown on INCLUDE columns and the primary key: peel // filters that reference only INCLUDE columns (or the PK, routed to diff --git a/pkg/sql/plan/apply_indices_match.go b/pkg/sql/plan/apply_indices_match.go new file mode 100644 index 0000000000000..6b7a8d14a09af --- /dev/null +++ b/pkg/sql/plan/apply_indices_match.go @@ -0,0 +1,99 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file holds the small, domain-neutral helpers shared by BOTH the classic +// fulltext MATCH path (apply_indices_fulltext.go) and the bm25 MATCH path +// (apply_indices_bm25.go). They operate purely on the shared fulltext_match() +// function ABI (Args[0]=pattern, Args[1]=mode, Args[2:]=index columns) and the +// generic node-context table — no classic-fulltext- or bm25-specific behavior — +// so sharing them avoids duplication without coupling the two index semantics. +package plan + +import ( + "strings" + + "github.com/matrixorigin/matrixone/pkg/pb/plan" +) + +// equalsMatchFunc reports whether two match functions are equivalent: same function +// (a bm25_match and a fulltext_match are never equal, even on the same column/pattern), +// same pattern, same mode, and the same set of index column names. +func (builder *QueryBuilder) equalsMatchFunc(fn1 *plan.Function, fn2 *plan.Function) bool { + + if fn1.Func.ObjName != fn2.Func.ObjName { + return false + } + + nargs1 := len(fn1.Args) + nargs2 := len(fn2.Args) + + if nargs1 != nargs2 { + return false + } + + // check search pattern and mode + var pattern1, pattern2 string + var mode1, mode2 int64 + + pattern1 = fn1.Args[0].GetLit().GetSval() + mode1 = fn1.Args[1].GetLit().GetI64Val() + + pattern2 = fn2.Args[0].GetLit().GetSval() + mode2 = fn2.Args[1].GetLit().GetI64Val() + + if pattern1 != pattern2 || mode1 != mode2 { + return false + } + + // check index parts + for i := 2; i < nargs1; i++ { + if !strings.EqualFold(fn1.Args[i].GetCol().GetName(), fn2.Args[i].GetCol().GetName()) { + return false + } + } + + return true +} + +// findEqualMatchFunc returns map[projid]filter_id -- position of the projids and +// filterids but NOT position of ProjectList and FilterList +func (builder *QueryBuilder) findEqualMatchFunc(projNode *plan.Node, scanNode *plan.Node, projids, filterids []int32) map[int32]int32 { + + eqmap := make(map[int32]int32) + + for i, projid := range projids { + prexpr := projNode.ProjectList[projid] + for j, fid := range filterids { + fexpr := scanNode.FilterList[fid] + eq := builder.equalsMatchFunc(prexpr.GetF(), fexpr.GetF()) + if eq { + eqmap[int32(i)] = int32(j) + } + } + } + + return eqmap +} + +// matchRewriteContextNodeID picks a valid node-context id for a MATCH rewrite, +// preferring preferredNodeID and falling back to the scan node's id. +func (builder *QueryBuilder) matchRewriteContextNodeID(preferredNodeID int32, scanNode *plan.Node) int32 { + if preferredNodeID >= 0 && int(preferredNodeID) < len(builder.ctxByNode) && builder.ctxByNode[preferredNodeID] != nil { + return preferredNodeID + } + if scanNode != nil && scanNode.NodeId >= 0 && int(scanNode.NodeId) < len(builder.ctxByNode) && builder.ctxByNode[scanNode.NodeId] != nil { + return scanNode.NodeId + } + return preferredNodeID +} diff --git a/pkg/sql/plan/base_binder.go b/pkg/sql/plan/base_binder.go index d59f51016ab4e..711d965e1fe4c 100644 --- a/pkg/sql/plan/base_binder.go +++ b/pkg/sql/plan/base_binder.go @@ -1207,7 +1207,13 @@ func (b *baseBinder) bindFullTextMatchExpr(astExpr *tree.FullTextMatchExpr, dept args[i+2] = c } - return BindFuncExprImplByPlanExpr(b.GetContext(), "fulltext_match", args) + // BM25(col) AGAINST('q') binds to the distinct bm25_match function so the + // planner routes it to bm25_search; classic MATCH(...) binds to fulltext_match. + funcName := "fulltext_match" + if astExpr.IsBm25 { + funcName = "bm25_match" + } + return BindFuncExprImplByPlanExpr(b.GetContext(), funcName, args) } func (b *baseBinder) bindFuncExprImplByAstExpr(name string, astArgs []tree.Expr, depth int32) (*plan.Expr, error) { diff --git a/pkg/sql/plan/bm25_compact.go b/pkg/sql/plan/bm25_compact.go new file mode 100644 index 0000000000000..20aa3ada3808d --- /dev/null +++ b/pkg/sql/plan/bm25_compact.go @@ -0,0 +1,54 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plan + +import ( + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" +) + +// bm25CompactColDefs — the compact table function returns a single status row: +// merged_subs, the number of tag=0 sub-indexes the merge produced. Callers +// (ALTER … REINDEX … BM25 MERGE / idxcron) run it for its side effect and +// discard the row. +func bm25CompactColDefs() []*plan.ColDef { + tp := types.New(types.T_int64, 0, 0) + return []*plan.ColDef{{ + Name: "merged_subs", + Typ: plan.Type{Id: int32(tp.Oid), NotNullable: true}, + }} +} + +// buildBm25Compact builds a FUNCTION_SCAN node for the standalone +// `bm25_compact(db, store, meta, capacity)` compaction table function — no +// driving table, four varchar args (passed as-is, no leading param strip). +func (builder *QueryBuilder) buildBm25Compact(tbl *tree.TableFunction, ctx *BindContext, exprs []*plan.Expr, children []int32) int32 { + node := &plan.Node{ + NodeType: plan.Node_FUNCTION_SCAN, + Stats: &plan.Stats{}, + TableDef: &plan.TableDef{ + TableType: "func_table", + TblFunc: &plan.TableFunction{ + Name: "bm25_compact", + }, + Cols: bm25CompactColDefs(), + }, + BindingTags: []int32{builder.genNewBindTag()}, + Children: children, + TblFuncExprList: exprs, + } + return builder.appendNode(node, ctx) +} diff --git a/pkg/sql/plan/build_ddl.go b/pkg/sql/plan/build_ddl.go index 41af69b54014e..8cab1d7393f54 100644 --- a/pkg/sql/plan/build_ddl.go +++ b/pkg/sql/plan/build_ddl.go @@ -3743,6 +3743,7 @@ func buildAlterTableInplace(stmt *tree.AlterTable, ctx CompilerContext) (*Plan, // merge + reject happens at compile in Compile.ValidateReindexParams, // reading the options straight off the parse tree. alterTableReIndex.ForceSync = opt.ForceSync + alterTableReIndex.Merge = opt.Merge name_not_found := true // check index diff --git a/pkg/sql/plan/build_index_util.go b/pkg/sql/plan/build_index_util.go index 105b9a8d7de35..43fd1fffddc97 100644 --- a/pkg/sql/plan/build_index_util.go +++ b/pkg/sql/plan/build_index_util.go @@ -21,6 +21,8 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/container/types" + indexplugin "github.com/matrixorigin/matrixone/pkg/indexplugin" + catalogplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/catalog" "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" ) @@ -183,13 +185,27 @@ func indexColumnCheckKind(indexType tree.IndexType) string { return "ivfflat" case tree.INDEX_TYPE_HNSW: return "hnsw" + case tree.INDEX_TYPE_CAGRA: + return "cagra" + case tree.INDEX_TYPE_IVFPQ: + return "ivfpq" case tree.INDEX_TYPE_RTREE: return "rtree" + case tree.INDEX_TYPE_BM25: + return "bm25" default: return "secondary" } } +// bm25IndexableColumn reports whether a column type can back a bm25 index +// (the same text-ish set the classic fulltext index accepts). +func bm25IndexableColumn(id int32) bool { + return id == int32(types.T_text) || id == int32(types.T_char) || + id == int32(types.T_varchar) || id == int32(types.T_json) || + id == int32(types.T_datalink) +} + func checkIndexColumnSupportability(ctx context.Context, col *ColDef, keyPart *tree.KeyPart, indexKind string) error { if col == nil || keyPart == nil || keyPart.ColName == nil { return moerr.NewInternalError(ctx, "index column definition is nil") @@ -197,6 +213,16 @@ func checkIndexColumnSupportability(ctx context.Context, col *ColDef, keyPart *t colName := keyPart.ColName.ColNameOrigin() + // A bm25 ranked-retrieval index tokenizes a text column, so it accepts the + // same text/char/varchar/text/json/datalink types the classic fulltext index + // does (fulltext takes a separate build path and never reaches this check). + if indexKind == "bm25" { + if bm25IndexableColumn(col.Typ.Id) { + return nil + } + return moerr.NewNotSupported(ctx, fmt.Sprintf("bm25 index only supports CHAR/VARCHAR/TEXT/JSON/DATALINK columns, not '%s'", colName)) + } + switch col.Typ.Id { case int32(types.T_blob): if keyPart.Length > 0 && indexKind != "primary" { @@ -212,8 +238,17 @@ func checkIndexColumnSupportability(ctx context.Context, col *ColDef, keyPart *t return moerr.NewNotSupported(ctx, fmt.Sprintf("DATALINK column '%s' cannot be in index", colName)) case int32(types.T_json): return moerr.NewNotSupported(ctx, fmt.Sprintf("JSON column '%s' cannot be in index", colName)) - case int32(types.T_array_float32), int32(types.T_array_float64): - if indexKind == "ivfflat" || indexKind == "hnsw" { + case int32(types.T_array_float32), int32(types.T_array_float64), + int32(types.T_array_float16), int32(types.T_array_bf16), + int32(types.T_array_int8), int32(types.T_array_uint8): + // A vector column is valid only as the key of a vector index, AND only if + // that algorithm supports this element type. Delegate to the plugin's + // catalog hook (SupportedVectorTypes) rather than hardcoding — each algo + // differs (ivfflat: f32/f64/f16/bf16/int8/uint8; cagra/ivfpq: f32/f16 only; + // hnsw: f32/f64). Non-vector index kinds (secondary/primary/unique/rtree) + // have no plugin, so the vector column is rejected. + if p, ok := indexplugin.Get(indexKind); ok && + catalogplugin.SupportsVectorType(p.Catalog(), types.T(col.Typ.Id)) { return nil } return moerr.NewNotSupported(ctx, fmt.Sprintf("VECTOR column '%s' cannot be in index", colName)) diff --git a/pkg/sql/plan/build_index_util_test.go b/pkg/sql/plan/build_index_util_test.go index c05d2ebd9d8ec..8740d7a506949 100644 --- a/pkg/sql/plan/build_index_util_test.go +++ b/pkg/sql/plan/build_index_util_test.go @@ -169,10 +169,20 @@ func TestCheckIndexColumnSupportability(t *testing.T) { require.Error(t, checkIndexColumnSupportability(ctx, colOf(types.T_json), keyPart, "secondary")) }) - t.Run("vector only allowed for ivfflat and hnsw", func(t *testing.T) { - require.NoError(t, checkIndexColumnSupportability(ctx, colOf(types.T_array_float32), keyPart, "ivfflat")) + t.Run("vector type support is delegated to the plugin per algo", func(t *testing.T) { + // ivfflat accepts every vector element type (f32/f64 + narrow f16/bf16/int8/uint8). + for _, ty := range []types.T{ + types.T_array_float32, types.T_array_float64, types.T_array_float16, + types.T_array_bf16, types.T_array_int8, types.T_array_uint8, + } { + require.NoError(t, checkIndexColumnSupportability(ctx, colOf(ty), keyPart, "ivfflat")) + } require.NoError(t, checkIndexColumnSupportability(ctx, colOf(types.T_array_float64), keyPart, "hnsw")) + // A vector column in a non-vector index kind has no plugin → rejected, + // for both wide and narrow element types. require.Error(t, checkIndexColumnSupportability(ctx, colOf(types.T_array_float32), keyPart, "secondary")) + require.Error(t, checkIndexColumnSupportability(ctx, colOf(types.T_array_int8), keyPart, "secondary")) + require.Error(t, checkIndexColumnSupportability(ctx, colOf(types.T_array_float16), keyPart, "unique")) }) t.Run("enum rejected only in primary key", func(t *testing.T) { diff --git a/pkg/sql/plan/build_show_util.go b/pkg/sql/plan/build_show_util.go index fbbd3d32f4d5d..5bc8fb7d7e6ce 100644 --- a/pkg/sql/plan/build_show_util.go +++ b/pkg/sql/plan/build_show_util.go @@ -938,7 +938,7 @@ func FormatColType(colType plan.Type) string { case types.T_bit, types.T_char, types.T_varchar, types.T_binary, types.T_varbinary: suffix = fmt.Sprintf("(%d)", colType.Width) - case types.T_array_float32, types.T_array_float64: + case types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: suffix = fmt.Sprintf("(%d)", colType.Width) } diff --git a/pkg/sql/plan/build_show_util_test.go b/pkg/sql/plan/build_show_util_test.go index a04325ce598ad..54df434667040 100644 --- a/pkg/sql/plan/build_show_util_test.go +++ b/pkg/sql/plan/build_show_util_test.go @@ -441,6 +441,17 @@ func TestFormatColTypeArrayMetadata(t *testing.T) { })) } +func TestFormatColTypeVector(t *testing.T) { + // Every vector type must round-trip its dimension in SHOW CREATE, not just + // f32/f64 (the narrow types were previously missing the (N) suffix). + require.Equal(t, "VECF32(3)", FormatColType(plan.Type{Id: int32(types.T_array_float32), Width: 3})) + require.Equal(t, "VECF64(3)", FormatColType(plan.Type{Id: int32(types.T_array_float64), Width: 3})) + require.Equal(t, "VECBF16(3)", FormatColType(plan.Type{Id: int32(types.T_array_bf16), Width: 3})) + require.Equal(t, "VECF16(3)", FormatColType(plan.Type{Id: int32(types.T_array_float16), Width: 3})) + require.Equal(t, "VECINT8(3)", FormatColType(plan.Type{Id: int32(types.T_array_int8), Width: 3})) + require.Equal(t, "VECUINT8(3)", FormatColType(plan.Type{Id: int32(types.T_array_uint8), Width: 3})) +} + // TestShowCreateExternalWriteFilePattern ensures SHOW CREATE TABLE formatting // keeps WRITE_FILE_PATTERN for writable external tables, in both the INFILE // and the URL s3option forms; without it the recreated table silently degrades diff --git a/pkg/sql/plan/build_util.go b/pkg/sql/plan/build_util.go index 60e17167805a5..64d2d318c1788 100644 --- a/pkg/sql/plan/build_util.go +++ b/pkg/sql/plan/build_util.go @@ -157,7 +157,7 @@ func getTypeFromAst(ctx context.Context, typ tree.ResolvableTypeReference) (plan // create table t1(a char) -> DisplayWith = -1；but get width=1 in MySQL and PgSQL if fstr == "char" || fstr == "binary" { width = 1 - } else if fstr == "vecf32" || fstr == "vecf64" { + } else if fstr == types.ArrayFloat32SQLName || fstr == types.ArrayFloat64SQLName || fstr == types.ArrayBF16SQLName || fstr == types.ArrayFloat16SQLName || fstr == types.ArrayInt8SQLName || fstr == types.ArrayUint8SQLName { width = types.MaxArrayDimension } else { width = types.MaxVarcharLen @@ -168,7 +168,7 @@ func getTypeFromAst(ctx context.Context, typ tree.ResolvableTypeReference) (plan return plan.Type{}, moerr.NewOutOfRangef(ctx, fstr, " typeLen is over the MaxCharLen: %v", types.MaxCharLen) } else if (fstr == "varchar" || fstr == "varbinary") && width > types.MaxVarcharLen { return plan.Type{}, moerr.NewOutOfRangef(ctx, fstr, " typeLen is over the MaxVarcharLen: %v", types.MaxVarcharLen) - } else if fstr == "vecf32" || fstr == "vecf64" { + } else if fstr == types.ArrayFloat32SQLName || fstr == types.ArrayFloat64SQLName || fstr == types.ArrayBF16SQLName || fstr == types.ArrayFloat16SQLName || fstr == types.ArrayInt8SQLName || fstr == types.ArrayUint8SQLName { if width > types.MaxArrayDimension { return plan.Type{}, moerr.NewOutOfRangef(ctx, fstr, " typeLen is over the MaxVectorLen : %v", types.MaxArrayDimension) } @@ -183,10 +183,18 @@ func getTypeFromAst(ctx context.Context, typ tree.ResolvableTypeReference) (plan return plan.Type{Id: int32(types.T_binary), Width: width}, nil case "varchar": return plan.Type{Id: int32(types.T_varchar), Width: width}, nil - case "vecf32": + case types.ArrayFloat32SQLName: return plan.Type{Id: int32(types.T_array_float32), Width: width}, nil - case "vecf64": + case types.ArrayFloat64SQLName: return plan.Type{Id: int32(types.T_array_float64), Width: width}, nil + case types.ArrayBF16SQLName: + return plan.Type{Id: int32(types.T_array_bf16), Width: width}, nil + case types.ArrayFloat16SQLName: + return plan.Type{Id: int32(types.T_array_float16), Width: width}, nil + case types.ArrayInt8SQLName: + return plan.Type{Id: int32(types.T_array_int8), Width: width}, nil + case types.ArrayUint8SQLName: + return plan.Type{Id: int32(types.T_array_uint8), Width: width}, nil } // varbinary return plan.Type{Id: int32(types.T_varbinary), Width: width}, nil diff --git a/pkg/sql/plan/function/func_binary.go b/pkg/sql/plan/function/func_binary.go index 2f05c82a2d978..b8492f5c9c7d9 100644 --- a/pkg/sql/plan/function/func_binary.go +++ b/pkg/sql/plan/function/func_binary.go @@ -11661,6 +11661,73 @@ func CosineDistanceArray[T types.RealNumbers](ivecs []*vector.Vector, result vec }, selectList) } +// arrayDistanceNarrow computes a binary vector distance for the narrow element +// types (bf16/f16/int8/uint8) using the NATIVE metric kernel for T — int8/uint8 +// run the INTEGER kernels (int32/int64 accumulate, no float upcast), bf16/f16 run +// the fused decode-to-float32 kernels (no intermediate []float32 materialized). +// This is the same kernel ivfflat's brute-force centroid scan uses, so the SQL +// re-rank (l2_distance over a narrow entries column — the hot path) no longer +// detours through the float32 bridge. It deliberately bypasses +// batchArrayDistanceSync (the GPU/usearch path), which only supports native float +// element types. +// +// m selects the kernel; sqrtResult sqrts the result for TRUE L2 (the kernel +// returns squared L2 for Metric_L2Distance, matching ResolveDistanceFn). The int8 +// squared sum is exact in int64, so sqrt-in-float64 is at least as accurate as the +// old float32 bridge and preserves ranking order. +func arrayDistanceNarrow[T types.ArrayElement]( + ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList, + m metric.MetricType, sqrtResult bool) error { + kernel, err := metric.ResolveDistanceFn[T, float64](m) + if err != nil { + return err + } + return opBinaryBytesBytesToFixedWithErrorCheck[float64](ivecs, result, proc, length, func(v1, v2 []byte) (float64, error) { + d, e := kernel(types.BytesToArray[T](v1), types.BytesToArray[T](v2)) + if e != nil { + return 0, e + } + if sqrtResult { + d = math.Sqrt(d) + } + return d, nil + }, selectList) +} + +// arrayDistanceViaF32 is retained only for cosine_similarity, whose float32 +// downcast corner-case handling (see moarray.CosineSimilarity) has no integer- +// kernel equivalent. Operands are upcast to []float32 and run through the f32 +// kernel. +func arrayDistanceViaF32[T types.ArrayElement]( + ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList, + kernel func(v1, v2 []float32) (float64, error)) error { + return opBinaryBytesBytesToFixedWithErrorCheck[float64](ivecs, result, proc, length, func(v1, v2 []byte) (float64, error) { + f1 := types.ToFloat32Array[T](types.BytesToArray[T](v1)) + f2 := types.ToFloat32Array[T](types.BytesToArray[T](v2)) + return kernel(f1, f2) + }, selectList) +} + +func L2DistanceArrayViaF32[T types.ArrayElement](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { + return arrayDistanceNarrow[T](ivecs, result, proc, length, selectList, metric.Metric_L2Distance, true) +} + +func L2DistanceSqArrayViaF32[T types.ArrayElement](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { + return arrayDistanceNarrow[T](ivecs, result, proc, length, selectList, metric.Metric_L2sqDistance, false) +} + +func InnerProductArrayViaF32[T types.ArrayElement](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { + return arrayDistanceNarrow[T](ivecs, result, proc, length, selectList, metric.Metric_InnerProduct, false) +} + +func CosineDistanceArrayViaF32[T types.ArrayElement](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { + return arrayDistanceNarrow[T](ivecs, result, proc, length, selectList, metric.Metric_CosineDistance, false) +} + +func CosineSimilarityArrayViaF32[T types.ArrayElement](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { + return arrayDistanceViaF32[T](ivecs, result, proc, length, selectList, moarray.CosineSimilarity[float32]) +} + func castBinaryArrayToInt(array []uint8) int64 { var result int64 for i, value := range array { diff --git a/pkg/sql/plan/function/func_cast.go b/pkg/sql/plan/function/func_cast.go index ef872f401bdde..2126dfa6ef519 100644 --- a/pkg/sql/plan/function/func_cast.go +++ b/pkg/sql/plan/function/func_cast.go @@ -35,7 +35,6 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/nulls" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" - "github.com/matrixorigin/matrixone/pkg/vectorize/moarray" "github.com/matrixorigin/matrixone/pkg/vm/process" "golang.org/x/exp/constraints" ) @@ -55,6 +54,7 @@ var supportedTypeCast = map[types.T][]types.T{ types.T_time, types.T_timestamp, types.T_year, types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32, }, @@ -299,6 +299,7 @@ var supportedTypeCast = map[types.T][]types.T{ types.T_char, types.T_varchar, types.T_blob, types.T_text, types.T_binary, types.T_varbinary, types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32, types.T_TS, }, @@ -347,6 +348,7 @@ var supportedTypeCast = map[types.T][]types.T{ types.T_char, types.T_varchar, types.T_blob, types.T_text, types.T_binary, types.T_varbinary, types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32, }, @@ -365,6 +367,7 @@ var supportedTypeCast = map[types.T][]types.T{ types.T_char, types.T_varchar, types.T_blob, types.T_text, types.T_binary, types.T_varbinary, types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32, }, types.T_geometry: { @@ -427,9 +430,27 @@ var supportedTypeCast = map[types.T][]types.T{ types.T_array_float32: { types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, }, types.T_array_float64: { types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, + }, + types.T_array_bf16: { + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, + }, + types.T_array_float16: { + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, + }, + types.T_array_int8: { + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, + }, + types.T_array_uint8: { + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, }, } @@ -528,7 +549,7 @@ func newCast(parameters []*vector.Vector, result vector.FunctionResultWrapper, p case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_blob, types.T_text, types.T_datalink, types.T_geometry, types.T_geometry32: s := vector.GenerateFunctionStrParameter(from) err = strTypeToOthers(proc, s, *toType, result, length, selectList, strictStringWidth) - case types.T_array_float32, types.T_array_float64: + case types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: //NOTE: Don't mix T_array and T_varchar. // T_varchar will have "[1,2,3]" string // T_array will have "@@@#@!#@!@#!" binary. @@ -647,7 +668,7 @@ func scalarNullToOthers(ctx context.Context, return appendNulls[uint64](result, length, selectList) case types.T_char, types.T_varchar, types.T_blob, types.T_binary, types.T_varbinary, types.T_text, types.T_json, - types.T_array_float32, types.T_array_float64, types.T_datalink, types.T_geometry: + types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry: return appendNulls[types.Varlena](result, length, selectList) case types.T_float32: return appendNulls[float32](result, length, selectList) @@ -1888,6 +1909,18 @@ func strTypeToOthers(proc *process.Process, case types.T_array_float64: rs := vector.MustFunctionResult[types.Varlena](result) return blobToArray[float64](ctx, source, rs, length, toType) + case types.T_array_bf16: + rs := vector.MustFunctionResult[types.Varlena](result) + return blobToArray[types.BF16](ctx, source, rs, length, toType) + case types.T_array_float16: + rs := vector.MustFunctionResult[types.Varlena](result) + return blobToArray[types.Float16](ctx, source, rs, length, toType) + case types.T_array_int8: + rs := vector.MustFunctionResult[types.Varlena](result) + return blobToArray[int8](ctx, source, rs, length, toType) + case types.T_array_uint8: + rs := vector.MustFunctionResult[types.Varlena](result) + return blobToArray[uint8](ctx, source, rs, length, toType) // NOTE 1: don't add `switch default` and panic here. If `T_blob` to `ARRAY` is not required, // then continue to the `str` to `Other` code. // NOTE 2: don't create a switch T_blob case in NewCast() as @@ -1970,6 +2003,18 @@ func strTypeToOthers(proc *process.Process, case types.T_array_float64: rs := vector.MustFunctionResult[types.Varlena](result) return strToArray[float64](ctx, source, rs, length, toType) + case types.T_array_bf16: + rs := vector.MustFunctionResult[types.Varlena](result) + return strToArray[types.BF16](ctx, source, rs, length, toType) + case types.T_array_float16: + rs := vector.MustFunctionResult[types.Varlena](result) + return strToArray[types.Float16](ctx, source, rs, length, toType) + case types.T_array_int8: + rs := vector.MustFunctionResult[types.Varlena](result) + return strToArray[int8](ctx, source, rs, length, toType) + case types.T_array_uint8: + rs := vector.MustFunctionResult[types.Varlena](result) + return strToArray[uint8](ctx, source, rs, length, toType) case types.T_year: rs := vector.MustFunctionResult[types.MoYear](result) return strToYear(ctx, source, rs, length, selectList) @@ -1986,24 +2031,45 @@ func arrayTypeToOthers(proc *process.Process, switch fromType.Oid { case types.T_array_float32: - switch toType.Oid { - case types.T_array_float32: - return arrayToArray[float32, float32](proc.Ctx, source, rs, length, toType) - case types.T_array_float64: - return arrayToArray[float32, float64](proc.Ctx, source, rs, length, toType) - } + return arrayToArrayDispatch[float32](proc, source, rs, length, toType) case types.T_array_float64: - switch toType.Oid { - case types.T_array_float32: - return arrayToArray[float64, float32](proc.Ctx, source, rs, length, toType) - case types.T_array_float64: - return arrayToArray[float64, float64](proc.Ctx, source, rs, length, toType) - } + return arrayToArrayDispatch[float64](proc, source, rs, length, toType) + case types.T_array_bf16: + return arrayToArrayDispatch[types.BF16](proc, source, rs, length, toType) + case types.T_array_float16: + return arrayToArrayDispatch[types.Float16](proc, source, rs, length, toType) + case types.T_array_int8: + return arrayToArrayDispatch[int8](proc, source, rs, length, toType) + case types.T_array_uint8: + return arrayToArrayDispatch[uint8](proc, source, rs, length, toType) } return moerr.NewInternalError(ctx, fmt.Sprintf("unsupported cast from %s to %s", fromType, toType)) } +// arrayToArrayDispatch resolves the target element type for a vector->vector +// cast whose source element type I is already known, then runs the float32 +// bridge in arrayToArray. Covers all 25 (5x5) vector-pair casts. +func arrayToArrayDispatch[I types.ArrayElement](proc *process.Process, + source vector.FunctionParameterWrapper[types.Varlena], + rs *vector.FunctionResult[types.Varlena], length int, toType types.Type) error { + switch toType.Oid { + case types.T_array_float32: + return arrayToArray[I, float32](proc.Ctx, source, rs, length, toType) + case types.T_array_float64: + return arrayToArray[I, float64](proc.Ctx, source, rs, length, toType) + case types.T_array_bf16: + return arrayToArray[I, types.BF16](proc.Ctx, source, rs, length, toType) + case types.T_array_float16: + return arrayToArray[I, types.Float16](proc.Ctx, source, rs, length, toType) + case types.T_array_int8: + return arrayToArray[I, int8](proc.Ctx, source, rs, length, toType) + case types.T_array_uint8: + return arrayToArray[I, uint8](proc.Ctx, source, rs, length, toType) + } + return moerr.NewInternalError(proc.Ctx, fmt.Sprintf("unsupported cast to %s", toType)) +} + func uuidToOthers(ctx context.Context, source vector.FunctionParameterWrapper[types.Uuid], toType types.Type, result vector.FunctionResultWrapper, length int, selectList *FunctionSelectList) error { @@ -5804,7 +5870,7 @@ func strToBit( return nil } -func strToArray[T types.RealNumbers]( +func strToArray[T types.ArrayElement]( _ context.Context, from vector.FunctionParameterWrapper[types.Varlena], to *vector.FunctionResult[types.Varlena], length int, _ types.Type) error { @@ -5840,7 +5906,7 @@ func strToArray[T types.RealNumbers]( return nil } -func blobToArray[T types.RealNumbers]( +func blobToArray[T types.ArrayElement]( _ context.Context, from vector.FunctionParameterWrapper[types.Varlena], to *vector.FunctionResult[types.Varlena], length int, _ types.Type) error { @@ -5870,7 +5936,7 @@ func blobToArray[T types.RealNumbers]( return nil } -func arrayToArray[I types.RealNumbers, O types.RealNumbers]( +func arrayToArray[I types.ArrayElement, O types.ArrayElement]( _ context.Context, from vector.FunctionParameterWrapper[types.Varlena], to *vector.FunctionResult[types.Varlena], length int, _ types.Type) error { @@ -5891,18 +5957,20 @@ func arrayToArray[I types.RealNumbers, O types.RealNumbers]( // cases b/b and b+sqrt(b) fails. if from.GetType().Oid == to.GetType().Oid { - // Eg:- VECF32(3) --> VECF32(3) + // Eg:- VECF32(3) --> VECF32(3): identical byte layout, copy as-is. if err := to.AppendBytes(v, false); err != nil { return err } } else { - // Eg:- VECF32(3) --> VECF64(3) + // Eg:- VECF32(3) --> VECF64(3), VECF32 --> VECINT8, etc. + // All 25 vector-pair casts route through the float32 bridge: + // upcast the source element type to []float32, then narrow to the + // target element type (int8 rounds+clamps; bf16/f16 round-to-even). + // This replaces moarray.Cast[I,O], which only handled float pairs. _v := types.BytesToArray[I](v) - cast, err := moarray.Cast[I, O](_v) - if err != nil { - return err - } - bytes := types.ArrayToBytes[O](cast) + f32 := types.ToFloat32Array[I](_v) + out := types.FromFloat32Array[O](f32) + bytes := types.ArrayToBytes[O](out) if err := to.AppendBytes(bytes, false); err != nil { return err } diff --git a/pkg/sql/plan/function/func_compare.go b/pkg/sql/plan/function/func_compare.go index 9677d20c4fb2e..499f12452ae5c 100644 --- a/pkg/sql/plan/function/func_compare.go +++ b/pkg/sql/plan/function/func_compare.go @@ -45,6 +45,7 @@ func otherCompareOperatorSupports(typ1, typ2 types.Type) bool { case types.T_uuid: case types.T_Rowid: case types.T_array_float32, types.T_array_float64: + case types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: case types.T_year: default: return false @@ -83,6 +84,7 @@ func equalAndNotEqualOperatorSupports(typ1, typ2 types.Type) bool { case types.T_uuid: case types.T_Rowid: case types.T_array_float32, types.T_array_float64: + case types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: case types.T_enum: case types.T_year: default: @@ -238,6 +240,30 @@ func nullSafeEqualFn(parameters []*vector.Vector, result vector.FunctionResultWr _v2 := types.BytesToArray[float64](v2) return types.ArrayCompare[float64](_v1, _v2) == 0 }, selectList) + case types.T_array_bf16: + return opBinaryBytesBytesToFixedNullSafe(parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.BF16](v1) + _v2 := types.BytesToArray[types.BF16](v2) + return types.ArrayElementCompare[types.BF16](_v1, _v2) == 0 + }, selectList) + case types.T_array_float16: + return opBinaryBytesBytesToFixedNullSafe(parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.Float16](v1) + _v2 := types.BytesToArray[types.Float16](v2) + return types.ArrayElementCompare[types.Float16](_v1, _v2) == 0 + }, selectList) + case types.T_array_int8: + return opBinaryBytesBytesToFixedNullSafe(parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[int8](v1) + _v2 := types.BytesToArray[int8](v2) + return types.ArrayElementCompare[int8](_v1, _v2) == 0 + }, selectList) + case types.T_array_uint8: + return opBinaryBytesBytesToFixedNullSafe(parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[uint8](v1) + _v2 := types.BytesToArray[uint8](v2) + return types.ArrayElementCompare[uint8](_v1, _v2) == 0 + }, selectList) case types.T_date: return opBinaryFixedFixedToFixedNullSafe[types.Date](parameters, rs, proc, length, func(a, b types.Date) bool { return a == b @@ -376,6 +402,22 @@ func equalFn(parameters []*vector.Vector, result vector.FunctionResultWrapper, p return types.ArrayCompare[float64](_v1, _v2) == 0 }, selectList) + case types.T_array_bf16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + return types.ArrayElementCompare[types.BF16](types.BytesToArray[types.BF16](v1), types.BytesToArray[types.BF16](v2)) == 0 + }, selectList) + case types.T_array_float16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + return types.ArrayElementCompare[types.Float16](types.BytesToArray[types.Float16](v1), types.BytesToArray[types.Float16](v2)) == 0 + }, selectList) + case types.T_array_int8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + return types.ArrayElementCompare[int8](types.BytesToArray[int8](v1), types.BytesToArray[int8](v2)) == 0 + }, selectList) + case types.T_array_uint8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + return types.ArrayElementCompare[uint8](types.BytesToArray[uint8](v1), types.BytesToArray[uint8](v2)) == 0 + }, selectList) case types.T_date: return opBinaryFixedFixedToFixed[types.Date, types.Date, bool](parameters, rs, proc, length, func(a, b types.Date) bool { return a == b @@ -762,6 +804,30 @@ func greatThanFn(parameters []*vector.Vector, result vector.FunctionResultWrappe return types.ArrayCompare[float64](_v1, _v2) > 0 }, selectList) + case types.T_array_bf16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.BF16](v1) + _v2 := types.BytesToArray[types.BF16](v2) + return types.ArrayElementCompare[types.BF16](_v1, _v2) > 0 + }, selectList) + case types.T_array_float16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.Float16](v1) + _v2 := types.BytesToArray[types.Float16](v2) + return types.ArrayElementCompare[types.Float16](_v1, _v2) > 0 + }, selectList) + case types.T_array_int8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[int8](v1) + _v2 := types.BytesToArray[int8](v2) + return types.ArrayElementCompare[int8](_v1, _v2) > 0 + }, selectList) + case types.T_array_uint8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[uint8](v1) + _v2 := types.BytesToArray[uint8](v2) + return types.ArrayElementCompare[uint8](_v1, _v2) > 0 + }, selectList) case types.T_date: return opBinaryFixedFixedToFixed[types.Date, types.Date, bool](parameters, rs, proc, length, func(a, b types.Date) bool { return a > b @@ -889,6 +955,30 @@ func greatEqualFn(parameters []*vector.Vector, result vector.FunctionResultWrapp return types.ArrayCompare[float64](_v1, _v2) >= 0 }, selectList) + case types.T_array_bf16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.BF16](v1) + _v2 := types.BytesToArray[types.BF16](v2) + return types.ArrayElementCompare[types.BF16](_v1, _v2) >= 0 + }, selectList) + case types.T_array_float16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.Float16](v1) + _v2 := types.BytesToArray[types.Float16](v2) + return types.ArrayElementCompare[types.Float16](_v1, _v2) >= 0 + }, selectList) + case types.T_array_int8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[int8](v1) + _v2 := types.BytesToArray[int8](v2) + return types.ArrayElementCompare[int8](_v1, _v2) >= 0 + }, selectList) + case types.T_array_uint8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[uint8](v1) + _v2 := types.BytesToArray[uint8](v2) + return types.ArrayElementCompare[uint8](_v1, _v2) >= 0 + }, selectList) case types.T_date: return opBinaryFixedFixedToFixed[types.Date, types.Date, bool](parameters, rs, proc, length, func(a, b types.Date) bool { return a >= b @@ -1016,6 +1106,30 @@ func notEqualFn(parameters []*vector.Vector, result vector.FunctionResultWrapper return types.ArrayCompare[float64](_v1, _v2) != 0 }, selectList) + case types.T_array_bf16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.BF16](v1) + _v2 := types.BytesToArray[types.BF16](v2) + return types.ArrayElementCompare[types.BF16](_v1, _v2) != 0 + }, selectList) + case types.T_array_float16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.Float16](v1) + _v2 := types.BytesToArray[types.Float16](v2) + return types.ArrayElementCompare[types.Float16](_v1, _v2) != 0 + }, selectList) + case types.T_array_int8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[int8](v1) + _v2 := types.BytesToArray[int8](v2) + return types.ArrayElementCompare[int8](_v1, _v2) != 0 + }, selectList) + case types.T_array_uint8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[uint8](v1) + _v2 := types.BytesToArray[uint8](v2) + return types.ArrayElementCompare[uint8](_v1, _v2) != 0 + }, selectList) case types.T_date: return opBinaryFixedFixedToFixed[types.Date, types.Date, bool](parameters, rs, proc, length, func(a, b types.Date) bool { return a != b @@ -1143,6 +1257,30 @@ func lessThanFn(parameters []*vector.Vector, result vector.FunctionResultWrapper return types.ArrayCompare[float64](_v1, _v2) < 0 }, selectList) + case types.T_array_bf16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.BF16](v1) + _v2 := types.BytesToArray[types.BF16](v2) + return types.ArrayElementCompare[types.BF16](_v1, _v2) < 0 + }, selectList) + case types.T_array_float16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.Float16](v1) + _v2 := types.BytesToArray[types.Float16](v2) + return types.ArrayElementCompare[types.Float16](_v1, _v2) < 0 + }, selectList) + case types.T_array_int8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[int8](v1) + _v2 := types.BytesToArray[int8](v2) + return types.ArrayElementCompare[int8](_v1, _v2) < 0 + }, selectList) + case types.T_array_uint8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[uint8](v1) + _v2 := types.BytesToArray[uint8](v2) + return types.ArrayElementCompare[uint8](_v1, _v2) < 0 + }, selectList) case types.T_date: return opBinaryFixedFixedToFixed[types.Date, types.Date, bool](parameters, rs, proc, length, func(a, b types.Date) bool { return a < b @@ -1270,6 +1408,30 @@ func lessEqualFn(parameters []*vector.Vector, result vector.FunctionResultWrappe return types.ArrayCompare[float64](_v1, _v2) <= 0 }, selectList) + case types.T_array_bf16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.BF16](v1) + _v2 := types.BytesToArray[types.BF16](v2) + return types.ArrayElementCompare[types.BF16](_v1, _v2) <= 0 + }, selectList) + case types.T_array_float16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.Float16](v1) + _v2 := types.BytesToArray[types.Float16](v2) + return types.ArrayElementCompare[types.Float16](_v1, _v2) <= 0 + }, selectList) + case types.T_array_int8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[int8](v1) + _v2 := types.BytesToArray[int8](v2) + return types.ArrayElementCompare[int8](_v1, _v2) <= 0 + }, selectList) + case types.T_array_uint8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[uint8](v1) + _v2 := types.BytesToArray[uint8](v2) + return types.ArrayElementCompare[uint8](_v1, _v2) <= 0 + }, selectList) case types.T_date: return opBinaryFixedFixedToFixed[types.Date, types.Date, bool](parameters, rs, proc, length, func(a, b types.Date) bool { return a <= b diff --git a/pkg/sql/plan/function/func_compare_narrow_test.go b/pkg/sql/plan/function/func_compare_narrow_test.go new file mode 100644 index 0000000000000..8d724c2719e76 --- /dev/null +++ b/pkg/sql/plan/function/func_compare_narrow_test.go @@ -0,0 +1,79 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package function + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/testutil" + "github.com/matrixorigin/matrixone/pkg/vm/process" + "github.com/stretchr/testify/require" +) + +// TestNarrowArrayCompareSupports covers the narrow-vector (bf16/f16/int8/uint8) +// branches of the comparison-operator type-support gates. +func TestNarrowArrayCompareSupports(t *testing.T) { + for _, oid := range []types.T{ + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, + } { + typ := oid.ToType() + require.True(t, equalAndNotEqualOperatorSupports(typ, typ), oid.String()) + require.True(t, otherCompareOperatorSupports(typ, typ), oid.String()) + } +} + +type narrowCompareFn = func([]*vector.Vector, vector.FunctionResultWrapper, *process.Process, int, *FunctionSelectList) error + +// runNarrowCompareOps drives every comparison operator over three rows — (b,a), +// (a,b), (a,a) with a < b elementwise — and checks each operator's narrow-vector +// branch against the expected boolean pattern. +func runNarrowCompareOps[T types.ArrayElement](t *testing.T, proc *process.Process, oid types.T, a, b []T) { + ops := []struct { + name string + fn narrowCompareFn + exp []bool // results for rows (b?a), (a?b), (a?a) + }{ + {"equal", equalFn, []bool{false, false, true}}, + {"notEqual", notEqualFn, []bool{true, true, false}}, + {"greatThan", greatThanFn, []bool{true, false, false}}, + {"greatEqual", greatEqualFn, []bool{true, false, true}}, + {"lessThan", lessThanFn, []bool{false, true, false}}, + {"lessEqual", lessEqualFn, []bool{false, true, true}}, + } + for _, op := range ops { + inputs := []FunctionTestInput{ + NewFunctionTestInput(oid.ToType(), [][]T{b, a, a}, []bool{false, false, false}), + NewFunctionTestInput(oid.ToType(), [][]T{a, b, a}, []bool{false, false, false}), + } + expect := NewFunctionTestResult(types.T_bool.ToType(), false, op.exp, []bool{false, false, false}) + fc := NewFunctionTestCase(proc, inputs, expect, op.fn) + ok, info := fc.Run() + require.True(t, ok, info, oid.String()+"/"+op.name) + } +} + +// TestNarrowArrayCompareOps covers the narrow-vector branches of every comparison +// operator (=, <>, >, >=, <, <=) for bf16/f16/int8/uint8. +func TestNarrowArrayCompareOps(t *testing.T) { + proc := testutil.NewProcess(t) + runNarrowCompareOps(t, proc, types.T_array_bf16, + types.Float32ToBF16Slice([]float32{1, 2}), types.Float32ToBF16Slice([]float32{3, 4})) + runNarrowCompareOps(t, proc, types.T_array_float16, + types.Float32ToFloat16Slice([]float32{1, 2}), types.Float32ToFloat16Slice([]float32{3, 4})) + runNarrowCompareOps(t, proc, types.T_array_int8, []int8{1, 2}, []int8{3, 4}) + runNarrowCompareOps(t, proc, types.T_array_uint8, []uint8{1, 2}, []uint8{3, 4}) +} diff --git a/pkg/sql/plan/function/func_compare_test.go b/pkg/sql/plan/function/func_compare_test.go index f42394d00726e..04059709adbb6 100644 --- a/pkg/sql/plan/function/func_compare_test.go +++ b/pkg/sql/plan/function/func_compare_test.go @@ -567,4 +567,70 @@ func TestNullSafeEqualFn(t *testing.T) { fcTCArrF64 := NewFunctionTestCase(proc, tcArrF64.inputs, tcArrF64.expect, nullSafeEqualFn) s, info = fcTCArrF64.Run() require.True(t, s, info, tcArrF64.info) + + // Narrow array types (bf16/f16/int8/uint8) — same <=> equality pattern. + { + bf1 := types.Float32ToBF16Slice([]float32{1, 2}) + bf2 := types.Float32ToBF16Slice([]float32{3, 4}) + tc := tcTemp{ + info: "<=> array bf16 test", + inputs: []FunctionTestInput{ + NewFunctionTestInput(types.T_array_bf16.ToType(), [][]types.BF16{bf1, bf1, bf1, bf2}, []bool{false, false, false, true}), + NewFunctionTestInput(types.T_array_bf16.ToType(), [][]types.BF16{bf1, bf2, bf2, bf2}, []bool{false, false, true, true}), + }, + expect: NewFunctionTestResult(types.T_bool.ToType(), false, + []bool{true, false, false, true}, []bool{false, false, false, false}), + } + fc := NewFunctionTestCase(proc, tc.inputs, tc.expect, nullSafeEqualFn) + s, info = fc.Run() + require.True(t, s, info, tc.info) + } + { + f1 := types.Float32ToFloat16Slice([]float32{1, 2}) + f2 := types.Float32ToFloat16Slice([]float32{3, 4}) + tc := tcTemp{ + info: "<=> array f16 test", + inputs: []FunctionTestInput{ + NewFunctionTestInput(types.T_array_float16.ToType(), [][]types.Float16{f1, f1, f1, f2}, []bool{false, false, false, true}), + NewFunctionTestInput(types.T_array_float16.ToType(), [][]types.Float16{f1, f2, f2, f2}, []bool{false, false, true, true}), + }, + expect: NewFunctionTestResult(types.T_bool.ToType(), false, + []bool{true, false, false, true}, []bool{false, false, false, false}), + } + fc := NewFunctionTestCase(proc, tc.inputs, tc.expect, nullSafeEqualFn) + s, info = fc.Run() + require.True(t, s, info, tc.info) + } + { + i1 := []int8{1, 2} + i2 := []int8{3, 4} + tc := tcTemp{ + info: "<=> array int8 test", + inputs: []FunctionTestInput{ + NewFunctionTestInput(types.T_array_int8.ToType(), [][]int8{i1, i1, i1, i2}, []bool{false, false, false, true}), + NewFunctionTestInput(types.T_array_int8.ToType(), [][]int8{i1, i2, i2, i2}, []bool{false, false, true, true}), + }, + expect: NewFunctionTestResult(types.T_bool.ToType(), false, + []bool{true, false, false, true}, []bool{false, false, false, false}), + } + fc := NewFunctionTestCase(proc, tc.inputs, tc.expect, nullSafeEqualFn) + s, info = fc.Run() + require.True(t, s, info, tc.info) + } + { + u1 := []uint8{1, 2} + u2 := []uint8{3, 4} + tc := tcTemp{ + info: "<=> array uint8 test", + inputs: []FunctionTestInput{ + NewFunctionTestInput(types.T_array_uint8.ToType(), [][]uint8{u1, u1, u1, u2}, []bool{false, false, false, true}), + NewFunctionTestInput(types.T_array_uint8.ToType(), [][]uint8{u1, u2, u2, u2}, []bool{false, false, true, true}), + }, + expect: NewFunctionTestResult(types.T_bool.ToType(), false, + []bool{true, false, false, true}, []bool{false, false, false, false}), + } + fc := NewFunctionTestCase(proc, tc.inputs, tc.expect, nullSafeEqualFn) + s, info = fc.Run() + require.True(t, s, info, tc.info) + } } diff --git a/pkg/sql/plan/function/func_testcase.go b/pkg/sql/plan/function/func_testcase.go index 0bb7b7958e596..de70243dfe5fd 100644 --- a/pkg/sql/plan/function/func_testcase.go +++ b/pkg/sql/plan/function/func_testcase.go @@ -15,6 +15,7 @@ package function import ( + "bytes" "fmt" "strings" @@ -715,6 +716,27 @@ func (fc *FunctionTestCase) Run() (succeed bool, errInfo string) { i+1, types.BytesToArray[float64](want), types.BytesToArray[float64](get)) } } + case types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: + // Narrow vector types compare byte-exact (their stored representation is + // the comparison ground truth; ArrayCompare only covers float32/float64). + r := vector.GenerateFunctionStrParameter(v) + s := vector.GenerateFunctionStrParameter(vExpected) + for i = 0; i < uint64(fc.fnLength); i++ { + want, null1 := s.GetStrValue(i) + get, null2 := r.GetStrValue(i) + if null1 { + if null2 { + continue + } + return false, fmt.Sprintf("the %dth row expected NULL, but get not null", i+1) + } + if null2 { + return false, fmt.Sprintf("the %dth row expected %v, but get NULL", i+1, want) + } + if !bytes.Equal(want, get) { + return false, fmt.Sprintf("the %dth row expected %v, but get %v", i+1, want, get) + } + } case types.T_uuid: r := vector.GenerateFunctionFixedTypeParameter[types.Uuid](v) s := vector.GenerateFunctionFixedTypeParameter[types.Uuid](vExpected) @@ -917,6 +939,18 @@ func newVectorByType(mp *mpool.MPool, typ types.Type, val any, nsp *nulls.Nulls) case types.T_array_float64: values := val.([][]float64) vector.AppendArrayList[float64](vec, values, nil, mp) + case types.T_array_bf16: + values := val.([][]types.BF16) + vector.AppendArrayList[types.BF16](vec, values, nil, mp) + case types.T_array_float16: + values := val.([][]types.Float16) + vector.AppendArrayList[types.Float16](vec, values, nil, mp) + case types.T_array_int8: + values := val.([][]int8) + vector.AppendArrayList[int8](vec, values, nil, mp) + case types.T_array_uint8: + values := val.([][]uint8) + vector.AppendArrayList[uint8](vec, values, nil, mp) case types.T_uuid: values := val.([]types.Uuid) vector.AppendFixedList(vec, values, nil, mp) diff --git a/pkg/sql/plan/function/func_unary.go b/pkg/sql/plan/function/func_unary.go index 7ba49577bbbe8..91995cb284eaa 100644 --- a/pkg/sql/plan/function/func_unary.go +++ b/pkg/sql/plan/function/func_unary.go @@ -233,7 +233,7 @@ var ( } ) -func NormalizeL2Array[T types.RealNumbers](parameters []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { +func NormalizeL2Array[T types.ArrayElement](parameters []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { source := vector.GenerateFunctionStrParameter(parameters[0]) rs := vector.MustFunctionResult[types.Varlena](result) @@ -290,6 +290,14 @@ func NormalizeL2Array[T types.RealNumbers](parameters []*vector.Vector, result v *outArrayF64Ptr = outArrayF64 arrayF64Pool.Put(outArrayF64Ptr) + case types.T_array_bf16: + _ = appendNormalizedNarrowArray[types.BF16](rs, data) + case types.T_array_float16: + _ = appendNormalizedNarrowArray[types.Float16](rs, data) + case types.T_array_int8: + _ = appendNormalizedNarrowArray[int8](rs, data) + case types.T_array_uint8: + _ = appendNormalizedNarrowArray[uint8](rs, data) } } @@ -297,6 +305,17 @@ func NormalizeL2Array[T types.RealNumbers](parameters []*vector.Vector, result v return nil } +// appendNormalizedNarrowArray normalizes a narrow-typed vector (bf16/f16/int8) +// by upcasting to float32, normalizing in float32, then narrowing back to T. +// int8 normalization is mostly degenerate (unit vectors round to 0/±1) but is +// supported for completeness. +func appendNormalizedNarrowArray[T types.ArrayElement](rs *vector.FunctionResult[types.Varlena], data []byte) error { + in := types.ToFloat32Array[T](types.BytesToArray[T](data)) + out := make([]float32, len(in)) + _ = moarray.NormalizeL2(in, out) + return rs.AppendBytes(types.ArrayToBytes[T](types.FromFloat32Array[T](out)), false) +} + func L1NormArray[T types.RealNumbers](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { return opUnaryBytesToFixedWithErrorCheck[float64](ivecs, result, proc, length, func(in []byte) (float64, error) { _in := types.BytesToArray[T](in) @@ -311,7 +330,7 @@ func L2NormArray[T types.RealNumbers](ivecs []*vector.Vector, result vector.Func }, selectList) } -func VectorDimsArray[T types.RealNumbers](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { +func VectorDimsArray[T types.ArrayElement](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { return opUnaryBytesToFixed[int64](ivecs, result, proc, length, func(in []byte) (out int64) { _in := types.BytesToArray[T](in) return int64(len(_in)) @@ -634,6 +653,12 @@ func bitCountFromFloat[T constraints.Float](v T, proc *process.Process) (uint64, if rounded >= ULLONG_MAX_DOUBLE { return bitCountFromUint64(uint64(math.MaxUint64)), nil } + // Converting a negative float directly to uint64 is undefined in Go (the + // result is implementation-specific: two's-complement on amd64, 0 on arm64), + // so route negatives through int64 first and reinterpret the bit pattern. + if rounded < 0 { + return bitCountFromSignedInt64Pattern(int64(rounded)), nil + } return bitCountFromUint64(uint64(rounded)), nil } @@ -5688,7 +5713,7 @@ func FromBase64(parameters []*vector.Vector, result vector.FunctionResultWrapper // VecFromBase64 decodes a base64-encoded string into a vector (vecf32 or vecf64). // The base64 payload must be the raw little-endian bytes of the vector elements, // as produced by to_base64(vecf32_col) or to_base64(vecf64_col). -func VecFromBase64[T types.RealNumbers](parameters []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { +func VecFromBase64[T types.ArrayElement](parameters []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { source := vector.GenerateFunctionStrParameter(parameters[0]) rs := vector.MustFunctionResult[types.Varlena](result) @@ -5698,6 +5723,14 @@ func VecFromBase64[T types.RealNumbers](parameters []*vector.Vector, result vect elemSize = 4 case float64: elemSize = 8 + case types.BF16, types.Float16: + elemSize = 2 + case int8, uint8: + elemSize = 1 + default: + // Guard: an unhandled element type would leave elemSize==0 and panic at + // the `n % elemSize` check below. Fail explicitly instead. + return moerr.NewInternalErrorNoCtx("vec_from_base64: unsupported vector element type") } // Pre-extend area: peek at the first non-null input to estimate per-row decoded size. @@ -5731,11 +5764,11 @@ func VecFromBase64[T types.RealNumbers](parameters []*vector.Vector, result vect } n, err := base64.StdEncoding.Decode(buf, data) if err != nil { - return moerr.NewInternalErrorNoCtxf("vecf%d_from_base64: invalid base64 input", elemSize*8) + return moerr.NewInternalErrorNoCtx("vec_from_base64: invalid base64 input") } if n%elemSize != 0 { - return moerr.NewInternalErrorNoCtxf("vecf%d_from_base64: decoded length %d is not a multiple of %d bytes", elemSize*8, n, elemSize) + return moerr.NewInternalErrorNoCtxf("vec_from_base64: decoded length %d is not a multiple of %d bytes", n, elemSize) } if err = rs.AppendBytes(buf[:n], false); err != nil { diff --git a/pkg/sql/plan/function/func_unary_test.go b/pkg/sql/plan/function/func_unary_test.go index 35b2da0038bb1..5fa4f3fe07d21 100644 --- a/pkg/sql/plan/function/func_unary_test.go +++ b/pkg/sql/plan/function/func_unary_test.go @@ -4507,6 +4507,55 @@ func TestVecFromBase64(t *testing.T) { require.True(t, s, fmt.Sprintf("vecf64 case failed: %s", info)) } +// TestVecFromBase64Narrow exercises VecFromBase64's narrow elemSize branches +// (int8=1, bf16/f16=2) and its error paths via the function-UT harness. +func TestVecFromBase64Narrow(t *testing.T) { + proc := testutil.NewProcess(t) + + mkInput := func(b64 string) []FunctionTestInput { + return []FunctionTestInput{NewFunctionTestInput(types.T_varchar.ToType(), []string{b64}, []bool{})} + } + runCase := func(in []FunctionTestInput, res FunctionTestResult, fn fEvalFn) (bool, string) { + fcTC := NewFunctionTestCase(proc, in, res, fn) + return fcTC.Run() + } + + // int8 roundtrip (elemSize 1). + i8 := []int8{1, -2, 127, -128} + ok, info := runCase(mkInput(types.ArrayToBase64(i8)), + NewFunctionTestResult(types.T_array_int8.ToType(), false, [][]int8{i8}, []bool{}), VecFromBase64[int8]) + require.Truef(t, ok, "vecint8 roundtrip: %s", info) + + // uint8 roundtrip (elemSize 1). Regression: with the uint8 case missing from + // the decoder, elemSize was 0 and `n % elemSize` panicked (divide by zero). + u8 := []uint8{0, 255, 128, 1} + ok, info = runCase(mkInput(types.ArrayToBase64(u8)), + NewFunctionTestResult(types.T_array_uint8.ToType(), false, [][]uint8{u8}, []bool{}), VecFromBase64[uint8]) + require.Truef(t, ok, "vecuint8 roundtrip: %s", info) + + // bf16 roundtrip (elemSize 2). + bf := types.Float32ToBF16Slice([]float32{1.5, -2.25, 0, 8}) + ok, info = runCase(mkInput(types.ArrayToBase64(bf)), + NewFunctionTestResult(types.T_array_bf16.ToType(), false, [][]types.BF16{bf}, []bool{}), VecFromBase64[types.BF16]) + require.Truef(t, ok, "vecbf16 roundtrip: %s", info) + + // f16 roundtrip. + f16 := types.Float32ToFloat16Slice([]float32{1.5, -2.25, 0, 8}) + ok, info = runCase(mkInput(types.ArrayToBase64(f16)), + NewFunctionTestResult(types.T_array_float16.ToType(), false, [][]types.Float16{f16}, []bool{}), VecFromBase64[types.Float16]) + require.Truef(t, ok, "vecf16 roundtrip: %s", info) + + // invalid base64 -> error. + ok, info = runCase(mkInput("!!!not-base64!!!"), + NewFunctionTestResult(types.T_array_int8.ToType(), true, [][]int8{nil}, []bool{}), VecFromBase64[int8]) + require.Truef(t, ok, "invalid base64 should error: %s", info) + + // "AQID" decodes to 3 bytes, not a multiple of 2 (bf16 elemSize) -> error. + ok, info = runCase(mkInput("AQID"), + NewFunctionTestResult(types.T_array_bf16.ToType(), true, [][]types.BF16{nil}, []bool{}), VecFromBase64[types.BF16]) + require.Truef(t, ok, "odd length should error: %s", info) +} + func initValidatePasswordStrengthTestCase() []tcTemp { return []tcTemp{ { diff --git a/pkg/sql/plan/function/func_vecnarrow_test.go b/pkg/sql/plan/function/func_vecnarrow_test.go new file mode 100644 index 0000000000000..cdd24f80074ae --- /dev/null +++ b/pkg/sql/plan/function/func_vecnarrow_test.go @@ -0,0 +1,122 @@ +// Copyright 2021 - 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package function + +import ( + "fmt" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/testutil" + "github.com/stretchr/testify/require" +) + +// l2 of [1,2,3] vs [4,6,8]: sqrt(9+16+25)=sqrt(50)=7.0710678118654755 +// (the distance is computed in float64; the framework's InEpsilonF64 1e-9 +// tolerance absorbs cross-platform variance). +func TestL2DistanceNarrowArray(t *testing.T) { + proc := testutil.NewProcess(t) + + // int8: exact integer values, so distance matches float reference exactly. + t.Run("int8", func(t *testing.T) { + tc := NewFunctionTestCase(proc, + []FunctionTestInput{ + NewFunctionTestInput(types.T_array_int8.ToType(), [][]int8{{1, 2, 3}}, []bool{false}), + NewFunctionTestInput(types.T_array_int8.ToType(), [][]int8{{4, 6, 8}}, []bool{false}), + }, + NewFunctionTestResult(types.T_float64.ToType(), false, []float64{7.0710678118654755}, []bool{false}), + L2DistanceArrayViaF32[int8]) + s, info := tc.Run() + require.True(t, s, info) + }) + + // uint8: exact unsigned integer values, distance matches the float reference. + t.Run("uint8", func(t *testing.T) { + tc := NewFunctionTestCase(proc, + []FunctionTestInput{ + NewFunctionTestInput(types.T_array_uint8.ToType(), [][]uint8{{1, 2, 3}}, []bool{false}), + NewFunctionTestInput(types.T_array_uint8.ToType(), [][]uint8{{4, 6, 8}}, []bool{false}), + }, + NewFunctionTestResult(types.T_float64.ToType(), false, []float64{7.0710678118654755}, []bool{false}), + L2DistanceArrayViaF32[uint8]) + s, info := tc.Run() + require.True(t, s, info) + }) + + // bf16: small integers are exactly representable in bf16, so still exact. + t.Run("bf16", func(t *testing.T) { + mk := func(vs ...float32) []types.BF16 { + out := make([]types.BF16, len(vs)) + for i, v := range vs { + out[i] = types.BF16FromFloat32(v) + } + return out + } + tc := NewFunctionTestCase(proc, + []FunctionTestInput{ + NewFunctionTestInput(types.T_array_bf16.ToType(), [][]types.BF16{mk(1, 2, 3)}, []bool{false}), + NewFunctionTestInput(types.T_array_bf16.ToType(), [][]types.BF16{mk(4, 6, 8)}, []bool{false}), + }, + NewFunctionTestResult(types.T_float64.ToType(), false, []float64{7.0710678118654755}, []bool{false}), + L2DistanceArrayViaF32[types.BF16]) + s, info := tc.Run() + require.True(t, s, info) + }) + + // float16: same exact small integers. + t.Run("f16", func(t *testing.T) { + mk := func(vs ...float32) []types.Float16 { + out := make([]types.Float16, len(vs)) + for i, v := range vs { + out[i] = types.Float16FromFloat32(v) + } + return out + } + tc := NewFunctionTestCase(proc, + []FunctionTestInput{ + NewFunctionTestInput(types.T_array_float16.ToType(), [][]types.Float16{mk(1, 2, 3)}, []bool{false}), + NewFunctionTestInput(types.T_array_float16.ToType(), [][]types.Float16{mk(4, 6, 8)}, []bool{false}), + }, + NewFunctionTestResult(types.T_float64.ToType(), false, []float64{7.0710678118654755}, []bool{false}), + L2DistanceArrayViaF32[types.Float16]) + s, info := tc.Run() + require.True(t, s, info) + }) +} + +// Sanity: inner_product of [1,2,3]·[4,5,6] = 4+10+18 = 32. +func TestInnerProductNarrowArray(t *testing.T) { + proc := testutil.NewProcess(t) + tc := NewFunctionTestCase(proc, + []FunctionTestInput{ + NewFunctionTestInput(types.T_array_int8.ToType(), [][]int8{{1, 2, 3}}, []bool{false}), + NewFunctionTestInput(types.T_array_int8.ToType(), [][]int8{{4, 5, 6}}, []bool{false}), + }, + NewFunctionTestResult(types.T_float64.ToType(), false, []float64{-32}, []bool{false}), + InnerProductArrayViaF32[int8]) + s, info := tc.Run() + require.True(t, s, fmt.Sprintf("inner_product int8: %s", info)) + + // uint8 sibling: same dot product over unsigned values. + tc = NewFunctionTestCase(proc, + []FunctionTestInput{ + NewFunctionTestInput(types.T_array_uint8.ToType(), [][]uint8{{1, 2, 3}}, []bool{false}), + NewFunctionTestInput(types.T_array_uint8.ToType(), [][]uint8{{4, 5, 6}}, []bool{false}), + }, + NewFunctionTestResult(types.T_float64.ToType(), false, []float64{-32}, []bool{false}), + InnerProductArrayViaF32[uint8]) + s, info = tc.Run() + require.True(t, s, fmt.Sprintf("inner_product uint8: %s", info)) +} diff --git a/pkg/sql/plan/function/function_id.go b/pkg/sql/plan/function/function_id.go index 95cecd118a9ba..375c06a40d406 100644 --- a/pkg/sql/plan/function/function_id.go +++ b/pkg/sql/plan/function/function_id.go @@ -765,9 +765,26 @@ const ( JSON_CONTAINS = 543 JSON_REMOVE = 544 + // vec{bf16,f16,int8}_from_base64: decode a base64 payload of the narrow type's + // raw bytes into that narrow vector type — the narrow siblings of + // vecf32_from_base64 / vecf64_from_base64. Used by the ivfflat narrow re-rank, + // where the query must be a constant narrow vec literal matching the narrow + // entries (a cast of vecf32_from_base64 does not constant-fold, breaking the + // ORDER BY index pushdown). + // Renumbered after the main merge, which took 524-544 for the S2/H3/ST_POINT/ + // CAST_STRICT/DATE_TRUNC/JSON_CONTAINS/JSON_REMOVE functions. These IDs are + // referenced by name only (name map + list_builtIn registration), so renumbering + // is safe. + VECBF16_FROM_BASE64 = 545 + VECF16_FROM_BASE64 = 546 + VECINT8_FROM_BASE64 = 547 + VECUINT8_FROM_BASE64 = 548 + + BM25_MATCH = 549 + // FUNCTION_END_NUMBER is not a function, just a flag to record the max number of function. // TODO: every one should put the new function id in front of this one if you want to make a new function. - FUNCTION_END_NUMBER = 545 + FUNCTION_END_NUMBER = 550 ) // functionIdRegister is what function we have registered already. @@ -1080,6 +1097,10 @@ var functionIdRegister = map[string]int32{ "from_base64": FROM_BASE64, "vecf32_from_base64": VECF32_FROM_BASE64, "vecf64_from_base64": VECF64_FROM_BASE64, + "vecbf16_from_base64": VECBF16_FROM_BASE64, + "vecf16_from_base64": VECF16_FROM_BASE64, + "vecint8_from_base64": VECINT8_FROM_BASE64, + "vecuint8_from_base64": VECUINT8_FROM_BASE64, "serial": SERIAL, "serial_full": SERIAL_FULL, "serial_extract": SERIAL_EXTRACT, @@ -1346,6 +1367,7 @@ var functionIdRegister = map[string]int32{ // match function "fulltext_match": FULLTEXT_MATCH, "fulltext_match_score": FULLTEXT_MATCH_SCORE, + "bm25_match": BM25_MATCH, // starlark function "starlark": STARLARK, diff --git a/pkg/sql/plan/function/function_id_test.go b/pkg/sql/plan/function/function_id_test.go index 66cd8257483f5..5ef560e344f6c 100644 --- a/pkg/sql/plan/function/function_id_test.go +++ b/pkg/sql/plan/function/function_id_test.go @@ -598,9 +598,14 @@ var predefinedFunids = map[int]int{ DATE_TRUNC: 542, JSON_CONTAINS: 543, JSON_REMOVE: 544, + VECBF16_FROM_BASE64: 545, + VECF16_FROM_BASE64: 546, + VECINT8_FROM_BASE64: 547, + VECUINT8_FROM_BASE64: 548, + BM25_MATCH: 549, // FUNCTION_END_NUMBER is not a function, just a flag to record the max number of function. // TODO: every one should put the new function id in front of this one if you want to make a new function. - FUNCTION_END_NUMBER: 545, + FUNCTION_END_NUMBER: 550, } func Test_funids(t *testing.T) { diff --git a/pkg/sql/plan/function/list_agg.go b/pkg/sql/plan/function/list_agg.go index 0c4c5c6b2f04c..dee8768ff5d07 100644 --- a/pkg/sql/plan/function/list_agg.go +++ b/pkg/sql/plan/function/list_agg.go @@ -733,6 +733,8 @@ var AnyValueSupportedTypes = []types.T{ types.T_uuid, types.T_binary, types.T_varbinary, types.T_json, types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, + types.T_array_int8, types.T_array_uint8, types.T_geometry, types.T_geometry32, types.T_enum, types.T_Rowid, diff --git a/pkg/sql/plan/function/list_builtIn.go b/pkg/sql/plan/function/list_builtIn.go index e896bbc6eec7a..9f2a09a1bbf78 100644 --- a/pkg/sql/plan/function/list_builtIn.go +++ b/pkg/sql/plan/function/list_builtIn.go @@ -3108,6 +3108,90 @@ var supportedStringBuiltIns = []FuncNew{ }, }, + // vecbf16_from_base64 + { + functionId: VECBF16_FROM_BASE64, + class: plan.Function_STRICT, + layout: STANDARD_FUNCTION, + checkFn: fixedTypeMatch, + + Overloads: []overload{ + { + overloadId: 0, + args: []types.T{types.T_varchar}, + retType: func(parameters []types.Type) types.Type { + return types.T_array_bf16.ToType() + }, + newOp: func() executeLogicOfOverload { + return VecFromBase64[types.BF16] + }, + }, + }, + }, + + // vecf16_from_base64 + { + functionId: VECF16_FROM_BASE64, + class: plan.Function_STRICT, + layout: STANDARD_FUNCTION, + checkFn: fixedTypeMatch, + + Overloads: []overload{ + { + overloadId: 0, + args: []types.T{types.T_varchar}, + retType: func(parameters []types.Type) types.Type { + return types.T_array_float16.ToType() + }, + newOp: func() executeLogicOfOverload { + return VecFromBase64[types.Float16] + }, + }, + }, + }, + + // vecint8_from_base64 + { + functionId: VECINT8_FROM_BASE64, + class: plan.Function_STRICT, + layout: STANDARD_FUNCTION, + checkFn: fixedTypeMatch, + + Overloads: []overload{ + { + overloadId: 0, + args: []types.T{types.T_varchar}, + retType: func(parameters []types.Type) types.Type { + return types.T_array_int8.ToType() + }, + newOp: func() executeLogicOfOverload { + return VecFromBase64[int8] + }, + }, + }, + }, + + // vecuint8_from_base64 + { + functionId: VECUINT8_FROM_BASE64, + class: plan.Function_STRICT, + layout: STANDARD_FUNCTION, + checkFn: fixedTypeMatch, + + Overloads: []overload{ + { + overloadId: 0, + args: []types.T{types.T_varchar}, + retType: func(parameters []types.Type) types.Type { + return types.T_array_uint8.ToType() + }, + newOp: func() executeLogicOfOverload { + return VecFromBase64[uint8] + }, + }, + }, + }, + // compress { functionId: COMPRESS, @@ -6129,6 +6213,30 @@ var supportedArrayOperations = []FuncNew{ return VectorDimsArray[float64] }, }, + { + overloadId: 2, + args: []types.T{types.T_array_bf16}, + retType: func(parameters []types.Type) types.Type { return types.T_int64.ToType() }, + newOp: func() executeLogicOfOverload { return VectorDimsArray[types.BF16] }, + }, + { + overloadId: 3, + args: []types.T{types.T_array_float16}, + retType: func(parameters []types.Type) types.Type { return types.T_int64.ToType() }, + newOp: func() executeLogicOfOverload { return VectorDimsArray[types.Float16] }, + }, + { + overloadId: 4, + args: []types.T{types.T_array_int8}, + retType: func(parameters []types.Type) types.Type { return types.T_int64.ToType() }, + newOp: func() executeLogicOfOverload { return VectorDimsArray[int8] }, + }, + { + overloadId: 5, + args: []types.T{types.T_array_uint8}, + retType: func(parameters []types.Type) types.Type { return types.T_int64.ToType() }, + newOp: func() executeLogicOfOverload { return VectorDimsArray[uint8] }, + }, }, }, @@ -6160,6 +6268,30 @@ var supportedArrayOperations = []FuncNew{ return InnerProductArray[float64] }, }, + { + overloadId: 2, + args: []types.T{types.T_array_bf16, types.T_array_bf16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return InnerProductArrayViaF32[types.BF16] }, + }, + { + overloadId: 3, + args: []types.T{types.T_array_float16, types.T_array_float16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return InnerProductArrayViaF32[types.Float16] }, + }, + { + overloadId: 4, + args: []types.T{types.T_array_int8, types.T_array_int8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return InnerProductArrayViaF32[int8] }, + }, + { + overloadId: 5, + args: []types.T{types.T_array_uint8, types.T_array_uint8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return InnerProductArrayViaF32[uint8] }, + }, }, }, @@ -6191,6 +6323,30 @@ var supportedArrayOperations = []FuncNew{ return CosineSimilarityArray[float64] }, }, + { + overloadId: 2, + args: []types.T{types.T_array_bf16, types.T_array_bf16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return CosineSimilarityArrayViaF32[types.BF16] }, + }, + { + overloadId: 3, + args: []types.T{types.T_array_float16, types.T_array_float16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return CosineSimilarityArrayViaF32[types.Float16] }, + }, + { + overloadId: 4, + args: []types.T{types.T_array_int8, types.T_array_int8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return CosineSimilarityArrayViaF32[int8] }, + }, + { + overloadId: 5, + args: []types.T{types.T_array_uint8, types.T_array_uint8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return CosineSimilarityArrayViaF32[uint8] }, + }, }, }, @@ -6222,6 +6378,30 @@ var supportedArrayOperations = []FuncNew{ return L2DistanceArray[float64] }, }, + { + overloadId: 2, + args: []types.T{types.T_array_bf16, types.T_array_bf16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return L2DistanceArrayViaF32[types.BF16] }, + }, + { + overloadId: 3, + args: []types.T{types.T_array_float16, types.T_array_float16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return L2DistanceArrayViaF32[types.Float16] }, + }, + { + overloadId: 4, + args: []types.T{types.T_array_int8, types.T_array_int8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return L2DistanceArrayViaF32[int8] }, + }, + { + overloadId: 5, + args: []types.T{types.T_array_uint8, types.T_array_uint8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return L2DistanceArrayViaF32[uint8] }, + }, }, }, @@ -6284,6 +6464,30 @@ var supportedArrayOperations = []FuncNew{ return L2DistanceSqArray[float64] }, }, + { + overloadId: 2, + args: []types.T{types.T_array_bf16, types.T_array_bf16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return L2DistanceSqArrayViaF32[types.BF16] }, + }, + { + overloadId: 3, + args: []types.T{types.T_array_float16, types.T_array_float16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return L2DistanceSqArrayViaF32[types.Float16] }, + }, + { + overloadId: 4, + args: []types.T{types.T_array_int8, types.T_array_int8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return L2DistanceSqArrayViaF32[int8] }, + }, + { + overloadId: 5, + args: []types.T{types.T_array_uint8, types.T_array_uint8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return L2DistanceSqArrayViaF32[uint8] }, + }, }, }, @@ -6346,6 +6550,30 @@ var supportedArrayOperations = []FuncNew{ return CosineDistanceArray[float64] }, }, + { + overloadId: 2, + args: []types.T{types.T_array_bf16, types.T_array_bf16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return CosineDistanceArrayViaF32[types.BF16] }, + }, + { + overloadId: 3, + args: []types.T{types.T_array_float16, types.T_array_float16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return CosineDistanceArrayViaF32[types.Float16] }, + }, + { + overloadId: 4, + args: []types.T{types.T_array_int8, types.T_array_int8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return CosineDistanceArrayViaF32[int8] }, + }, + { + overloadId: 5, + args: []types.T{types.T_array_uint8, types.T_array_uint8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return CosineDistanceArrayViaF32[uint8] }, + }, }, }, // function `normalize_l2` @@ -6376,6 +6604,30 @@ var supportedArrayOperations = []FuncNew{ return NormalizeL2Array[float64] }, }, + { + overloadId: 3, + args: []types.T{types.T_array_bf16}, + retType: func(parameters []types.Type) types.Type { return parameters[0] }, + newOp: func() executeLogicOfOverload { return NormalizeL2Array[types.BF16] }, + }, + { + overloadId: 4, + args: []types.T{types.T_array_float16}, + retType: func(parameters []types.Type) types.Type { return parameters[0] }, + newOp: func() executeLogicOfOverload { return NormalizeL2Array[types.Float16] }, + }, + { + overloadId: 5, + args: []types.T{types.T_array_int8}, + retType: func(parameters []types.Type) types.Type { return parameters[0] }, + newOp: func() executeLogicOfOverload { return NormalizeL2Array[int8] }, + }, + { + overloadId: 6, + args: []types.T{types.T_array_uint8}, + retType: func(parameters []types.Type) types.Type { return parameters[0] }, + newOp: func() executeLogicOfOverload { return NormalizeL2Array[uint8] }, + }, }, }, // function `subvector` @@ -13757,6 +14009,21 @@ var supportedOthersBuiltIns = []FuncNew{ Overloads: fulltext_expand_overload(types.T_float32), }, + // function `BM25_MATCH` — bm25's ranked-retrieval surface, BM25(col) AGAINST('q'). + // Same signature as fulltext_match ([pattern, mode, cols...] -> bool/float32); the + // distinct name is what lets the planner route it to bm25_search. If it ever + // reaches execution unrewritten (no bm25 index) it errors like fulltext_match. + // (No separate bm25_match_score: a matched bm25 projection is replaced by the + // join's score ColRef, and an unmatched one keeps this function's float32 overload.) + { + functionId: BM25_MATCH, + class: plan.Function_STRICT, + layout: STANDARD_FUNCTION, + checkFn: fixedDirectlyTypeMatch, + + Overloads: fulltext_expand_overload(types.T_float32), + }, + // function `mo_tuple_expr` { functionId: MO_TUPLE_EXPR, diff --git a/pkg/sql/plan/function/type_check.go b/pkg/sql/plan/function/type_check.go index 1badc44d91ce9..3e75d136f4dd1 100644 --- a/pkg/sql/plan/function/type_check.go +++ b/pkg/sql/plan/function/type_check.go @@ -1035,6 +1035,10 @@ func initFixed1() { {types.T_varchar, types.T_text, types.T_varchar, types.T_varchar}, {types.T_varchar, types.T_array_float32, types.T_array_float32, types.T_array_float32}, {types.T_varchar, types.T_array_float64, types.T_array_float64, types.T_array_float64}, + {types.T_varchar, types.T_array_bf16, types.T_array_bf16, types.T_array_bf16}, + {types.T_varchar, types.T_array_float16, types.T_array_float16, types.T_array_float16}, + {types.T_varchar, types.T_array_int8, types.T_array_int8, types.T_array_int8}, + {types.T_varchar, types.T_array_uint8, types.T_array_uint8, types.T_array_uint8}, {types.T_json, types.T_any, types.T_json, types.T_json}, {types.T_json, types.T_bool, types.T_bool, types.T_bool}, {types.T_json, types.T_int8, types.T_int8, types.T_int8}, @@ -1174,6 +1178,21 @@ func initFixed1() { {types.T_text, types.T_array_float32, types.T_array_float32, types.T_array_float32}, {types.T_array_float64, types.T_text, types.T_array_float64, types.T_array_float64}, {types.T_text, types.T_array_float64, types.T_array_float64, types.T_array_float64}, + // narrow vector types: string<->narrow for comparison/equality only. + // (No scalar-arithmetic rules below are added for these types, so + - * / + // still fail to resolve — arithmetic requires an explicit CAST to vecf32.) + {types.T_array_bf16, types.T_varchar, types.T_array_bf16, types.T_array_bf16}, + {types.T_array_bf16, types.T_text, types.T_array_bf16, types.T_array_bf16}, + {types.T_text, types.T_array_bf16, types.T_array_bf16, types.T_array_bf16}, + {types.T_array_float16, types.T_varchar, types.T_array_float16, types.T_array_float16}, + {types.T_array_float16, types.T_text, types.T_array_float16, types.T_array_float16}, + {types.T_text, types.T_array_float16, types.T_array_float16, types.T_array_float16}, + {types.T_array_int8, types.T_varchar, types.T_array_int8, types.T_array_int8}, + {types.T_array_int8, types.T_text, types.T_array_int8, types.T_array_int8}, + {types.T_text, types.T_array_int8, types.T_array_int8, types.T_array_int8}, + {types.T_array_uint8, types.T_varchar, types.T_array_uint8, types.T_array_uint8}, + {types.T_array_uint8, types.T_text, types.T_array_uint8, types.T_array_uint8}, + {types.T_text, types.T_array_uint8, types.T_array_uint8, types.T_array_uint8}, /** VEC Scalar => VEC **/ // VECF32 Scalar => VECF32 @@ -1717,6 +1736,10 @@ func initFixed2() { //A {types.T_varchar, types.T_array_float32, types.T_array_float32, types.T_array_float32}, {types.T_varchar, types.T_array_float64, types.T_array_float64, types.T_array_float64}, + {types.T_varchar, types.T_array_bf16, types.T_array_bf16, types.T_array_bf16}, + {types.T_varchar, types.T_array_float16, types.T_array_float16, types.T_array_float16}, + {types.T_varchar, types.T_array_int8, types.T_array_int8, types.T_array_int8}, + {types.T_varchar, types.T_array_uint8, types.T_array_uint8, types.T_array_uint8}, {types.T_binary, types.T_any, types.T_float64, types.T_float64}, {types.T_binary, types.T_int8, types.T_float64, types.T_float64}, {types.T_binary, types.T_int16, types.T_float64, types.T_float64}, @@ -1791,6 +1814,15 @@ func initFixed2() { {types.T_array_float32, types.T_array_float32, types.T_array_float32, types.T_array_float32}, {types.T_array_float64, types.T_varchar, types.T_array_float64, types.T_array_float64}, {types.T_array_float64, types.T_array_float32, types.T_array_float64, types.T_array_float64}, + // narrow vector types: narrow<->string for comparison/equality only. + {types.T_array_bf16, types.T_varchar, types.T_array_bf16, types.T_array_bf16}, + {types.T_array_bf16, types.T_array_bf16, types.T_array_bf16, types.T_array_bf16}, + {types.T_array_float16, types.T_varchar, types.T_array_float16, types.T_array_float16}, + {types.T_array_float16, types.T_array_float16, types.T_array_float16, types.T_array_float16}, + {types.T_array_int8, types.T_varchar, types.T_array_int8, types.T_array_int8}, + {types.T_array_int8, types.T_array_int8, types.T_array_int8, types.T_array_int8}, + {types.T_array_uint8, types.T_varchar, types.T_array_uint8, types.T_array_uint8}, + {types.T_array_uint8, types.T_array_uint8, types.T_array_uint8, types.T_array_uint8}, /** VEC Scalar => VEC **/ // VECF32 Scalar => VECF32 {types.T_array_float32, types.T_int32, types.T_array_float32, types.T_float32}, @@ -2289,6 +2321,10 @@ func initFixed3() { //C {toType: types.T_array_float32, preferLevel: 2}, {toType: types.T_array_float64, preferLevel: 2}, + {toType: types.T_array_bf16, preferLevel: 2}, + {toType: types.T_array_float16, preferLevel: 2}, + {toType: types.T_array_int8, preferLevel: 2}, + {toType: types.T_array_uint8, preferLevel: 2}, }, }, @@ -2407,6 +2443,10 @@ func initFixed3() { {toType: types.T_blob, preferLevel: 2}, {toType: types.T_array_float32, preferLevel: 2}, {toType: types.T_array_float64, preferLevel: 2}, + {toType: types.T_array_bf16, preferLevel: 2}, + {toType: types.T_array_float16, preferLevel: 2}, + {toType: types.T_array_int8, preferLevel: 2}, + {toType: types.T_array_uint8, preferLevel: 2}, }, }, { diff --git a/pkg/sql/plan/make.go b/pkg/sql/plan/make.go index 49cae2a00bd2a..0e6af79bdd649 100644 --- a/pkg/sql/plan/make.go +++ b/pkg/sql/plan/make.go @@ -370,6 +370,62 @@ func makePlan2Vecf64ConstExprWithType(v string, l int32) *plan.Expr { } } +var MakePlan2VecBf16ConstExprWithType = makePlan2VecBf16ConstExprWithType + +// makePlan2VecBf16ConstExprWithType makes a vecbf16 const expr. +func makePlan2VecBf16ConstExprWithType(v string, l int32) *plan.Expr { + return &plan.Expr{ + Expr: makePlan2Vecf32ConstExpr(v), + Typ: plan.Type{ + Id: int32(types.T_array_bf16), + Width: l, + NotNullable: true, + }, + } +} + +var MakePlan2VecF16ConstExprWithType = makePlan2VecF16ConstExprWithType + +// makePlan2VecF16ConstExprWithType makes a vecf16 const expr. +func makePlan2VecF16ConstExprWithType(v string, l int32) *plan.Expr { + return &plan.Expr{ + Expr: makePlan2Vecf32ConstExpr(v), + Typ: plan.Type{ + Id: int32(types.T_array_float16), + Width: l, + NotNullable: true, + }, + } +} + +var MakePlan2VecInt8ConstExprWithType = makePlan2VecInt8ConstExprWithType + +// makePlan2VecInt8ConstExprWithType makes a vecint8 const expr. +func makePlan2VecInt8ConstExprWithType(v string, l int32) *plan.Expr { + return &plan.Expr{ + Expr: makePlan2Vecf32ConstExpr(v), + Typ: plan.Type{ + Id: int32(types.T_array_int8), + Width: l, + NotNullable: true, + }, + } +} + +var MakePlan2VecUint8ConstExprWithType = makePlan2VecUint8ConstExprWithType + +// makePlan2VecUint8ConstExprWithType makes a vecuint8 const expr. +func makePlan2VecUint8ConstExprWithType(v string, l int32) *plan.Expr { + return &plan.Expr{ + Expr: makePlan2Vecf32ConstExpr(v), + Typ: plan.Type{ + Id: int32(types.T_array_uint8), + Width: l, + NotNullable: true, + }, + } +} + var MakePlan2StringVecExprWithType = makePlan2StringVecExprWithType func makePlan2StringVecExprWithType(mp *mpool.MPool, vals ...string) *plan.Expr { diff --git a/pkg/sql/plan/make_test.go b/pkg/sql/plan/make_test.go index 4b7183263afc5..c30256152001e 100644 --- a/pkg/sql/plan/make_test.go +++ b/pkg/sql/plan/make_test.go @@ -55,6 +55,29 @@ func Test_MakePlan2Vecf64ConstExprWithType(t *testing.T) { require.Equal(t, "[1,2,3]", actual) } +func Test_MakePlan2VecNarrowConstExprWithType(t *testing.T) { + cases := []struct { + name string + fn func(string, int32) *plan.Expr + oid types.T + }{ + {"bf16", MakePlan2VecBf16ConstExprWithType, types.T_array_bf16}, + {"f16", MakePlan2VecF16ConstExprWithType, types.T_array_float16}, + {"int8", MakePlan2VecInt8ConstExprWithType, types.T_array_int8}, + {"uint8", MakePlan2VecUint8ConstExprWithType, types.T_array_uint8}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + e := c.fn("[1,2,3]", 3) + actual := e.Expr.(*plan.Expr_Lit).Lit.GetValue().(*plan.Literal_Sval).Sval + require.Equal(t, "[1,2,3]", actual) + require.Equal(t, int32(c.oid), e.Typ.Id) + require.Equal(t, int32(3), e.Typ.Width) + require.True(t, e.Typ.NotNullable) + }) + } +} + func Test_isSameColumnType(t *testing.T) { require.True(t, isSameColumnType( plan.Type{Id: int32(types.T_varchar), Width: 32}, diff --git a/pkg/sql/plan/query_builder.go b/pkg/sql/plan/query_builder.go index 7089d4b68046e..0e98e74bc2cbb 100644 --- a/pkg/sql/plan/query_builder.go +++ b/pkg/sql/plan/query_builder.go @@ -5833,6 +5833,8 @@ func (builder *QueryBuilder) buildTableFunction(tbl *tree.TableFunction, ctx *Bi nodeId, err = builder.buildFullTextIndexScan(tbl, ctx, exprs, children) case "fulltext_index_tokenize": nodeId, err = builder.buildFullTextIndexTokenize(tbl, ctx, exprs, children) + case "bm25_compact": + nodeId = builder.buildBm25Compact(tbl, ctx, exprs, children) case "stage_list": nodeId, err = builder.buildStageList(tbl, ctx, exprs, children) case "moplugin_table": diff --git a/pkg/sql/plan/rule/constant_fold.go b/pkg/sql/plan/rule/constant_fold.go index 17527cc5e97f4..86229388ac92f 100644 --- a/pkg/sql/plan/rule/constant_fold.go +++ b/pkg/sql/plan/rule/constant_fold.go @@ -440,7 +440,8 @@ func GetConstantValue(vec *vector.Vector, transAll bool, row uint64) *plan.Liter decimalValue.A = int64(vector.MustFixedColNoTypeCheck[types.Decimal128](vec)[row].B0_63) decimalValue.B = int64(vector.MustFixedColNoTypeCheck[types.Decimal128](vec)[row].B64_127) return &plan.Literal{Value: &plan.Literal_Decimal128Val{Decimal128Val: decimalValue}} - case types.T_array_float32, types.T_array_float64: + case types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: data := vec.GetStringAt(int(row)) return &plan.Literal{ Value: &plan.Literal_VecVal{ @@ -577,7 +578,8 @@ func GetConstantValue2(proc *process.Process, expr *plan.Expr, vec *vector.Vecto err = vector.AppendBytes(vec, nil, false, proc.Mp()) return false, err } - case types.T_array_float32, types.T_array_float64: + case types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: if val, ok := cExpr.Lit.Value.(*plan.Literal_VecVal); ok { val := val.VecVal err = vector.AppendBytes(vec, []byte(val), false, proc.Mp()) diff --git a/pkg/sql/plan/rule/constant_fold_narrow_test.go b/pkg/sql/plan/rule/constant_fold_narrow_test.go new file mode 100644 index 0000000000000..d70f534aca557 --- /dev/null +++ b/pkg/sql/plan/rule/constant_fold_narrow_test.go @@ -0,0 +1,55 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rule + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/stretchr/testify/require" +) + +// Narrow vector constants must fold to a VecVal literal (carrying the raw bytes), +// like float32/float64 — else the ivfflat narrow ORDER BY pushdown can't fold the +// query and the const executor nil-panics on materialization. +func TestGetConstantValueNarrowVec(t *testing.T) { + mp := mpool.MustNewZero() + defer mpool.DeleteMPool(mp) + + cases := []struct { + oid types.T + data []byte + }{ + {types.T_array_float32, types.ArrayToBytes([]float32{1, 2, 3})}, + {types.T_array_float64, types.ArrayToBytes([]float64{1, 2, 3})}, + {types.T_array_bf16, types.ArrayToBytes(types.Float32ToBF16Slice([]float32{1, 2, 3}))}, + {types.T_array_float16, types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{1, 2, 3}))}, + {types.T_array_int8, types.ArrayToBytes([]int8{1, 2, 3})}, + {types.T_array_uint8, types.ArrayToBytes([]uint8{1, 2, 3})}, + } + for _, c := range cases { + vec := vector.NewVec(c.oid.ToType()) + require.NoError(t, vector.AppendBytes(vec, c.data, false, mp)) + lit := GetConstantValue(vec, true, 0) + require.NotNilf(t, lit, "%s should fold", c.oid) + vv, ok := lit.Value.(*plan.Literal_VecVal) + require.Truef(t, ok, "%s -> VecVal literal", c.oid) + require.Equalf(t, c.data, []byte(vv.VecVal), "%s bytes preserved", c.oid) + vec.Free(mp) + } +} diff --git a/pkg/sql/plan/utils.go b/pkg/sql/plan/utils.go index 683a524fc55a9..a49c628480d7e 100644 --- a/pkg/sql/plan/utils.go +++ b/pkg/sql/plan/utils.go @@ -383,7 +383,10 @@ func splitAndBindCondition(astExpr tree.Expr, expandAlias ExpandAliasMode, ctx * needCast := true fn := expr.GetF() if fn != nil { - needCast = fn.Func.ObjName != "fulltext_match" + // fulltext_match / bm25_match are rewritten to an index-scan join by the + // optimizer; leave them un-cast so the rewrite can find them by name in the + // filter list (a wrapping cast(... AS BOOL) would hide the function). + needCast = fn.Func.ObjName != "fulltext_match" && fn.Func.ObjName != "bm25_match" } // expr must be bool type, if not, try to do type convert // but just ignore the subQuery. It will be solved at optimizer. diff --git a/pkg/vectorindex/brute_force/brute_force.go b/pkg/vectorindex/brute_force/brute_force.go index 2d29aced6e826..5aacf4a2d8730 100644 --- a/pkg/vectorindex/brute_force/brute_force.go +++ b/pkg/vectorindex/brute_force/brute_force.go @@ -41,7 +41,10 @@ type UsearchBruteForceIndex[T types.RealNumbers] struct { deallocator malloc.Deallocator } -type GoBruteForceIndex[T types.RealNumbers] struct { +// GoBruteForceIndex holds vectors of element type T and computes distances in +// result type R (float32 for f32/narrow inputs, float64 for f64). R only ever +// differs from "float32" for f64 input, so the common path stays float32. +type GoBruteForceIndex[T types.ArrayElement, R types.RealNumbers] struct { Dataset [][]T // flattend vector Metric metric.MetricType Dimension uint @@ -49,7 +52,7 @@ type GoBruteForceIndex[T types.RealNumbers] struct { } var _ cache.VectorIndexSearchIf = &UsearchBruteForceIndex[float32]{} -var _ cache.VectorIndexSearchIf = &GoBruteForceIndex[float32]{} +var _ cache.VectorIndexSearchIf = &GoBruteForceIndex[float32, float32]{} func GetUsearchQuantizationFromType(v any) (usearch.Quantization, error) { switch v.(type) { @@ -62,25 +65,57 @@ func GetUsearchQuantizationFromType(v any) (usearch.Quantization, error) { } } -func NewCpuBruteForceIndex[T types.RealNumbers](dataset [][]T, +// NewCpuBruteForceIndex builds a pure-Go brute-force index for any ArrayElement. +// It dispatches by concrete element type and picks the distance result type R: +// float64 only for float64 input, float32 for everything else (f32 + the narrow +// quantizations bf16/f16/int8/uint8 — whose kernels the resolver casts to float32). +func NewCpuBruteForceIndex[T types.ArrayElement](dataset [][]T, dimension uint, m metric.MetricType, elemsz uint) (cache.VectorIndexSearchIf, error) { - return NewGoBruteForceIndex(dataset, dimension, m, elemsz) + // R = element type for f32/f64; float32 for the narrow quantizations. + switch ds := any(dataset).(type) { + case [][]float32: + return newGoBruteForce[float32, float32](ds, dimension, m), nil + case [][]float64: + return newGoBruteForce[float64, float64](ds, dimension, m), nil + case [][]types.BF16: + return newGoBruteForce[types.BF16, float32](ds, dimension, m), nil + case [][]types.Float16: + return newGoBruteForce[types.Float16, float32](ds, dimension, m), nil + case [][]int8: + return newGoBruteForce[int8, float32](ds, dimension, m), nil + case [][]uint8: + return newGoBruteForce[uint8, float32](ds, dimension, m), nil + default: + return nil, moerr.NewInternalErrorNoCtx(fmt.Sprintf("brute force: unsupported element type %T", *new(T))) + } +} + +// newGoBruteForce constructs a GoBruteForceIndex with explicit element type T and +// distance result type R. The single constructor for both the public f32/f64 +// entry point and the narrow (R=float32) dispatch. +func newGoBruteForce[T types.ArrayElement, R types.RealNumbers](dataset [][]T, + dimension uint, + m metric.MetricType) cache.VectorIndexSearchIf { + + return &GoBruteForceIndex[T, R]{ + Dataset: dataset, + Metric: m, + Dimension: dimension, + Count: uint(len(dataset)), + } } +// NewGoBruteForceIndex builds an f32/f64 index whose result type equals the +// element type (R=T). Kept as the public one-type-param entry point. func NewGoBruteForceIndex[T types.RealNumbers](dataset [][]T, dimension uint, m metric.MetricType, elemsz uint) (cache.VectorIndexSearchIf, error) { - idx := &GoBruteForceIndex[T]{} - idx.Metric = m - idx.Dimension = dimension - idx.Count = uint(len(dataset)) - idx.Dataset = dataset - return idx, nil + return newGoBruteForce[T, T](dataset, dimension, m), nil } func NewUsearchBruteForceIndex[T types.RealNumbers](dataset [][]T, @@ -278,26 +313,26 @@ func (idx *UsearchBruteForceIndex[T]) Destroy() { } } -func (idx *GoBruteForceIndex[T]) Load(sqlproc *sqlexec.SqlProcess) error { +func (idx *GoBruteForceIndex[T, R]) Load(sqlproc *sqlexec.SqlProcess) error { return nil } -func (idx *GoBruteForceIndex[T]) UpdateConfig(sif cache.VectorIndexSearchIf) error { +func (idx *GoBruteForceIndex[T, R]) UpdateConfig(sif cache.VectorIndexSearchIf) error { return nil } -func (idx *GoBruteForceIndex[T]) Destroy() { +func (idx *GoBruteForceIndex[T, R]) Destroy() { } // SearchFloat32 implements VectorIndexSearchIf — writes results directly into caller-provided // slices, eliminating the intermediate []int64 and []float64 heap allocations of Search. -func (idx *GoBruteForceIndex[T]) SearchFloat32(proc *sqlexec.SqlProcess, _queries any, rt vectorindex.RuntimeConfig, outKeys []int64, outDists []float32) error { +func (idx *GoBruteForceIndex[T, R]) SearchFloat32(proc *sqlexec.SqlProcess, _queries any, rt vectorindex.RuntimeConfig, outKeys []int64, outDists []float32) error { queries, ok := _queries.([][]T) if !ok { return moerr.NewInternalErrorNoCtx("queries type invalid") } - distfn, err := metric.ResolveDistanceFn[T](idx.Metric) + distfn, err := metric.ResolveDistanceFn[T, R](idx.Metric) if err != nil { return err } @@ -316,10 +351,10 @@ func (idx *GoBruteForceIndex[T]) SearchFloat32(proc *sqlexec.SqlProcess, _querie nqueries, func(ctx context.Context, thread_id int, start, end int) error { var heapKeysBuf []int64 - var heapDistBuf []T + var heapDistBuf []R if limit > 1 { heapKeysBuf = make([]int64, limit) - heapDistBuf = make([]T, limit) + heapDistBuf = make([]R, limit) } for k := start; k < end; k++ { @@ -329,7 +364,7 @@ func (idx *GoBruteForceIndex[T]) SearchFloat32(proc *sqlexec.SqlProcess, _querie } if limit == 1 { - minDist := metric.MaxFloat[T]() + minDist := metric.MaxFloat[R]() minIdx := -1 for j := range idx.Dataset { dist, err2 := distfn(q, idx.Dataset[j]) @@ -346,7 +381,7 @@ func (idx *GoBruteForceIndex[T]) SearchFloat32(proc *sqlexec.SqlProcess, _querie continue } - h := vectorindex.NewFastMaxHeap[T, int64](limit, heapKeysBuf, heapDistBuf) + h := vectorindex.NewFastMaxHeap[R, int64](limit, heapKeysBuf, heapDistBuf) for j := range idx.Dataset { dist, err2 := distfn(q, idx.Dataset[j]) if err2 != nil { @@ -371,13 +406,13 @@ func (idx *GoBruteForceIndex[T]) SearchFloat32(proc *sqlexec.SqlProcess, _querie }) } -func (idx *GoBruteForceIndex[T]) Search(proc *sqlexec.SqlProcess, _queries any, rt vectorindex.RuntimeConfig) (keys any, distances []float64, err error) { +func (idx *GoBruteForceIndex[T, R]) Search(proc *sqlexec.SqlProcess, _queries any, rt vectorindex.RuntimeConfig) (keys any, distances []float64, err error) { queries, ok := _queries.([][]T) if !ok { return nil, nil, moerr.NewInternalErrorNoCtx("queries type invalid") } - distfn, err := metric.ResolveDistanceFn[T](idx.Metric) + distfn, err := metric.ResolveDistanceFn[T, R](idx.Metric) if err != nil { return nil, nil, err } @@ -401,10 +436,10 @@ func (idx *GoBruteForceIndex[T]) Search(proc *sqlexec.SqlProcess, _queries any, func(ctx context.Context, thread_id int, start, end int) (err2 error) { // Pre-allocate heap buffers for this thread var heapKeysBuf []int64 - var heapDistBuf []T + var heapDistBuf []R if limit > 1 { heapKeysBuf = make([]int64, limit) - heapDistBuf = make([]T, limit) + heapDistBuf = make([]R, limit) } for k := start; k < end; k++ { @@ -414,7 +449,7 @@ func (idx *GoBruteForceIndex[T]) Search(proc *sqlexec.SqlProcess, _queries any, } if limit == 1 { - minDist := metric.MaxFloat[T]() + minDist := metric.MaxFloat[R]() minIdx := -1 for j := range idx.Dataset { dist, err2 := distfn(q, idx.Dataset[j]) @@ -432,7 +467,7 @@ func (idx *GoBruteForceIndex[T]) Search(proc *sqlexec.SqlProcess, _queries any, } // Max-heap logic for K > 1 - h := vectorindex.NewFastMaxHeap[T, int64](limit, heapKeysBuf, heapDistBuf) + h := vectorindex.NewFastMaxHeap[R, int64](limit, heapKeysBuf, heapDistBuf) for j := range idx.Dataset { dist, err2 := distfn(q, idx.Dataset[j]) diff --git a/pkg/vectorindex/brute_force/brute_force_narrow_test.go b/pkg/vectorindex/brute_force/brute_force_narrow_test.go new file mode 100644 index 0000000000000..8a057a91b6580 --- /dev/null +++ b/pkg/vectorindex/brute_force/brute_force_narrow_test.go @@ -0,0 +1,96 @@ +//go:build !gpu + +// Copyright 2022 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package brute_force + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/testutil" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + "github.com/matrixorigin/matrixone/pkg/vectorindex/metric" + "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" + "github.com/stretchr/testify/require" +) + +// TestNewCpuBruteForceIndexNarrow covers the bf16 / f16 / int8 / uint8 dispatch +// arms of NewCpuBruteForceIndex. The f32/f64 arms are covered by the existing +// tests; the default arm is unreachable because ArrayElement is exactly these +// six element types. Queries are typed [][]T to match GoBruteForceIndex.Search. +func TestNewCpuBruteForceIndexNarrow(t *testing.T) { + m := mpool.MustNewZero() + proc := testutil.NewProcessWithMPool(t, "", m) + sqlproc := sqlexec.NewSqlProcess(proc) + rt := vectorindex.RuntimeConfig{Limit: 2, NThreads: 1} + const dim = uint(3) + + bf := func(vs ...float32) []types.BF16 { + out := make([]types.BF16, len(vs)) + for i, v := range vs { + out[i] = types.BF16FromFloat32(v) + } + return out + } + f16 := func(vs ...float32) []types.Float16 { + out := make([]types.Float16, len(vs)) + for i, v := range vs { + out[i] = types.Float16FromFloat32(v) + } + return out + } + + t.Run("bf16", func(t *testing.T) { + ds := [][]types.BF16{bf(1, 2, 3), bf(3, 4, 5)} + idx, err := NewCpuBruteForceIndex[types.BF16](ds, dim, metric.Metric_L2sqDistance, 2) + require.NoError(t, err) + keys, dists, err := idx.Search(sqlproc, ds, rt) + require.NoError(t, err) + require.NotNil(t, keys) + require.Len(t, dists, 4) + }) + + t.Run("f16", func(t *testing.T) { + ds := [][]types.Float16{f16(1, 2, 3), f16(3, 4, 5)} + idx, err := NewCpuBruteForceIndex[types.Float16](ds, dim, metric.Metric_L2sqDistance, 2) + require.NoError(t, err) + keys, dists, err := idx.Search(sqlproc, ds, rt) + require.NoError(t, err) + require.NotNil(t, keys) + require.Len(t, dists, 4) + }) + + t.Run("int8", func(t *testing.T) { + ds := [][]int8{{1, 2, 3}, {3, 4, 5}} + idx, err := NewCpuBruteForceIndex[int8](ds, dim, metric.Metric_L2sqDistance, 1) + require.NoError(t, err) + keys, dists, err := idx.Search(sqlproc, ds, rt) + require.NoError(t, err) + require.NotNil(t, keys) + require.Len(t, dists, 4) + }) + + t.Run("uint8", func(t *testing.T) { + ds := [][]uint8{{1, 2, 3}, {3, 4, 5}} + idx, err := NewCpuBruteForceIndex[uint8](ds, dim, metric.Metric_L2sqDistance, 1) + require.NoError(t, err) + keys, dists, err := idx.Search(sqlproc, ds, rt) + require.NoError(t, err) + require.NotNil(t, keys) + require.Len(t, dists, 4) + }) +} diff --git a/pkg/vectorindex/brute_force/brute_force_test.go b/pkg/vectorindex/brute_force/brute_force_test.go index a3573f24a0e15..07fff159ed735 100644 --- a/pkg/vectorindex/brute_force/brute_force_test.go +++ b/pkg/vectorindex/brute_force/brute_force_test.go @@ -383,7 +383,7 @@ func TestGoBruteForceLifecycle(t *testing.T) { idx, err := NewGoBruteForceIndex[float32](dataset, 3, metric.Metric_L2sqDistance, 4) require.NoError(t, err) - bf := idx.(*GoBruteForceIndex[float32]) + bf := idx.(*GoBruteForceIndex[float32, float32]) require.NoError(t, bf.Load(nil)) require.NoError(t, bf.UpdateConfig(nil)) bf.Destroy() diff --git a/pkg/vectorindex/brute_force/cpu.go b/pkg/vectorindex/brute_force/cpu.go index c14bb7756765c..a52af753c0576 100644 --- a/pkg/vectorindex/brute_force/cpu.go +++ b/pkg/vectorindex/brute_force/cpu.go @@ -25,7 +25,7 @@ import ( // gpuMode is accepted-but-ignored in non-gpu builds — CPU is the only // option here. The signature matches the gpu.go version so callers // pass the flag uniformly regardless of build tag. -func NewBruteForceIndex[T types.RealNumbers](dataset [][]T, +func NewBruteForceIndex[T types.ArrayElement](dataset [][]T, dimension uint, m metric.MetricType, elemsz uint, diff --git a/pkg/vectorindex/brute_force/gpu.go b/pkg/vectorindex/brute_force/gpu.go index 160a738fb9085..65b80f2c8b17d 100644 --- a/pkg/vectorindex/brute_force/gpu.go +++ b/pkg/vectorindex/brute_force/gpu.go @@ -65,8 +65,8 @@ func NewAdhocBruteForceIndex[T types.RealNumbers](dataset [][]T, switch dset := any(dataset).(type) { case [][]float32: return NewGpuAdhocBruteForceIndex[float32](dset, dimension, m, elemsz) - case [][]uint16: - // Convert [][]uint16 to [][]cuvs.Float16 to pass to NewGpuAdhocBruteForceIndex + case [][]types.Float16: + // types.Float16 (NOT a bare uint16, which could also be BF16) -> cuvs.Float16. f16dset := make([][]cuvs.Float16, len(dset)) for i, v := range dset { f16dset[i] = util.UnsafeSliceCast[cuvs.Float16](v) @@ -224,7 +224,7 @@ func (idx *GpuAdhocBruteForceIndex[T]) Destroy() { } type GpuBruteForceIndex[T cuvs.VectorType] struct { - index *cuvs.GpuBruteForce[T] + index *cuvs.GpuBruteForce[T, T] dimension uint count uint } @@ -248,7 +248,7 @@ func resolveCuvsDistance(m metric.MetricType) cuvs.DistanceType { } } -func NewBruteForceIndex[T types.RealNumbers](dataset [][]T, +func NewBruteForceIndex[T types.ArrayElement](dataset [][]T, dimension uint, m metric.MetricType, elemsz uint, @@ -261,20 +261,20 @@ func NewBruteForceIndex[T types.RealNumbers](dataset [][]T, return NewCpuBruteForceIndex[T](dataset, dimension, m, elemsz) } + // cuVS brute force supports float32 and Float16 only. Switch on the distinct + // Go named type so types.BF16 (also uint16-backed) is never mistaken for f16. switch dset := any(dataset).(type) { - case [][]float64: - return NewCpuBruteForceIndex[T](dataset, dimension, m, elemsz) case [][]float32: return NewGpuBruteForceIndex[float32](dset, dimension, m, elemsz, nthread) - case [][]uint16: - // Convert [][]uint16 to [][]cuvs.Float16 to pass to NewGpuBruteForceIndex + case [][]types.Float16: f16dset := make([][]cuvs.Float16, len(dset)) for i, v := range dset { f16dset[i] = util.UnsafeSliceCast[cuvs.Float16](v) } return NewGpuBruteForceIndex[cuvs.Float16](f16dset, dimension, m, elemsz, nthread) default: - return nil, moerr.NewInternalErrorNoCtx("type not supported for BruteForceIndex") + // float64, bf16, int8, uint8 -> pure-Go CPU brute force. + return NewCpuBruteForceIndex[T](dataset, dimension, m, elemsz) } } @@ -320,7 +320,7 @@ func NewGpuBruteForceIndex[T cuvs.VectorType](dataset [][]T, } deviceID := cuvs.GetNextGpuDeviceId() - km, err := cuvs.NewGpuBruteForce[T](flattened, uint64(len(dataset)), uint32(dimension), resolveCuvsDistance(m), uint32(nthread), deviceID) + km, err := cuvs.NewGpuBruteForce[T, T](flattened, uint64(len(dataset)), uint32(dimension), resolveCuvsDistance(m), uint32(nthread), deviceID) if err != nil { return nil, err } diff --git a/pkg/vectorindex/cagra/build_gpu.go b/pkg/vectorindex/cagra/build_gpu.go index 6c3b4be74cc7e..9109d441eee34 100644 --- a/pkg/vectorindex/cagra/build_gpu.go +++ b/pkg/vectorindex/cagra/build_gpu.go @@ -23,6 +23,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/sqlquote" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/cuvs" "github.com/matrixorigin/matrixone/pkg/vectorindex" ) @@ -31,17 +32,28 @@ import ( // When the current sub-index reaches IndexCapacity, it is finalized (Build called) and a // new sub-index is created, mirroring the HnswBuild pattern. // +// CagraBuild carries two element types: base/quantizer-source B (the decoded +// source column type — f32 or f16) and storage Q (the cuVS sub-index storage +// type). For a direct index B==Q; for a quantized index (e.g. vecf16 base -> +// int8 storage) B is the base type and Q the 1-byte storage type. +// // CagraBuild is single-threaded; the cagra_create table function runs with IsSingle=true. -type CagraBuild[T cuvs.VectorType] struct { +type CagraBuild[B, Q cuvs.VectorType] struct { uid string idxcfg vectorindex.IndexConfig tblcfg vectorindex.IndexTableConfig - indexes []*CagraModel[T] // completed sub-indexes (Build already called) - current *CagraModel[T] // sub-index currently being filled + indexes []*CagraModel[B, Q] // completed sub-indexes (Build already called) + current *CagraModel[B, Q] // sub-index currently being filled nthread uint32 devices []int count int64 // vectors in current sub-index - idBuf [1]int64 // reusable buffer for AddFloat to avoid per-call heap allocation + idBuf [1]int64 // reusable buffer for AddRow to avoid per-call heap allocation + + // (B, Q) routing tags computed once at construction. bIsHalf: the base + // type is f16. qIsHalf: the storage type is f16 (so a half base goes + // native rather than quantized). + bIsHalf bool + qIsHalf bool // Filter column metadata (INCLUDE columns). Stashed once via SetFilterColumns // and re-applied to every new sub-index allocated by getOrCreateCurrent, so @@ -49,31 +61,33 @@ type CagraBuild[T cuvs.VectorType] struct { filterColMetaJSON string } -// NewCagraBuild creates a new CagraBuild ready for AddFloat calls. -func NewCagraBuild[T cuvs.VectorType]( +// NewCagraBuild creates a new CagraBuild ready for AddRow calls. +func NewCagraBuild[B, Q cuvs.VectorType]( uid string, idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig, nthread uint32, devices []int, -) (*CagraBuild[T], error) { - return &CagraBuild[T]{ +) (*CagraBuild[B, Q], error) { + return &CagraBuild[B, Q]{ uid: uid, idxcfg: idxcfg, tblcfg: tblcfg, - indexes: make([]*CagraModel[T], 0, 4), + indexes: make([]*CagraModel[B, Q], 0, 4), nthread: nthread, devices: devices, + bIsHalf: cuvs.GetQuantization[B]() == cuvs.F16, + qIsHalf: cuvs.GetQuantization[Q]() == cuvs.F16, }, nil } -func (b *CagraBuild[T]) createKey(n int) string { +func (b *CagraBuild[B, Q]) createKey(n int) string { return fmt.Sprintf("%s:%d", b.uid, n) } // getOrCreateCurrent returns the current sub-index, creating a new one if needed. // When the current sub-index is full it is finalized (Build called) and a new one is started. -func (b *CagraBuild[T]) getOrCreateCurrent() (*CagraModel[T], error) { +func (b *CagraBuild[B, Q]) getOrCreateCurrent() (*CagraModel[B, Q], error) { capacity := b.idxcfg.IndexCapacity if b.current != nil && b.count >= capacity { @@ -88,7 +102,7 @@ func (b *CagraBuild[T]) getOrCreateCurrent() (*CagraModel[T], error) { if b.current == nil { key := b.createKey(len(b.indexes)) - m, err := NewCagraModelForBuild[T](key, b.idxcfg, b.nthread, b.devices) + m, err := NewCagraModelForBuild[B, Q](key, b.idxcfg, b.nthread, b.devices) if err != nil { return nil, err } @@ -111,33 +125,46 @@ func (b *CagraBuild[T]) getOrCreateCurrent() (*CagraModel[T], error) { // SetFilterColumns registers pre-filter (INCLUDE column) metadata. The JSON // is re-applied to each new sub-index allocated during the build. Must be -// called before the first AddFloat. -func (b *CagraBuild[T]) SetFilterColumns(colMetaJSON string) { +// called before the first AddRow. +func (b *CagraBuild[B, Q]) SetFilterColumns(colMetaJSON string) { b.filterColMetaJSON = colMetaJSON } // AddFilterChunk appends nrows raw filter-column bytes to the *current* // sub-index being filled. Call once per filter column per row batch, in the -// same cadence as AddFloat (which drives sub-index rotation). +// same cadence as AddRow (which drives sub-index rotation). // nullBitmap is a packed []uint32 (LSB-first, bit i = 1 means row i IS NULL) // of ceil(nrows/32) entries, or nil when the chunk has no nulls. -func (b *CagraBuild[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { +func (b *CagraBuild[B, Q]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { if b.current == nil { - return moerr.NewInternalErrorNoCtx("CagraBuild.AddFilterChunk: no current sub-index (call AddFloat first)") + return moerr.NewInternalErrorNoCtx("CagraBuild.AddFilterChunk: no current sub-index (call AddRow first)") } return b.current.Index.AddFilterChunk(colIdx, data, nullBitmap, nrows) } -// AddFloat appends one float32 vector with the given int64 id. -// The internal quantization (T) is handled by AddChunkFloat. +// AddRow buffers one source row. vecBytes is the raw little-endian base-type +// bytes of one vector (4*dim for an f32 base, 2*dim for an f16 base) — the +// non-generic cagraBuilder interface can't name the concrete element type B, so +// the bytes are reinterpreted here with UnsafeSliceCast (zero-copy, no per-row +// heap alloc). Routing by (B, Q): +// - f16 base, f16 storage (direct, Q==B): native AddChunk([]Q). +// - otherwise (f32 base, or f16 base -> int8/uint8): AddChunkQuantize([]B), +// which converts B -> Q on device (B==Q copy, or learned/cast quantizer). +// // idBuf is reused across calls to avoid a per-call heap allocation. -func (b *CagraBuild[T]) AddFloat(id int64, vec []float32) error { +func (b *CagraBuild[B, Q]) AddRow(id int64, vecBytes []byte) error { idx, err := b.getOrCreateCurrent() if err != nil { return err } b.idBuf[0] = id - if err = idx.AddChunkFloat(vec, 1, b.idBuf[:]); err != nil { + + if b.bIsHalf && b.qIsHalf { + err = idx.AddChunk(util.UnsafeSliceCast[Q](vecBytes), 1, b.idBuf[:]) + } else { + err = idx.AddChunkQuantize(util.UnsafeSliceCast[B](vecBytes), 1, b.idBuf[:]) + } + if err != nil { return err } b.count++ @@ -146,7 +173,7 @@ func (b *CagraBuild[T]) AddFloat(id int64, vec []float32) error { // ToInsertSql finalizes any in-progress sub-index, serializes all sub-indexes to the // storage table, and returns INSERT SQL statements (storage chunks + single metadata row). -func (b *CagraBuild[T]) ToInsertSql(ts int64) ([]string, error) { +func (b *CagraBuild[B, Q]) ToInsertSql(ts int64) ([]string, error) { // Finalize the current sub-index if it contains vectors. if b.current != nil && b.count > 0 { if err := b.current.Build(); err != nil { @@ -181,7 +208,7 @@ func (b *CagraBuild[T]) ToInsertSql(ts int64) ([]string, error) { } // Destroy frees all GPU memory and removes any temporary files. -func (b *CagraBuild[T]) Destroy() error { +func (b *CagraBuild[B, Q]) Destroy() error { var errs error if b.current != nil { if err := b.current.Destroy(); err != nil { @@ -199,6 +226,6 @@ func (b *CagraBuild[T]) Destroy() error { } // GetIndexes returns the completed sub-indexes (for testing). -func (b *CagraBuild[T]) GetIndexes() []*CagraModel[T] { +func (b *CagraBuild[B, Q]) GetIndexes() []*CagraModel[B, Q] { return b.indexes } diff --git a/pkg/vectorindex/cagra/cdc_load_test.go b/pkg/vectorindex/cagra/cdc_load_test.go index bf9627e221a29..ce49d305b9843 100644 --- a/pkg/vectorindex/cagra/cdc_load_test.go +++ b/pkg/vectorindex/cagra/cdc_load_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" @@ -66,7 +67,11 @@ func encodeChunk(t *testing.T, dim, includeBytesPerRow int, ops []cuvscdc.CdcOp, } insIdx++ } - out, err := cuvscdc.EncodeEventRecord(buf, op, pkids[i], v, inc, dim, includeBytesPerRow) + var vb []byte + if v != nil { + vb = util.UnsafeSliceToBytes(v) + } + out, err := cuvscdc.EncodeEventRecord(buf, op, pkids[i], vb, inc, 4*dim, includeBytesPerRow) require.NoError(t, err) buf = out } @@ -95,7 +100,7 @@ func TestLoadCdcEventsFromDB_RoundTrip(t *testing.T) { } defer func() { runSql = orig }() - idx := &CagraModel[float32]{Id: "idx-1"} + idx := &CagraModel[float32, float32]{Id: "idx-1"} got, err := idx.loadCdcEventsFromDB(sqlproc, tblcfg) require.NoError(t, err) require.Len(t, got, 1) @@ -115,7 +120,7 @@ func TestLoadCdcEventsFromDB_Empty(t *testing.T) { } defer func() { runSql = orig }() - idx := &CagraModel[float32]{Id: "idx-1"} + idx := &CagraModel[float32, float32]{Id: "idx-1"} got, err := idx.loadCdcEventsFromDB(sqlproc, testTblcfg()) require.NoError(t, err) require.Empty(t, got) @@ -134,7 +139,7 @@ func TestReplayEventChunks_DeleteInsertDelete(t *testing.T) { ) chunks := []cuvscdc.EventChunk{{ChunkId: 0, Data: chunkBytes}} - delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks(chunks, dim, 0) + delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks[float32](chunks, dim, 0) require.NoError(t, err) require.Equal(t, []int64{1}, delPkids) require.Empty(t, ovPkids) @@ -154,7 +159,7 @@ func TestReplayEventChunks_FlattenOverflow(t *testing.T) { ) chunks := []cuvscdc.EventChunk{{ChunkId: 0, Data: chunkBytes}} - delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks(chunks, dim, 0) + delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks[float32](chunks, dim, 0) require.NoError(t, err) require.Empty(t, delPkids) require.Equal(t, []int64{10, 20}, ovPkids) @@ -178,7 +183,7 @@ func TestReplayEventChunks_MultiChunkOrder(t *testing.T) { {ChunkId: 1, Data: chunk1}, {ChunkId: 0, Data: chunk0}, } - delPkids, ovPkids, _, _, err := replayEventChunks(chunks, dim, 0) + delPkids, ovPkids, _, _, err := replayEventChunks[float32](chunks, dim, 0) require.NoError(t, err) require.Equal(t, []int64{5}, delPkids, "INSERT@chunk0 then DELETE@chunk1 → deleted={5}") @@ -254,7 +259,7 @@ func TestLoadIndex_WithCdcDeltas(t *testing.T) { } defer func() { runSql = origRunSql }() - models, err := LoadMetadata[float32](sqlproc, tblcfg.DbName, tblcfg.MetadataTable) + models, err := LoadMetadata[float32, float32](sqlproc, tblcfg.DbName, tblcfg.MetadataTable) require.NoError(t, err) require.Equal(t, 1, len(models)) diff --git a/pkg/vectorindex/cagra/model_gpu.go b/pkg/vectorindex/cagra/model_gpu.go index e8685b889eb85..cdde2343650bd 100644 --- a/pkg/vectorindex/cagra/model_gpu.go +++ b/pkg/vectorindex/cagra/model_gpu.go @@ -29,6 +29,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/sqlquote" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/cuvs" "github.com/matrixorigin/matrixone/pkg/logutil" @@ -45,9 +46,9 @@ var runSql_streaming = sqlexec.RunStreamingSql // CagraModel wraps a GpuCagra index and handles load/save to the secondary index tables. // The serialized form is a tar file produced by cuvs.Pack / cuvs.Unpack. // T must satisfy cuvs.VectorType (float32 | Float16 | int8 | uint8). -type CagraModel[T cuvs.VectorType] struct { +type CagraModel[B, Q cuvs.VectorType] struct { Id string - Index *cuvs.GpuCagra[T] + Index *cuvs.GpuCagra[B, Q] Path string // local tar file path; empty when index is in GPU memory only FileSize int64 MaxCapacity uint64 @@ -77,7 +78,7 @@ type CagraModel[T cuvs.VectorType] struct { // (quantizer params live in the model tar, not available at CDC write // time). OverflowPkids []int64 - OverflowVecs []float32 // len = len(OverflowPkids) * dim + OverflowVecs []B // len = len(OverflowPkids) * dim (native base type B) // INCLUDE column data carried alongside each overflow row. Layout // matches the EncodeEventRecord INSERT-record include section: @@ -98,8 +99,8 @@ type CagraModel[T cuvs.VectorType] struct { // NewCagraModelForBuild creates a CagraModel ready for bulk-build. // Call InitEmpty once the total vector count is known, then AddChunk, then Build. -func NewCagraModelForBuild[T cuvs.VectorType](id string, cfg vectorindex.IndexConfig, nthread uint32, devices []int) (*CagraModel[T], error) { - return &CagraModel[T]{ +func NewCagraModelForBuild[B, Q cuvs.VectorType](id string, cfg vectorindex.IndexConfig, nthread uint32, devices []int) (*CagraModel[B, Q], error) { + return &CagraModel[B, Q]{ Id: id, Idxcfg: cfg, NThread: nthread, @@ -108,7 +109,7 @@ func NewCagraModelForBuild[T cuvs.VectorType](id string, cfg vectorindex.IndexCo } // cagraConfig returns the cuvs types derived from idx.Idxcfg. -func (idx *CagraModel[T]) cagraConfig() (cuvsMetric cuvs.DistanceType, bp cuvs.CagraBuildParams, mode cuvs.DistributionMode, err error) { +func (idx *CagraModel[B, Q]) cagraConfig() (cuvsMetric cuvs.DistanceType, bp cuvs.CagraBuildParams, mode cuvs.DistributionMode, err error) { cfg := idx.Idxcfg.CuvsCagra var ok bool cuvsMetric, ok = metric.MetricTypeToCuvsMetric[metric.MetricType(cfg.Metric)] @@ -129,7 +130,7 @@ func (idx *CagraModel[T]) cagraConfig() (cuvsMetric cuvs.DistanceType, bp cuvs.C // InitEmpty allocates the GPU buffer for totalCount vectors. // Must be called after NewCagraModelForBuild and before any AddChunk call. -func (idx *CagraModel[T]) InitEmpty(totalCount uint64) error { +func (idx *CagraModel[B, Q]) InitEmpty(totalCount uint64) error { if idx.Index != nil { return moerr.NewInternalErrorNoCtx("CagraModel: index already initialized") } @@ -137,7 +138,7 @@ func (idx *CagraModel[T]) InitEmpty(totalCount uint64) error { if err != nil { return err } - gi, err := cuvs.NewGpuCagraEmpty[T]( + gi, err := cuvs.NewGpuCagraEmpty[B, Q]( totalCount, uint32(idx.Idxcfg.CuvsCagra.Dimensions), cuvsMetric, @@ -159,7 +160,7 @@ func (idx *CagraModel[T]) InitEmpty(totalCount uint64) error { } // AddChunk appends a chunk of typed vectors to the pre-allocated GPU buffer. -func (idx *CagraModel[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) error { +func (idx *CagraModel[B, Q]) AddChunk(chunk []Q, chunkCount uint64, ids []int64) error { if idx.Index == nil { return moerr.NewInternalErrorNoCtx("CagraModel: index not initialized; call InitEmpty first") } @@ -175,17 +176,14 @@ func (idx *CagraModel[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) er return nil } -// AddChunkFloat appends a chunk of float32 vectors, quantizing on the fly when T is a 1-byte type. -func (idx *CagraModel[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []int64) error { +// AddChunkQuantize appends a chunk of base-typed (B) vectors, quantizing +// natively to the 1-byte storage type Q (int8/uint8). Used for a vecf16 base +// with QUANTIZATION=int8/uint8 — no f32 detour. +func (idx *CagraModel[B, Q]) AddChunkQuantize(chunk []B, chunkCount uint64, ids []int64) error { if idx.Index == nil { return moerr.NewInternalErrorNoCtx("CagraModel: index not initialized; call InitEmpty first") } - /* - if len(ids) > 0 { - logutil.Infof("[DEBUG] CagraModel.AddChunkFloat: chunkCount=%d, first_id=%d, last_id=%d", chunkCount, ids[0], ids[len(ids)-1]) - } - */ - if err := idx.Index.AddChunkFloat(chunk, chunkCount, ids); err != nil { + if err := idx.Index.AddChunkQuantize(chunk, chunkCount, ids); err != nil { return err } idx.Len += int64(chunkCount) @@ -193,7 +191,7 @@ func (idx *CagraModel[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids } // Build constructs the CAGRA graph from the loaded vectors and starts the worker pool. -func (idx *CagraModel[T]) Build() error { +func (idx *CagraModel[B, Q]) Build() error { if idx.Index == nil { return moerr.NewInternalErrorNoCtx("CagraModel: index not initialized") } @@ -205,7 +203,7 @@ func (idx *CagraModel[T]) Build() error { } // Destroy frees GPU memory and removes the local tar file if present. -func (idx *CagraModel[T]) Destroy() error { +func (idx *CagraModel[B, Q]) Destroy() error { if idx.Index != nil { if err := idx.Index.Destroy(); err != nil { return err @@ -224,7 +222,7 @@ func (idx *CagraModel[T]) Destroy() error { // saveToFile serializes the CAGRA index to a local tar file and updates idx.Path / idx.Checksum. // If the index is clean (not dirty) or nil, it is a no-op. // On success the GPU memory is freed and idx.Index is set to nil. -func (idx *CagraModel[T]) saveToFile() error { +func (idx *CagraModel[B, Q]) saveToFile() error { if idx.Index == nil { return nil } @@ -280,7 +278,7 @@ func (idx *CagraModel[T]) saveToFile() error { // ToSql generates INSERT SQL statements to store the model in the secondary index storage table. // Mirrors HnswModel.ToSql — callers are responsible for generating the metadata INSERT. -func (idx *CagraModel[T]) ToSql(cfg vectorindex.IndexTableConfig) ([]string, error) { +func (idx *CagraModel[B, Q]) ToSql(cfg vectorindex.IndexTableConfig) ([]string, error) { if err := idx.saveToFile(); err != nil { return nil, err } @@ -330,7 +328,7 @@ func (idx *CagraModel[T]) ToSql(cfg vectorindex.IndexTableConfig) ([]string, err } // ToDeleteSql generates DELETE SQL for both the storage and metadata tables. -func (idx *CagraModel[T]) ToDeleteSql(cfg vectorindex.IndexTableConfig) ([]string, error) { +func (idx *CagraModel[B, Q]) ToDeleteSql(cfg vectorindex.IndexTableConfig) ([]string, error) { sqls := make([]string, 0, 2) sqls = append(sqls, fmt.Sprintf("DELETE FROM %s WHERE %s = %s", sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), catalog.Cagra_TblCol_Storage_Index_Id, sqlquote.String(idx.Id))) @@ -340,17 +338,17 @@ func (idx *CagraModel[T]) ToDeleteSql(cfg vectorindex.IndexTableConfig) ([]strin } // Empty returns true when no vectors have been added. -func (idx *CagraModel[T]) Empty() bool { +func (idx *CagraModel[B, Q]) Empty() bool { return idx.Len == 0 } // Full returns true when the index has reached its maximum capacity. -func (idx *CagraModel[T]) Full() bool { +func (idx *CagraModel[B, Q]) Full() bool { return idx.MaxCapacity > 0 && uint64(idx.Len) >= idx.MaxCapacity } // Search performs a KNN search and returns external PKs with distances. -func (idx *CagraModel[T]) Search(query []T, limit uint32) (keys []int64, distances []float32, err error) { +func (idx *CagraModel[B, Q]) Search(query []Q, limit uint32) (keys []int64, distances []float32, err error) { if idx.Index == nil { return nil, nil, moerr.NewInternalErrorNoCtx("CagraModel: index not loaded") } @@ -366,7 +364,7 @@ func (idx *CagraModel[T]) Search(query []T, limit uint32) (keys []int64, distanc } // loadChunk reads one streaming result batch and writes each chunk at the correct file offset. -func (idx *CagraModel[T]) loadChunk(ctx context.Context, +func (idx *CagraModel[B, Q]) loadChunk(ctx context.Context, sqlproc *sqlexec.SqlProcess, stream_chan chan executor.Result, error_chan chan error, @@ -414,7 +412,7 @@ func (idx *CagraModel[T]) loadChunk(ctx context.Context, // - tag=0: model tar chunks (streaming, multi-GB) // - tag=1: CDC event log (small KB–MB; replayed once after Unpack to derive // the deleted-pkid set and the brute-force overflow) -func (idx *CagraModel[T]) LoadIndex( +func (idx *CagraModel[B, Q]) LoadIndex( sqlproc *sqlexec.SqlProcess, idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig, @@ -550,7 +548,7 @@ func (idx *CagraModel[T]) LoadIndex( return err } - gi, err := cuvs.NewGpuCagraEmpty[T]( + gi, err := cuvs.NewGpuCagraEmpty[B, Q]( uint64(idxcfg.IndexCapacity), uint32(idxcfg.CuvsCagra.Dimensions), cuvsMetric, @@ -587,7 +585,7 @@ func (idx *CagraModel[T]) LoadIndex( } includeBytesPerRow = ibpr } - delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks(eventChunks, dim, includeBytesPerRow) + delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks[B](eventChunks, dim, includeBytesPerRow) if err != nil { gi.Destroy() return err @@ -627,7 +625,7 @@ func (idx *CagraModel[T]) LoadIndex( } // Unload persists dirty state to a local tar file and frees GPU memory. -func (idx *CagraModel[T]) Unload() error { +func (idx *CagraModel[B, Q]) Unload() error { if idx.Index == nil { return nil } @@ -650,7 +648,7 @@ func (idx *CagraModel[T]) Unload() error { // returns one EventChunk per row. The caller (LoadIndex / search) sorts by // chunk_id before replay since record ordering across chunks encodes the // temporal ordering between DELETE and INSERT events for the same pkid. -func (idx *CagraModel[T]) loadCdcEventsFromDB( +func (idx *CagraModel[B, Q]) loadCdcEventsFromDB( sqlproc *sqlexec.SqlProcess, tblcfg vectorindex.IndexTableConfig, ) ([]cuvscdc.EventChunk, error) { @@ -682,16 +680,20 @@ func (idx *CagraModel[T]) loadCdcEventsFromDB( // flattens the (deleted, overflow) replay state into the parallel slices the // CagraModel struct carries (pkids/vecs/include layout that buildOverflow // expects). Pass includeBytesPerRow=0 for indexes without INCLUDE columns. -func replayEventChunks( +func replayEventChunks[B cuvs.VectorType]( chunks []cuvscdc.EventChunk, dim int, includeBytesPerRow int, -) ([]int64, []int64, []float32, []byte, error) { +) ([]int64, []int64, []B, []byte, error) { if len(chunks) == 0 { return nil, nil, nil, nil, nil } cuvscdc.SortChunks(chunks) - state, err := cuvscdc.ReplayEventLog(chunks, dim, includeBytesPerRow) + // The codec stores vectors as opaque bytes; the per-row byte length is + // dim * sizeof(B). Reinterpret each row's bytes back to the native base + // type B for the overflow brute force — no f32 detour. + vecBytesPerRow := dim * int(util.UnsafeSizeOf[B]()) + state, err := cuvscdc.ReplayEventLog(chunks, vecBytesPerRow, includeBytesPerRow) if err != nil { return nil, nil, nil, nil, err } @@ -703,14 +705,15 @@ func replayEventChunks( return deletedPkids, nil, nil, nil, nil } ovPkids := make([]int64, len(state.Overflow)) - ovVecs := make([]float32, len(state.Overflow)*dim) + ovVecs := make([]B, len(state.Overflow)*dim) + ovVecBytes := util.UnsafeSliceToBytes(ovVecs) var ovInc []byte if includeBytesPerRow > 0 { ovInc = make([]byte, len(state.Overflow)*includeBytesPerRow) } for i, e := range state.Overflow { ovPkids[i] = e.Pkid - copy(ovVecs[i*dim:(i+1)*dim], e.Vec) + copy(ovVecBytes[i*vecBytesPerRow:(i+1)*vecBytesPerRow], e.Vec) if includeBytesPerRow > 0 { copy(ovInc[i*includeBytesPerRow:(i+1)*includeBytesPerRow], e.Include) } @@ -720,7 +723,7 @@ func replayEventChunks( // LoadMetadata loads CagraModel descriptors from the metadata table. // Each returned model has Id, Checksum, Timestamp, and FileSize set; Index is nil. -func LoadMetadata[T cuvs.VectorType](sqlproc *sqlexec.SqlProcess, dbname string, metatbl string) ([]*CagraModel[T], error) { +func LoadMetadata[B, Q cuvs.VectorType](sqlproc *sqlexec.SqlProcess, dbname string, metatbl string) ([]*CagraModel[B, Q], error) { sql := fmt.Sprintf("SELECT * FROM %s ORDER BY timestamp ASC", sqlquote.QualifiedIdent(dbname, metatbl)) res, err := runSql(sqlproc, sql) if err != nil { @@ -733,7 +736,7 @@ func LoadMetadata[T cuvs.VectorType](sqlproc *sqlexec.SqlProcess, dbname string, total += bat.RowCount() } - indexes := make([]*CagraModel[T], 0, total) + indexes := make([]*CagraModel[B, Q], 0, total) for _, bat := range res.Batches { idVec := bat.Vecs[0] chksumVec := bat.Vecs[1] @@ -744,7 +747,7 @@ func LoadMetadata[T cuvs.VectorType](sqlproc *sqlexec.SqlProcess, dbname string, chksum := chksumVec.GetStringAt(i) ts := vector.GetFixedAtWithTypeCheck[int64](tsVec, i) fs := vector.GetFixedAtWithTypeCheck[int64](fsVec, i) - idx := &CagraModel[T]{Id: id, Checksum: chksum, Timestamp: ts, FileSize: fs} + idx := &CagraModel[B, Q]{Id: id, Checksum: chksum, Timestamp: ts, FileSize: fs} indexes = append(indexes, idx) } } diff --git a/pkg/vectorindex/cagra/model_test.go b/pkg/vectorindex/cagra/model_test.go index 29ab71065aca1..e2091100df0d9 100644 --- a/pkg/vectorindex/cagra/model_test.go +++ b/pkg/vectorindex/cagra/model_test.go @@ -130,19 +130,19 @@ func makeIndexBatch(proc *process.Process, tarPath string) *batch.Batch { // buildTestModel builds, trains and saves a CagraModel, returning it with Index==nil and // Path/Checksum/FileSize set. The caller is responsible for removing the tar file. -func buildTestModel(t *testing.T, id string, ids []int64) *CagraModel[float32] { +func buildTestModel(t *testing.T, id string, ids []int64) *CagraModel[float32, float32] { t.Helper() idxcfg := testIdxcfg() data := generateTestData(testNVectors, testDim) - m, err := NewCagraModelForBuild[float32](id, idxcfg, 1, []int{0}) + m, err := NewCagraModelForBuild[float32, float32](id, idxcfg, 1, []int{0}) require.NoError(t, err) err = m.InitEmpty(testNVectors) require.NoError(t, err) - err = m.AddChunkFloat(data, testNVectors, ids) + err = m.AddChunkQuantize(data, testNVectors, ids) require.NoError(t, err) err = m.Build() @@ -183,7 +183,7 @@ func TestModelStreamError(t *testing.T) { defer func() { runSql = origRunSql }() // Manually create a model descriptor as if loaded from metadata. - idx := &CagraModel[float32]{ + idx := &CagraModel[float32, float32]{ Id: "test-stream-err", FileSize: 1024, // non-zero triggers DB download Checksum: "fake-checksum", @@ -211,13 +211,13 @@ func TestModelBuildAndLoad(t *testing.T) { } // ---- Build ---- - built, err := NewCagraModelForBuild[float32]("test-build", idxcfg, 1, []int{0}) + built, err := NewCagraModelForBuild[float32, float32]("test-build", idxcfg, 1, []int{0}) require.NoError(t, err) err = built.InitEmpty(testNVectors) require.NoError(t, err) - err = built.AddChunkFloat(data, testNVectors, ids) + err = built.AddChunkQuantize(data, testNVectors, ids) require.NoError(t, err) err = built.Build() @@ -247,7 +247,7 @@ func TestModelBuildAndLoad(t *testing.T) { defer func() { runSql = origRunSql }() // ---- Load from local tar (skips DB download since Path is set) ---- - loader := &CagraModel[float32]{ + loader := &CagraModel[float32, float32]{ Id: "test-build", Path: tarPath, Checksum: checksum, @@ -340,7 +340,7 @@ func TestModelLoadFromDB(t *testing.T) { defer func() { runSql = origRunSql }() // LoadMetadata — creates a model from DB metadata. - models, err := LoadMetadata[float32](sqlproc, tblcfg.DbName, tblcfg.MetadataTable) + models, err := LoadMetadata[float32, float32](sqlproc, tblcfg.DbName, tblcfg.MetadataTable) require.NoError(t, err) require.Equal(t, 1, len(models)) @@ -370,7 +370,7 @@ func TestModelNil(t *testing.T) { var tblcfg vectorindex.IndexTableConfig // Zero-value model: no index, no path. - idx := &CagraModel[float32]{} + idx := &CagraModel[float32, float32]{} // InitEmpty fails because Devices is empty. err := idx.InitEmpty(10) @@ -386,7 +386,7 @@ func TestModelNil(t *testing.T) { require.NotNil(t, err) // AddChunkFloat fails because Index is nil. - err = idx.AddChunkFloat([]float32{1, 2}, 1, []int64{1}) + err = idx.AddChunkQuantize([]float32{1, 2}, 1, []int64{1}) require.NotNil(t, err) // Search fails because Index is nil. @@ -394,7 +394,7 @@ func TestModelNil(t *testing.T) { require.NotNil(t, err) // Search with nil query fails. - idx2 := &CagraModel[float32]{} // still nil Index + idx2 := &CagraModel[float32, float32]{} // still nil Index _, _, err = idx2.Search(nil, 1) require.NotNil(t, err) @@ -432,7 +432,7 @@ func TestModelEmptyBuild(t *testing.T) { idxcfg := testIdxcfg() tblcfg := testTblcfg() - built, err := NewCagraModelForBuild[float32]("test-empty", idxcfg, 1, []int{0}) + built, err := NewCagraModelForBuild[float32, float32]("test-empty", idxcfg, 1, []int{0}) require.NoError(t, err) // InitEmpty with 0 would fail in CAGRA, so test saveToFile directly on empty Len. diff --git a/pkg/vectorindex/cagra/plugin/compile/compile.go b/pkg/vectorindex/cagra/plugin/compile/compile.go index 6246f7ec8ff2c..e275dac193fb4 100644 --- a/pkg/vectorindex/cagra/plugin/compile/compile.go +++ b/pkg/vectorindex/cagra/plugin/compile/compile.go @@ -80,7 +80,7 @@ func (h Hooks) HandleCreateIndex(ctx compileplugin.CompileContext, indexDefs map // forceSync. The idxcron background reindex executor passes // forceSync=true so the build happens synchronously inside the txn // before the CDC task picks up forward changes. Mirrors IVF-FLAT. -func (h Hooks) HandleReindex(ctx compileplugin.CompileContext, indexDefs map[string]*plan.IndexDef, forceSync bool) error { +func (h Hooks) HandleReindex(ctx compileplugin.CompileContext, indexDefs map[string]*plan.IndexDef, forceSync bool, _ bool) error { return h.handleCreate(ctx, indexDefs, forceSync) } @@ -234,17 +234,38 @@ func registerIdxcronUpdate( } func (Hooks) ValidateReindexParams(old map[string]string, alter compileplugin.ReindexParamUpdate) (map[string]string, error) { - return compileplugin.MergeReindexParams(old, alter, "cagra", + // Merge first, then validate the EFFECTIVE quantization via the per-algo + // catalog hook (the single home shared with CREATE). The merged map is the + // index's actual post-reindex config: the value the reindex set, or — when + // the reindex omitted QUANTIZATION (e.g. the idxcron-issued rebuild) — the + // value already stored on the index. Validating the merge (not the raw alter + // delta) means the check is never skipped just because the statement omitted + // quantization, and quantization and op_type come from one consistent source. + merged, err := compileplugin.MergeReindexParams(old, alter, "cagra", catalog.IndexAlgoParamMaxIndexCapacity, catalog.IntermediateGraphDegree, catalog.GraphDegree, catalog.ITopkSize, + catalog.Quantization, ) + if err != nil { + return nil, err + } + if err := (cagraruntime.CatalogHooks{}).ValidQuantization( + merged[catalog.Quantization], merged[catalog.IndexAlgoParamOpType]); err != nil { + return nil, err + } + return merged, nil } -// HandleDropIndex is a no-op: generic hidden-table cleanup is sufficient. func (Hooks) HandleDropIndex(_ compileplugin.CompileContext, defs map[string]*plan.IndexDef) error { logutil.Infof("[plugin] cagra HandleDropIndex: defs=%d", len(defs)) + // Evict the cached search index so its GPU resources are freed NOW, rather + // than lingering until the 5-min VectorIndexCacheTTL housekeeping reaps it. + // Mirrors the create-side cache.Cache.Remove(storageDef.IndexTableName). + if storageDef, ok := defs[catalog.Cagra_TblType_Storage]; ok { + cache.Cache.Remove(storageDef.IndexTableName) + } return nil } diff --git a/pkg/vectorindex/cagra/plugin/compile/compile_test.go b/pkg/vectorindex/cagra/plugin/compile/compile_test.go index 5a0ab619a1b42..dd80c7e51b497 100644 --- a/pkg/vectorindex/cagra/plugin/compile/compile_test.go +++ b/pkg/vectorindex/cagra/plugin/compile/compile_test.go @@ -375,7 +375,7 @@ func TestCagraHandleCreateIndex_AsyncFalseExplicit(t *testing.T) { func TestCagraHandleCreateIndex_BackgroundReentry(t *testing.T) { ctx := newHandleCtx(true) ctx.stubCompileContext.isFrontend = false - err := Hooks{}.HandleReindex(ctx, cagraIndexDefs(), true) + err := Hooks{}.HandleReindex(ctx, cagraIndexDefs(), true, false) require.NoError(t, err) require.True(t, ctx.stubCompileContext.lastCdcTask.called, "background re-entry still drives the CDC task") require.False(t, ctx.stubCompileContext.lastIdxcronUpdate.called, "background re-entry must NOT rewrite mo_index_update") @@ -385,7 +385,7 @@ func TestCagraHandleReindex_DelegatesToCreate(t *testing.T) { // HandleReindex is a thin pass-through to handleCreate; honors // the forceSync arg directly (unlike HandleCreateIndex, which // now reads catalog.IsIndexAsync). - err := Hooks{}.HandleReindex(newHandleCtx(true), cagraIndexDefs(), false) + err := Hooks{}.HandleReindex(newHandleCtx(true), cagraIndexDefs(), false, false) require.NoError(t, err) } @@ -424,3 +424,34 @@ func TestCagraValidateReindexParams_MergesGraphDegree(t *testing.T) { _, had := old[catalog.GraphDegree] require.False(t, had) } + +// TestCagraValidateReindexParams_Quantization: CAGRA (cuvs) accepts the +// cuvs-supported quantization names and rejects others (e.g. bf16, which the +// cuvs backend does not support even though IVF-FLAT does). +func TestCagraValidateReindexParams_Quantization(t *testing.T) { + got, err := Hooks{}.ValidateReindexParams(nil, compileplugin.ReindexParamUpdate{ + Params: map[string]string{catalog.Quantization: "float16"}, + }) + require.NoError(t, err) + require.Equal(t, "float16", got[catalog.Quantization]) + + _, err = Hooks{}.ValidateReindexParams(nil, compileplugin.ReindexParamUpdate{ + Params: map[string]string{catalog.Quantization: "bf16"}, + }) + require.Error(t, err) + + // int8/uint8 on a non-L2 (inner-product) index IS rejected at REINDEX via the + // ValidQuantization hook: the merged op_type is inner-product and the + // int8/uint8 affine quantizer only preserves L2 geometry. + _, err = Hooks{}.ValidateReindexParams( + map[string]string{catalog.IndexAlgoParamOpType: "vector_ip_ops"}, + compileplugin.ReindexParamUpdate{Params: map[string]string{catalog.Quantization: "int8"}}) + require.Error(t, err) + + // ...but int8 with L2 (the merged op_type) is accepted. + got, err = Hooks{}.ValidateReindexParams( + map[string]string{catalog.IndexAlgoParamOpType: "vector_l2_ops"}, + compileplugin.ReindexParamUpdate{Params: map[string]string{catalog.Quantization: "int8"}}) + require.NoError(t, err) + require.Equal(t, "int8", got[catalog.Quantization]) +} diff --git a/pkg/vectorindex/cagra/plugin/iscp/iscp.go b/pkg/vectorindex/cagra/plugin/iscp/iscp.go index 15eb40dc81d6c..ab0d4430c0af6 100644 --- a/pkg/vectorindex/cagra/plugin/iscp/iscp.go +++ b/pkg/vectorindex/cagra/plugin/iscp/iscp.go @@ -62,6 +62,6 @@ func (Hooks) Run(c *iscppkg.IndexConsumer, ctx context.Context, errch chan error iscppkg.RunCuvs(c, ctx, errch, r, func(sqlproc *sqlexec.SqlProcess) (iscppkg.CuvsSync, error) { w := c.SqlWriter().(*iscppkg.CuvsCdcWriter) return cagra.NewCagraSync(sqlproc, w.DbName(), w.TblName(), w.IndexName(), - w.IndexDef(), w.Dimension(), w.ColMetaJSON()) + w.IndexDef(), w.Dimension(), w.BaseVectorType(), w.ColMetaJSON()) }) } diff --git a/pkg/vectorindex/cagra/plugin/plan/plan_test.go b/pkg/vectorindex/cagra/plugin/plan/plan_test.go index b4f9661a447e0..968312d5681f1 100644 --- a/pkg/vectorindex/cagra/plugin/plan/plan_test.go +++ b/pkg/vectorindex/cagra/plugin/plan/plan_test.go @@ -168,6 +168,62 @@ func TestBuildSecondaryIndexDefs_OK(t *testing.T) { require.NotNil(t, tblDefs[1].Pkey) } +// indexOnQuant builds a single-column *tree.Index over colName with a +// QUANTIZATION option. +func indexOnQuant(colName, quant string) *tree.Index { + idx := indexOn(colName) + idx.IndexOption = &tree.IndexOption{Quantization: quant} + return idx +} + +// f16ColMap returns a colMap with an int64 pk and a vecf16 base column. +func f16ColMap(pkName, vecName string) map[string]*plan.ColDef { + m := vecColMap(pkName, vecName) + m[vecName].Typ.Id = int32(types.T_array_float16) + return m +} + +// TestBuildSecondaryIndexDefs_F16Base: a vecf16 base column is accepted. +func TestBuildSecondaryIndexDefs_F16Base(t *testing.T) { + idxDefs, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOn("vec"), f16ColMap("id", "vec"), nil, "id") + require.NoError(t, err) + require.Len(t, idxDefs, 2) +} + +// TestBuildSecondaryIndexDefs_UnsupportedBase: only vecf32 / vecf16 are valid +// base columns; vecf64 / vecbf16 / vecint8 / vecuint8 are rejected. +func TestBuildSecondaryIndexDefs_UnsupportedBase(t *testing.T) { + for _, oid := range []types.T{ + types.T_array_float64, types.T_array_bf16, types.T_array_int8, types.T_array_uint8, + } { + colMap := vecColMap("id", "vec") + colMap["vec"].Typ.Id = int32(oid) + _, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOn("vec"), colMap, nil, "id") + require.Error(t, err, "base type %s must be rejected", oid) + } +} + +// TestBuildSecondaryIndexDefs_F16UpcastRejected: vecf16 base + QUANTIZATION +// float32 is an upcast (4 > 2 bytes) and must be rejected by the downcast +// guard. (The accepted downcast path — f16 -> int8/uint8 — is exercised +// end-to-end by the GPU functional BVT, since the full def build past the +// guard needs a richer compiler context than this stub provides.) +func TestBuildSecondaryIndexDefs_F16UpcastRejected(t *testing.T) { + _, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOnQuant("vec", "float32"), f16ColMap("id", "vec"), nil, "id") + require.Error(t, err) +} + +// TestBuildSecondaryIndexDefs_BF16QuantRejected: QUANTIZATION 'bf16' has no GPU +// bfloat16 storage (cuVS has no bfloat16 index/quantizer), so it must be rejected +// rather than silently falling back to f32 storage — even though it passes the +// downcast width guard (bf16 is 2 bytes). Rejected on both f32 and f16 bases. +func TestBuildSecondaryIndexDefs_BF16QuantRejected(t *testing.T) { + _, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOnQuant("vec", "bf16"), vecColMap("id", "vec"), nil, "id") + require.Error(t, err, "f32 base + bf16 quant must be rejected") + _, _, err = Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOnQuant("vec", "bf16"), f16ColMap("id", "vec"), nil, "id") + require.Error(t, err, "f16 base + bf16 quant must be rejected") +} + // --- schema.go: BuildFullTextIndexDefs ------------------------------------- func TestBuildFullTextIndexDefs_Unsupported(t *testing.T) { diff --git a/pkg/vectorindex/cagra/plugin/plan/schema.go b/pkg/vectorindex/cagra/plugin/plan/schema.go index fea244c98c1fe..03e04804706b0 100644 --- a/pkg/vectorindex/cagra/plugin/plan/schema.go +++ b/pkg/vectorindex/cagra/plugin/plan/schema.go @@ -24,6 +24,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" "github.com/matrixorigin/matrixone/pkg/sql/util" cagrart "github.com/matrixorigin/matrixone/pkg/vectorindex/cagra/plugin/runtime" + "github.com/matrixorigin/matrixone/pkg/vectorindex/quantizer" ) // cagraCatalogHooks is the shared (stateless) catalog-hooks instance used for @@ -63,7 +64,40 @@ func (Hooks) BuildSecondaryIndexDefs( return nil, nil, moerr.NewInvalidInputf(ctx.GetContext(), "column '%s' is not exist", indexInfo.KeyParts[0].ColName.ColNameOrigin()) } if !catalogplugin.SupportsVectorType(cagraCatalogHooks, types.T(colMap[name].Typ.Id)) { - return nil, nil, moerr.NewNotSupported(ctx.GetContext(), "Cagra only supports VECF32 column types") + return nil, nil, moerr.NewNotSupported(ctx.GetContext(), "Cagra only supports VECF32 / VECF16 base column types") + } + // QUANTIZATION is downcast-only: the storage element must be the same width + // or narrower than the base column (f16 base -> int8/uint8 OK; f16 base -> + // float32 is an upcast and rejected). Mirrors ivfflat's guard. + if indexInfo.IndexOption != nil && indexInfo.IndexOption.Quantization != "" { + if qt, ok := quantizer.ToVectorType(indexInfo.IndexOption.Quantization); ok { + // bf16 storage does not exist on the GPU (cuVS/cgo has no bfloat16 + // index or quantizer), so reject it explicitly rather than silently + // falling back to f32 storage. Supported cuvs storage = f16/int8/uint8. + if qt == types.T_array_bf16 { + return nil, nil, moerr.NewNotSupportedf(ctx.GetContext(), + "Cagra does not support '%s' quantization (no GPU bfloat16 storage); use 'float16', 'int8', or 'uint8'", + indexInfo.IndexOption.Quantization) + } + baseSize := types.Type{Oid: types.T(colMap[name].Typ.Id)}.GetArrayElementSize() + quantSize := types.Type{Oid: qt}.GetArrayElementSize() + if quantSize > baseSize { + return nil, nil, moerr.NewNotSupportedf(ctx.GetContext(), + "Cagra QUANTIZATION '%s' (%d bytes/element) cannot upcast base column %s (%d bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type", + indexInfo.IndexOption.Quantization, quantSize, + types.T(colMap[name].Typ.Id).String(), baseSize) + } + // int8/uint8 quantization is L2-only (the affine quantizer breaks + // inner-product / cosine geometry). Gated by the per-algo catalog + // hook — the single home shared with REINDEX + // (compile/ValidateReindexParams) — so CREATE and REINDEX cannot + // drift. (bf16 and width/upcast are rejected above with base-column- + // aware messages before reaching here.) + if err := cagraCatalogHooks.ValidQuantization( + indexInfo.IndexOption.Quantization, indexInfo.IndexOption.AlgoParamVectorOpType); err != nil { + return nil, nil, err + } + } } for _, existedIndex := range existedIndexes { if existedIndex.IndexAlgo == catalog.MoIndexCagraAlgo.ToString() && existedIndex.Parts[0] == name { diff --git a/pkg/vectorindex/cagra/plugin/runtime/runtime.go b/pkg/vectorindex/cagra/plugin/runtime/runtime.go index 2a2e91fd778dc..2990af9781aab 100644 --- a/pkg/vectorindex/cagra/plugin/runtime/runtime.go +++ b/pkg/vectorindex/cagra/plugin/runtime/runtime.go @@ -99,12 +99,43 @@ const CagraIndexFlag = "experimental_cagra_index" // ExperimentalFlag: CAGRA DDL is gated by CagraIndexFlag. func (CatalogHooks) ExperimentalFlag() string { return CagraIndexFlag } -// SupportedVectorTypes: CAGRA (cuvs) indexes f32 vectors only. -func (CatalogHooks) SupportedVectorTypes() []types.T { return []types.T{types.T_array_float32} } +// SupportedVectorTypes: CAGRA (cuvs) accepts f32 and f16 base columns. f16 is +// stored natively as half, or downcast-quantized to int8/uint8 via QUANTIZATION. +// int8/uint8 base columns are unsupported (the CDC overflow brute force is f32/f16-only). +func (CatalogHooks) SupportedVectorTypes() []types.T { + return []types.T{types.T_array_float32, types.T_array_float16} +} // SupportedPrimaryKeyTypes: requires an int64 primary key. func (CatalogHooks) SupportedPrimaryKeyTypes() []types.T { return []types.T{types.T_int64} } +// ValidQuantization gates the (quantization, op_type) pair for CAGRA (cuvs): +// the value must be a cuvs storage type (float32/float16/int8/uint8 — bf16 and +// float64 are absent from CuvsQuantizationNameToType), and the 1-byte int8/uint8 +// scalar quantizer is L2-only (its affine map q(x)=scalar*x+offset preserves L2 +// ordering — the offset cancels in a difference — but biases inner-product and +// rotates cosine angles). One home for CREATE (plan/schema) and REINDEX +// (compile/ValidateReindexParams). quant=="" => no quantization (valid); op=="" +// => value rule only. +func (CatalogHooks) ValidQuantization(quant, op string) error { + if quant == "" { + return nil + } + quant = strings.ToLower(quant) + if !metric.ValidQuantization(quant) { + return moerr.NewNotSupportedNoCtxf( + "cagra quantization %q (supported: float32, float16, int8, uint8)", quant) + } + if quant == metric.Quantization_INT8_Str || quant == metric.Quantization_UINT8_Str { + switch strings.ToLower(op) { + case metric.OpType_InnerProduct, metric.OpType_CosineDistance: + return moerr.NewNotSupportedNoCtxf( + "cagra quantization %q is only supported with L2 (op_type 'vector_l2_ops'); the int8/uint8 affine quantizer does not preserve inner-product / cosine geometry", quant) + } + } + return nil +} + // SupportedIncludeColumnTypes: cuvs INCLUDE (pre-filter) columns accept // int32/int64/float32/float64 scalars. func (CatalogHooks) SupportedIncludeColumnTypes() []types.T { @@ -198,6 +229,9 @@ func (CatalogHooks) ParamsFromTree(idx *tree.Index) (map[string]string, error) { if idx.IndexOption.Hour > 0 { res[catalog.Hour] = strconv.FormatInt(idx.IndexOption.Hour, 10) } + if idx.IndexOption.Second > 0 { + res[catalog.Second] = strconv.FormatInt(idx.IndexOption.Second, 10) + } if len(idx.IndexOption.Quantization) > 0 { quantize := catalog.ToLower(idx.IndexOption.Quantization) diff --git a/pkg/vectorindex/cagra/plugin/runtime/runtime_test.go b/pkg/vectorindex/cagra/plugin/runtime/runtime_test.go index f165fcdeb0921..86776a04d48f7 100644 --- a/pkg/vectorindex/cagra/plugin/runtime/runtime_test.go +++ b/pkg/vectorindex/cagra/plugin/runtime/runtime_test.go @@ -181,3 +181,16 @@ func TestCagraJoinIncludeColumns(t *testing.T) { } require.Equal(t, "a,b", joinIncludeColumns(cols)) } + +// TestCagraValidQuantization exercises the per-algo (quant, op) catalog hook +// shared by CREATE and REINDEX. +func TestCagraValidQuantization(t *testing.T) { + h := CatalogHooks{} + require.NoError(t, h.ValidQuantization("", "vector_ip_ops")) // no quantization + require.NoError(t, h.ValidQuantization("float16", "vector_ip_ops")) // f16 fine with any op + require.NoError(t, h.ValidQuantization("int8", "vector_l2_ops")) // int8 + L2 ok + require.Error(t, h.ValidQuantization("int8", "vector_ip_ops")) // int8 + ip rejected + require.Error(t, h.ValidQuantization("uint8", "vector_cosine_ops")) // uint8 + cosine rejected + require.Error(t, h.ValidQuantization("bf16", "vector_l2_ops")) // bad value + require.Error(t, h.ValidQuantization("float64", "vector_l2_ops")) // bad value +} diff --git a/pkg/vectorindex/cagra/search_gpu.go b/pkg/vectorindex/cagra/search_gpu.go index 30547cfdde702..7a759e20599ae 100644 --- a/pkg/vectorindex/cagra/search_gpu.go +++ b/pkg/vectorindex/cagra/search_gpu.go @@ -31,19 +31,19 @@ import ( // CagraSearch implements cache.VectorIndexSearchIf for GPU CAGRA indexes. // Unlike HnswSearch, there is no concurrency gate (Cond/Mutex) because CAGRA // manages GPU thread concurrency internally via its worker pool. -type CagraSearch[T cuvs.VectorType] struct { +type CagraSearch[B, Q cuvs.VectorType] struct { Idxcfg vectorindex.IndexConfig Tblcfg vectorindex.IndexTableConfig - Indexes []*CagraModel[T] - MultiIndex *cuvs.MultiGpuCagra[T] // built once in Load; nil until indexes are loaded - Overflow *cuvs.GpuBruteForce[T] // CDC insert overflow; nil when no overflow records exist + Indexes []*CagraModel[B, Q] + MultiIndex *cuvs.MultiGpuCagra[B, Q] // built once in Load; nil until indexes are loaded + Overflow cuvs.BruteForceOverflow[B] // CDC insert overflow; nil when no overflow records exist Devices []int ThreadsSearch int64 } -func NewCagraSearch[T cuvs.VectorType](idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig, devices []int) *CagraSearch[T] { +func NewCagraSearch[B, Q cuvs.VectorType](idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig, devices []int) *CagraSearch[B, Q] { nthread := vectorindex.GetConcurrency(tblcfg.ThreadsSearch) - return &CagraSearch[T]{ + return &CagraSearch[B, Q]{ Idxcfg: idxcfg, Tblcfg: tblcfg, Devices: devices, @@ -52,11 +52,7 @@ func NewCagraSearch[T cuvs.VectorType](idxcfg vectorindex.IndexConfig, tblcfg ve } // Search implements cache.VectorIndexSearchIf. -func (s *CagraSearch[T]) Search(sqlproc *sqlexec.SqlProcess, anyquery any, rt vectorindex.RuntimeConfig) (keys any, distances []float64, err error) { - query, ok := anyquery.([]float32) - if !ok { - return nil, nil, moerr.NewInternalErrorNoCtx("CagraSearch: query type mismatch") - } +func (s *CagraSearch[B, Q]) Search(sqlproc *sqlexec.SqlProcess, anyquery any, rt vectorindex.RuntimeConfig) (keys any, distances []float64, err error) { if s.MultiIndex == nil { return []int64{}, []float64{}, nil @@ -84,10 +80,18 @@ func (s *CagraSearch[T]) Search(sqlproc *sqlexec.SqlProcess, anyquery any, rt ve neighbors64 []int64 dists32 []float32 ) + // Any base (f32 or vecf16) routes its native base-typed (B) query through the + // const-B* search_quantize path — cuVS converts B to storage Q on device (B==Q + // copy for a direct index, learned/cast quantizer for a compressed one). The + // query asserts to []B for both float32 (B==float) and Float16 (B==half) base. + qB, ok := anyquery.([]B) + if !ok { + return nil, nil, moerr.NewInternalErrorNoCtx("CagraSearch: query type mismatch") + } if rt.FilterJSON != "" { - neighbors64, dists32, err = s.MultiIndex.SearchFloat32WithFilter(query, 1, dim, uint32(limit), sp, rt.FilterJSON) + neighbors64, dists32, err = s.MultiIndex.SearchQuantizeWithFilter(qB, 1, dim, uint32(limit), sp, rt.FilterJSON) } else { - neighbors64, dists32, err = s.MultiIndex.SearchFloat32(query, 1, dim, uint32(limit), sp) + neighbors64, dists32, err = s.MultiIndex.SearchQuantize(qB, 1, dim, uint32(limit), sp) } if err != nil { return nil, nil, err @@ -114,7 +118,7 @@ func (s *CagraSearch[T]) Search(sqlproc *sqlexec.SqlProcess, anyquery any, rt ve // SearchFloat32 implements cache.VectorIndexSearchIf. // Writes results directly into caller-provided slices to avoid heap allocation. -func (s *CagraSearch[T]) SearchFloat32(proc *sqlexec.SqlProcess, query any, rt vectorindex.RuntimeConfig, outKeys []int64, outDists []float32) error { +func (s *CagraSearch[B, Q]) SearchFloat32(proc *sqlexec.SqlProcess, query any, rt vectorindex.RuntimeConfig, outKeys []int64, outDists []float32) error { keys, dists, err := s.Search(proc, query, rt) if err != nil { return err @@ -137,8 +141,8 @@ func (s *CagraSearch[T]) SearchFloat32(proc *sqlexec.SqlProcess, query any, rt v // into per-column data + null bitmap and feeds them to the brute-force index // in column order. Mirrors how the build path populates the cuvs main // index's FilterStore. -func addOverflowFilterChunks[T cuvs.VectorType]( - bf *cuvs.GpuBruteForce[T], +func addOverflowFilterChunks[B, OB cuvs.VectorType]( + bf *cuvs.GpuBruteForce[B, OB], colMetaJSON string, includeBytes []byte, nrows uint64, @@ -157,8 +161,8 @@ func addOverflowFilterChunks[T cuvs.VectorType]( } // Load implements cache.VectorIndexSearchIf: loads metadata then index data from the database. -func (s *CagraSearch[T]) Load(sqlproc *sqlexec.SqlProcess) (err error) { - indexes, err := LoadMetadata[T](sqlproc, s.Tblcfg.DbName, s.Tblcfg.MetadataTable) +func (s *CagraSearch[B, Q]) Load(sqlproc *sqlexec.SqlProcess) (err error) { + indexes, err := LoadMetadata[B, Q](sqlproc, s.Tblcfg.DbName, s.Tblcfg.MetadataTable) if err != nil { return err } @@ -201,7 +205,7 @@ func (s *CagraSearch[T]) Load(sqlproc *sqlexec.SqlProcess) (err error) { // index by construction (CDC writer side is fed the same colMetaJSON). If // no sub-index loaded (empty index — never built, or built and dropped), // we have no col-meta and skip; cdc_tail data is moot without a main index. -func (s *CagraSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { +func (s *CagraSearch[B, Q]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { var ( includeBytesPerRow int colMetaJSON string @@ -219,7 +223,7 @@ func (s *CagraSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { } } - stub := &CagraModel[T]{Id: vectorindex.CdcTailId} + stub := &CagraModel[B, Q]{Id: vectorindex.CdcTailId} chunks, err := stub.loadCdcEventsFromDB(sqlproc, s.Tblcfg) if err != nil { return err @@ -243,7 +247,7 @@ func (s *CagraSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { } dim := int(s.Idxcfg.CuvsCagra.Dimensions) - delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks(chunks, dim, includeBytesPerRow) + delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks[B](chunks, dim, includeBytesPerRow) if err != nil { return err } @@ -259,7 +263,7 @@ func (s *CagraSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { } } - s.Indexes = append(s.Indexes, &CagraModel[T]{ + s.Indexes = append(s.Indexes, &CagraModel[B, Q]{ Id: vectorindex.CdcTailId, DeletedPkids: delPkids, OverflowPkids: ovPkids, @@ -278,7 +282,7 @@ func (s *CagraSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { // When the underlying index has INCLUDE columns, the brute-force is set up // with the matching FilterStore so a filtered query can prefilter overflow // rows the same way the main cagra index does. -func (s *CagraSearch[T]) buildOverflow() error { +func (s *CagraSearch[B, Q]) buildOverflow() error { total := uint64(0) for _, m := range s.Indexes { total += uint64(len(m.OverflowPkids)) @@ -299,14 +303,43 @@ func (s *CagraSearch[T]) buildOverflow() error { device = s.Devices[0] } - bf, err := cuvs.NewGpuBruteForceEmpty[T]( - total, dim, cuvsMetric, uint32(s.ThreadsSearch), device) + // cuVS brute force can only store float/half. Pick the overflow storage type + // OB from the index storage Q: keep Q when it is float/half, else fall back to + // the base type B (which is always float/half) so the overflow is supported. + // The type-erased BruteForceOverflow[B] interface holds either concrete type. + var ( + ov cuvs.BruteForceOverflow[B] + err error + ) + switch cuvs.GetQuantization[Q]() { + case cuvs.F32, cuvs.F16: + ov, err = buildOverflowBF[B, Q](s.Indexes, total, dim, cuvsMetric, device, uint32(s.ThreadsSearch)) + default: // INT8/UINT8: brute force can't store these → store base B. + ov, err = buildOverflowBF[B, B](s.Indexes, total, dim, cuvsMetric, device, uint32(s.ThreadsSearch)) + } if err != nil { return err } + s.Overflow = ov + return nil +} + +// buildOverflowBF builds a concrete *cuvs.GpuBruteForce[B, OB] from every loaded +// model's CDC insert overflow and returns it behind the type-erased +// BruteForceOverflow[B] interface. OB is the overflow storage type chosen by the +// caller (Q when float/half, else B). Wires the FilterStore when the index has +// INCLUDE columns. +func buildOverflowBF[B, OB cuvs.VectorType, Q cuvs.VectorType]( + indexes []*CagraModel[B, Q], + total uint64, dim uint32, cuvsMetric cuvs.DistanceType, device int, threads uint32, +) (cuvs.BruteForceOverflow[B], error) { + bf, err := cuvs.NewGpuBruteForceEmpty[B, OB](total, dim, cuvsMetric, threads, device) + if err != nil { + return nil, err + } if err = bf.Start(); err != nil { bf.Destroy() - return err + return nil, err } // INCLUDE-column wiring — pull the col-meta JSON from the first loaded @@ -318,7 +351,7 @@ func (s *CagraSearch[T]) buildOverflow() error { colMetaJSON string includeBytesPerRow int ) - for _, m := range s.Indexes { + for _, m := range indexes { if m.Index != nil { colMetaJSON = m.Index.GetFilterColMetaJSON() includeBytesPerRow = m.IncludeBytesPerRow @@ -326,7 +359,7 @@ func (s *CagraSearch[T]) buildOverflow() error { } } if colMetaJSON == "" { - for _, m := range s.Indexes { + for _, m := range indexes { if m.OverflowColMetaJSON != "" { colMetaJSON = m.OverflowColMetaJSON includeBytesPerRow = m.IncludeBytesPerRow @@ -337,32 +370,33 @@ func (s *CagraSearch[T]) buildOverflow() error { if colMetaJSON != "" && includeBytesPerRow > 0 { if err = bf.SetFilterColumns(colMetaJSON, total); err != nil { bf.Destroy() - return err + return nil, err } } - for _, m := range s.Indexes { + for _, m := range indexes { if len(m.OverflowPkids) == 0 { continue } count := uint64(len(m.OverflowPkids)) - if err = bf.AddChunkFloat(m.OverflowVecs, count, m.OverflowPkids); err != nil { + // Overflow vectors are base-typed (B); AddChunkQuantize converts B -> Q + // storage on the C++ side (native store when B==Q, f32->f16 cast otherwise). + if err = bf.AddChunkQuantize(m.OverflowVecs, count, m.OverflowPkids); err != nil { bf.Destroy() - return err + return nil, err } if colMetaJSON != "" && includeBytesPerRow > 0 { if err = addOverflowFilterChunks(bf, colMetaJSON, m.OverflowIncludeBytes, count, includeBytesPerRow); err != nil { bf.Destroy() - return err + return nil, err } } } if err = bf.Build(); err != nil { bf.Destroy() - return err + return nil, err } - s.Overflow = bf - return nil + return bf, nil } // buildMultiIndex assembles a MultiGpuCagra from the loaded indexes. @@ -372,14 +406,14 @@ func (s *CagraSearch[T]) buildOverflow() error { // which returns []int64{}, []float64{} on s.MultiIndex == nil — that's // the load-bearing path for "no main index + no brute-force → empty // result". Any future regression here will fail TestCagraSearchEmpty. -func (s *CagraSearch[T]) buildMultiIndex() (*cuvs.MultiGpuCagra[T], error) { +func (s *CagraSearch[B, Q]) buildMultiIndex() (*cuvs.MultiGpuCagra[B, Q], error) { cuvsMetric, ok := metric.MetricTypeToCuvsMetric[metric.MetricType(s.Idxcfg.CuvsCagra.Metric)] if !ok { // Unsupported metric is a real error — surface it rather than returning a // nil index, which Search would treat as an (empty) success. return nil, moerr.NewInternalErrorNoCtxf("CagraSearch: unsupported metric type %v", s.Idxcfg.CuvsCagra.Metric) } - gpuIndices := make([]*cuvs.GpuCagra[T], 0, len(s.Indexes)) + gpuIndices := make([]*cuvs.GpuCagra[B, Q], 0, len(s.Indexes)) for _, model := range s.Indexes { if model.Index != nil { gpuIndices = append(gpuIndices, model.Index) @@ -396,7 +430,7 @@ func (s *CagraSearch[T]) buildMultiIndex() (*cuvs.MultiGpuCagra[T], error) { // loadIndexes loads each model's index data from the database. // On any error it destroys all partially-loaded indexes and returns the error. -func (s *CagraSearch[T]) loadIndexes(sqlproc *sqlexec.SqlProcess, indexes []*CagraModel[T]) ([]*CagraModel[T], error) { +func (s *CagraSearch[B, Q]) loadIndexes(sqlproc *sqlexec.SqlProcess, indexes []*CagraModel[B, Q]) ([]*CagraModel[B, Q], error) { for _, idx := range indexes { idx.Devices = s.Devices if err := idx.LoadIndex(sqlproc, s.Idxcfg, s.Tblcfg, s.ThreadsSearch, true); err != nil { @@ -410,7 +444,7 @@ func (s *CagraSearch[T]) loadIndexes(sqlproc *sqlexec.SqlProcess, indexes []*Cag } // Destroy implements cache.VectorIndexSearchIf. -func (s *CagraSearch[T]) Destroy() { +func (s *CagraSearch[B, Q]) Destroy() { s.MultiIndex = nil // does not own GPU resources; GpuCagra instances are owned by Indexes if s.Overflow != nil { s.Overflow.Destroy() @@ -423,6 +457,6 @@ func (s *CagraSearch[T]) Destroy() { } // UpdateConfig implements cache.VectorIndexSearchIf. -func (s *CagraSearch[T]) UpdateConfig(newalgo cache.VectorIndexSearchIf) error { +func (s *CagraSearch[B, Q]) UpdateConfig(newalgo cache.VectorIndexSearchIf) error { return nil } diff --git a/pkg/vectorindex/cagra/search_test.go b/pkg/vectorindex/cagra/search_test.go index 35c5018f3c77e..463c7732cf7c0 100644 --- a/pkg/vectorindex/cagra/search_test.go +++ b/pkg/vectorindex/cagra/search_test.go @@ -34,7 +34,7 @@ import ( // loadedModel builds an index, saves it to a tar, then reloads it into GPU // memory from the local file. Returns the model with Index != nil. -func loadedModel(t *testing.T, id string) *CagraModel[float32] { +func loadedModel(t *testing.T, id string) *CagraModel[float32, float32] { t.Helper() built := buildTestModel(t, id, nil) tarPath := built.Path @@ -53,7 +53,7 @@ func loadedModel(t *testing.T, id string) *CagraModel[float32] { } defer func() { runSql = origRunSql }() - loader := &CagraModel[float32]{ + loader := &CagraModel[float32, float32]{ Id: id, Path: tarPath, Checksum: built.Checksum, @@ -72,7 +72,7 @@ func TestCagraSearchEmpty(t *testing.T) { proc := testutil.NewProcessWithMPool(t, "", m) sqlproc := sqlexec.NewSqlProcess(proc) - s := NewCagraSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) + s := NewCagraSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) require.Empty(t, s.Indexes) rt := vectorindex.RuntimeConfig{Limit: 4} @@ -98,8 +98,8 @@ func TestCagraSearchTypeMismatch(t *testing.T) { idx := loadedModel(t, "type-mismatch") defer idx.Destroy() - s := NewCagraSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) - s.Indexes = []*CagraModel[float32]{idx} + s := NewCagraSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) + s.Indexes = []*CagraModel[float32, float32]{idx} rt := vectorindex.RuntimeConfig{Limit: 4} @@ -117,8 +117,8 @@ func TestCagraSearchAndSearchFloat32(t *testing.T) { idx := loadedModel(t, "search-single") defer idx.Destroy() - s := NewCagraSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) - s.Indexes = []*CagraModel[float32]{idx} + s := NewCagraSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) + s.Indexes = []*CagraModel[float32, float32]{idx} s.MultiIndex, _ = s.buildMultiIndex() data := generateTestData(testNVectors, testDim) @@ -158,8 +158,8 @@ func TestCagraSearchMultipleIndexes(t *testing.T) { idx1 := loadedModel(t, "multi-1") defer idx1.Destroy() - s := NewCagraSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) - s.Indexes = []*CagraModel[float32]{idx0, idx1} + s := NewCagraSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) + s.Indexes = []*CagraModel[float32, float32]{idx0, idx1} s.MultiIndex, _ = s.buildMultiIndex() data := generateTestData(testNVectors, testDim) @@ -212,7 +212,7 @@ func TestCagraSearchLoad(t *testing.T) { } defer func() { runSql_streaming = origStream }() - s := NewCagraSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) + s := NewCagraSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) err := s.Load(sqlproc) require.NoError(t, err) require.Equal(t, 1, len(s.Indexes)) diff --git a/pkg/vectorindex/cagra/sync.go b/pkg/vectorindex/cagra/sync.go index 4fbbc619fb652..cd1e338f1f869 100644 --- a/pkg/vectorindex/cagra/sync.go +++ b/pkg/vectorindex/cagra/sync.go @@ -50,6 +50,8 @@ import ( "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/util" + "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/pb/plan" @@ -74,6 +76,7 @@ type CagraSync struct { activeIndexId string dim int + vecBytesPerRow int // dim * base element size (4*dim for f32, 2*dim for f16) includeBytesPerRow int colMetaJSON string @@ -98,11 +101,19 @@ func NewCagraSync( idxname string, idxdefs []*plan.IndexDef, dimension int32, + baseType types.T, colMetaJSON string, ) (*CagraSync, error) { if dimension <= 0 { return nil, moerr.NewInternalErrorNoCtx("CagraSync: invalid dimension") } + // CDC records carry the vector as raw native base-type bytes: 2*dim for a + // vecf16 base, 4*dim otherwise. Must match the iscp writer's encode width + // and the search-side replayEventChunks[B] read width. + elemSize := 4 + if baseType == types.T_array_float16 { + elemSize = 2 + } var idxtblcfg vectorindex.IndexTableConfig idxtblcfg.DbName = db @@ -134,6 +145,7 @@ func NewCagraSync( tblcfg: idxtblcfg, idxname: idxname, dim: int(dimension), + vecBytesPerRow: int(dimension) * elemSize, includeBytesPerRow: includeBytesPerRow, colMetaJSON: colMetaJSON, activeIndexId: vectorindex.CdcTailId, @@ -226,7 +238,7 @@ func (s *CagraSync) AppendRecords(_ *sqlexec.SqlProcess, recordBytes []byte) err n = 9 // op (1) + pkid (8) case cuvscdc.CdcOpInsert, cuvscdc.CdcOpUpsert: // UPSERT shares INSERT's payload shape; only the op byte differs. - n = 9 + 4*s.dim + s.includeBytesPerRow + n = 9 + s.vecBytesPerRow + s.includeBytesPerRow default: return moerr.NewInternalErrorNoCtx(fmt.Sprintf( "CagraSync.AppendRecords: unknown op %d at offset %d", op, pos)) @@ -263,7 +275,10 @@ func (s *CagraSync) appendRecord(op cuvscdc.CdcOp, pkid int64, vec []float32, in } } before := len(s.pendingRecords) - out, err := cuvscdc.EncodeEventRecord(s.pendingRecords, op, pkid, vec, include, s.dim, s.includeBytesPerRow) + // This synchronous VectorIndexCdc[float32] path is f32-only (vec is + // []float32). vecf16 ongoing ingestion flows through the iscp writer → + // AppendRecords byte path instead, which honors s.vecBytesPerRow. + out, err := cuvscdc.EncodeEventRecord(s.pendingRecords, op, pkid, util.UnsafeSliceToBytes(vec), include, 4*s.dim, s.includeBytesPerRow) if err != nil { return err } diff --git a/pkg/vectorindex/cagra/sync_test.go b/pkg/vectorindex/cagra/sync_test.go index d8d52f70aa09c..7df662fb5f44f 100644 --- a/pkg/vectorindex/cagra/sync_test.go +++ b/pkg/vectorindex/cagra/sync_test.go @@ -24,6 +24,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" @@ -143,7 +144,7 @@ func TestCagraSync_Update_AllInsert(t *testing.T) { defer rec.install(t)() s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) require.Equal(t, vectorindex.CdcTailId, s.activeIndexId) @@ -164,7 +165,7 @@ func TestCagraSync_Update_AllInsert(t *testing.T) { // Round-trip: replay the persisted chunks and expect 2 overflow rows, no // deletes. - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 0) require.NoError(t, err) require.Empty(t, state.Deleted) require.Len(t, state.Overflow, 2) @@ -185,7 +186,7 @@ func TestCagraSync_Update_DeleteAndInsert(t *testing.T) { defer rec.install(t)() s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -202,7 +203,7 @@ func TestCagraSync_Update_DeleteAndInsert(t *testing.T) { // chunk_id == 7 (nextChunkId mock). require.Contains(t, rec.statements[0], "'cdc_tail', 7,") - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 7), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 7), 16, 0) require.NoError(t, err) require.Equal(t, []int64{42}, state.Deleted) require.Len(t, state.Overflow, 1) @@ -223,7 +224,7 @@ func TestCagraSync_Update_DeleteInsertDelete(t *testing.T) { defer rec.install(t)() s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -239,7 +240,7 @@ func TestCagraSync_Update_DeleteInsertDelete(t *testing.T) { require.NoError(t, s.Save(sqlproc)) - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 0) require.NoError(t, err) require.Equal(t, []int64{1}, state.Deleted, "final state must have pkid=1 deleted (last event was DELETE)") @@ -260,7 +261,7 @@ func TestCagraSync_Update_DeleteIdempotent(t *testing.T) { defer rec.install(t)() s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -275,7 +276,7 @@ func TestCagraSync_Update_DeleteIdempotent(t *testing.T) { require.NoError(t, s.Save(sqlproc)) - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 0) require.NoError(t, err) require.ElementsMatch(t, []int64{5, 7}, state.Deleted) } @@ -292,7 +293,7 @@ func TestCagraSync_Update_Upsert(t *testing.T) { defer rec.install(t)() s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -306,13 +307,13 @@ func TestCagraSync_Update_Upsert(t *testing.T) { "INSERT + UPSERT → 2 records (UPSERT is a single op, not DELETE+INSERT)") require.NoError(t, s.Save(sqlproc)) - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 0) require.NoError(t, err) require.ElementsMatch(t, []int64{100}, state.Deleted, "UPSERT marks pkid in deleted (filters any pre-rebuild main-index entry)") require.Len(t, state.Overflow, 1) require.Equal(t, int64(100), state.Overflow[0].Pkid) - require.Equal(t, []float32{9, 9, 9, 9}, state.Overflow[0].Vec, + require.Equal(t, []float32{9, 9, 9, 9}, util.UnsafeSliceCast[float32](state.Overflow[0].Vec), "UPSERT wrote the latest vec; replay surfaces it") } @@ -324,7 +325,7 @@ func TestCagraSync_Update_DimMismatch(t *testing.T) { sqlproc := sqlexec.NewSqlProcess(proc) s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -355,7 +356,7 @@ func TestCagraSync_Update_WithIncludeBytes(t *testing.T) { require.Equal(t, 9, expectedIBPR) s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, colMetaJSON) + idxdefs("__meta", "__storage"), 4, types.T_array_float32, colMetaJSON) require.NoError(t, err) require.Equal(t, 9, s.includeBytesPerRow) @@ -369,7 +370,7 @@ func TestCagraSync_Update_WithIncludeBytes(t *testing.T) { require.NoError(t, s.Update(sqlproc, cdc)) require.NoError(t, s.Save(sqlproc)) - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 9) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 9) require.NoError(t, err) require.Len(t, state.Overflow, 1) require.Equal(t, include, state.Overflow[0].Include) @@ -401,7 +402,7 @@ func TestCagraSync_Update_NoOpSaveSkipsSql(t *testing.T) { defer func() { runSql = origRun }() s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{} @@ -428,7 +429,7 @@ func TestCagraSync_NewSync_Stateless(t *testing.T) { defer func() { runSql = origRun }() s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) require.Equal(t, vectorindex.CdcTailId, s.activeIndexId) require.Equal(t, 0, called) @@ -445,7 +446,7 @@ func TestCagraSync_RunOnce(t *testing.T) { defer rec.install(t)() s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -488,7 +489,7 @@ func TestCagraSync_MultiFlush(t *testing.T) { defer rec.install(t)() s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) flush1 := &vectorindex.VectorIndexCdc[float32]{ diff --git a/pkg/vectorindex/cuvs/cdc.go b/pkg/vectorindex/cuvs/cdc.go index 30533f0789adb..ef9b59369c2a8 100644 --- a/pkg/vectorindex/cuvs/cdc.go +++ b/pkg/vectorindex/cuvs/cdc.go @@ -84,6 +84,12 @@ const ( cdcFrameOverhead = cdcHeaderSize + cdcFooterSize // 44 bytes, ex. header section ) +// CdcHeaderSize is the fixed frame header length — the minimum prefix CdcFrameLen +// needs. Exported so a streaming reader (e.g. the bm25 index's file-based tail +// loader) can read exactly the header to learn a frame's total length before +// reading it. +const CdcHeaderSize = cdcHeaderSize + // FrameCdcChunk wraps the given record bytes (plus an optional header, // typically colMetaJSON) into the on-wire chunk frame described above. // nInserts / nDeletes / nUpserts are the per-op record counts contained @@ -158,6 +164,26 @@ func UnframeCdcChunk(framed []byte) (records, header []byte, nInserts, nDeletes, return records, header, nInserts, nDeletes, nUpserts, nil } +// CdcFrameLen returns the total on-wire byte length of the CDC chunk frame whose +// leading bytes are `prefix` (at least cdcHeaderSize bytes — the frame header is +// self-describing). It lets a consumer that STORES one frame split across several +// fixed-size storage chunks reassemble it: read the first stored chunk's header, +// get the total length, then read that many bytes across the following chunks. +// cuVS's own tail packs small records so a frame is always <= one chunk; the bm25 +// retrieval index stores indivisible segment blobs that can exceed a chunk, hence +// this helper. Validates the start magic; does not require the full frame. +func CdcFrameLen(prefix []byte) (int, error) { + if len(prefix) < cdcHeaderSize { + return 0, moerr.NewInternalErrorNoCtxf("CdcFrameLen: prefix too short (%d < %d)", len(prefix), cdcHeaderSize) + } + if got := binary.LittleEndian.Uint32(prefix[0:4]); got != cdcChunkMagic { + return 0, moerr.NewInternalErrorNoCtxf("CdcFrameLen: bad start magic 0x%08x (want 0x%08x)", got, cdcChunkMagic) + } + plen := binary.LittleEndian.Uint32(prefix[20:24]) + hlen := binary.LittleEndian.Uint32(prefix[24:28]) + return cdcFrameOverhead + int(hlen) + int(plen), nil +} + // CDC event log helpers shared by CAGRA and IVF-PQ. // // CDC writes never touch the model tar (tag=0). They append op-tagged event @@ -207,24 +233,29 @@ const ( CdcOpUpsert CdcOp = 2 ) -// CdcEventRecord is the decoded form of one tag=1 record. +// CdcEventRecord is the decoded form of one tag=1 record. Vec holds the raw +// little-endian vector bytes in the index's native base element type (4 bytes +// per element for vecf32, 2 for a vecf16) — the codec is element-type-agnostic; +// the GPU layer reinterprets these bytes to []float32 / []cuvs.Float16. type CdcEventRecord struct { Op CdcOp Pkid int64 - Vec []float32 // populated only for CdcOpInsert - Include []byte // populated only for CdcOpInsert (and only when includeBytesPerRow > 0) + Vec []byte // populated only for CdcOpInsert (vecBytesPerRow bytes) + Include []byte // populated only for CdcOpInsert (and only when includeBytesPerRow > 0) } // EncodeEventRecord appends one record to dst and returns the new slice. // vec is required iff op==CdcOpInsert; include is required iff op==CdcOpInsert -// AND includeBytesPerRow > 0. dim must be the index's dimensionality. +// AND includeBytesPerRow > 0. vec carries the row's vector as raw native +// base-type bytes; vecBytesPerRow is its expected length (dim * element size, +// e.g. 4*dim for f32, 2*dim for f16). func EncodeEventRecord( dst []byte, op CdcOp, pkid int64, - vec []float32, + vec []byte, include []byte, - dim int, + vecBytesPerRow int, includeBytesPerRow int, ) ([]byte, error) { switch op { @@ -243,11 +274,11 @@ func EncodeEventRecord( if op == CdcOpUpsert { opName = "UPSERT" } - if dim <= 0 { - return nil, moerr.NewInternalErrorNoCtxf("EncodeEventRecord: %s requires positive dim, got %d", opName, dim) + if vecBytesPerRow <= 0 { + return nil, moerr.NewInternalErrorNoCtxf("EncodeEventRecord: %s requires positive vecBytesPerRow, got %d", opName, vecBytesPerRow) } - if len(vec) != dim { - return nil, moerr.NewInternalErrorNoCtxf("EncodeEventRecord: %s vec length %d != dim %d", opName, len(vec), dim) + if len(vec) != vecBytesPerRow { + return nil, moerr.NewInternalErrorNoCtxf("EncodeEventRecord: %s vec length %d != vecBytesPerRow %d", opName, len(vec), vecBytesPerRow) } if includeBytesPerRow > 0 && len(include) != includeBytesPerRow { return nil, moerr.NewInternalErrorNoCtxf("EncodeEventRecord: %s include length %d != includeBytesPerRow %d", @@ -260,11 +291,10 @@ func EncodeEventRecord( var pk [8]byte binary.LittleEndian.PutUint64(pk[:], uint64(pkid)) dst = append(dst, pk[:]...) - var f [4]byte - for _, v := range vec { - binary.LittleEndian.PutUint32(f[:], math.Float32bits(v)) - dst = append(dst, f[:]...) - } + // vec body: raw native base-type bytes, copied verbatim. For f32 this is + // byte-identical to the legacy math.Float32bits + PutUint32 form on + // little-endian targets, so existing f32 CDC streams are unchanged. + dst = append(dst, vec...) if includeBytesPerRow > 0 { dst = append(dst, include...) } @@ -278,12 +308,10 @@ func EncodeEventRecord( // and the number of bytes consumed. Returns ok=false when src cannot start a // valid record (e.g. unknown op byte, or fewer bytes than the record needs) // — the caller treats this as the end of the stream within the chunk. -// -// Vec and Include in the returned record alias into src; copy if you need to -// retain them past the next call. +// vecBytesPerRow is the INSERT-record vector byte length (dim * element size). func DecodeEventRecord( src []byte, - dim int, + vecBytesPerRow int, includeBytesPerRow int, ) (rec CdcEventRecord, n int, ok bool) { if len(src) < 9 { @@ -296,19 +324,18 @@ func DecodeEventRecord( rec.Pkid = int64(binary.LittleEndian.Uint64(src[1:9])) return rec, 9, true case CdcOpInsert, CdcOpUpsert: - need := 9 + 4*dim + includeBytesPerRow - if dim <= 0 || includeBytesPerRow < 0 || len(src) < need { + need := 9 + vecBytesPerRow + includeBytesPerRow + if vecBytesPerRow <= 0 || includeBytesPerRow < 0 || len(src) < need { return rec, 0, false } rec.Op = op rec.Pkid = int64(binary.LittleEndian.Uint64(src[1:9])) - rec.Vec = make([]float32, dim) - for k := 0; k < dim; k++ { - rec.Vec[k] = math.Float32frombits(binary.LittleEndian.Uint32(src[9+k*4:])) - } + // vec body: raw native base-type bytes, copied verbatim. + rec.Vec = make([]byte, vecBytesPerRow) + copy(rec.Vec, src[9:9+vecBytesPerRow]) if includeBytesPerRow > 0 { rec.Include = make([]byte, includeBytesPerRow) - copy(rec.Include, src[9+4*dim:need]) + copy(rec.Include, src[9+vecBytesPerRow:need]) } return rec, need, true default: @@ -446,18 +473,21 @@ type ReplayState struct { Deleted []int64 Overflow []OverflowEntry - // ColMetaJSON is the payload of a CdcOpHeader record observed during - // replay, when one was present (small-tail emit path writes it as - // the first record of chunk_id=0). Empty otherwise. Callers that - // need the INCLUDE-column layout but have no tag=0 sub-index read + // ColMetaJSON is the INCLUDE-column metadata carried in the chunk + // frame's header section (see FrameCdcChunk / PeekColMetaJSON), not a + // record — every chunk embeds it, so it is read from the first chunk + // during replay. Empty when the index has no INCLUDE columns. Callers + // that need the INCLUDE-column layout but have no tag=0 sub-index read // this back here. ColMetaJSON string } -// OverflowEntry is one row in the brute-force overflow. +// OverflowEntry is one row in the brute-force overflow. Vec holds the raw +// native base-type bytes (no f32 widening for vecf16); the GPU layer +// reinterprets them to the base element type B. type OverflowEntry struct { Pkid int64 - Vec []float32 + Vec []byte Include []byte } @@ -482,16 +512,16 @@ func PeekColMetaJSON(chunks []EventChunk) (string, error) { } // ReplayEventLog walks the chunks (assumed sorted by chunk_id) and applies -// each record in order, returning the final (deleted, overflow) state. dim -// and includeBytesPerRow describe the INSERT record layout. Replay is O(n) -// in event count. +// each record in order, returning the final (deleted, overflow) state. +// vecBytesPerRow (dim * element size) and includeBytesPerRow describe the +// INSERT record layout. Replay is O(n) in event count. func ReplayEventLog( chunks []EventChunk, - dim int, + vecBytesPerRow int, includeBytesPerRow int, ) (ReplayState, error) { - if dim <= 0 { - return ReplayState{}, moerr.NewInternalErrorNoCtxf("ReplayEventLog: invalid dim %d", dim) + if vecBytesPerRow <= 0 { + return ReplayState{}, moerr.NewInternalErrorNoCtxf("ReplayEventLog: invalid vecBytesPerRow %d", vecBytesPerRow) } if includeBytesPerRow < 0 { return ReplayState{}, moerr.NewInternalErrorNoCtxf("ReplayEventLog: negative includeBytesPerRow %d", includeBytesPerRow) @@ -514,14 +544,14 @@ func ReplayEventLog( colMetaJSON = string(header) } for len(data) > 0 { - rec, n, ok := DecodeEventRecord(data, dim, includeBytesPerRow) + rec, n, ok := DecodeEventRecord(data, vecBytesPerRow, includeBytesPerRow) if !ok { // Frame CRC already validated the payload, so any decode // failure here is a record-level bug (encoder/decoder mismatch // on dim or includeBytesPerRow). return ReplayState{}, moerr.NewInternalErrorNoCtxf( - "ReplayEventLog: chunk_id=%d: undecodable record at offset %d (dim=%d includeBytesPerRow=%d)", - ch.ChunkId, len(ch.Data)-cdcFooterSize-len(data), dim, includeBytesPerRow) + "ReplayEventLog: chunk_id=%d: undecodable record at offset %d (vecBytesPerRow=%d includeBytesPerRow=%d)", + ch.ChunkId, len(ch.Data)-cdcFooterSize-len(data), vecBytesPerRow, includeBytesPerRow) } switch rec.Op { case CdcOpDelete: diff --git a/pkg/vectorindex/cuvs/cdc_test.go b/pkg/vectorindex/cuvs/cdc_test.go index a028a7eeb599c..b29b9457e0660 100644 --- a/pkg/vectorindex/cuvs/cdc_test.go +++ b/pkg/vectorindex/cuvs/cdc_test.go @@ -24,6 +24,7 @@ import ( "strings" "testing" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/vectorindex" ) @@ -88,7 +89,11 @@ func encodeBatch( insertIdx++ } before := len(buf) - out, err := EncodeEventRecord(buf, op, pkids[i], v, inc, dim, includeBytesPerRow) + var vb []byte + if v != nil { + vb = util.UnsafeSliceToBytes(v) + } + out, err := EncodeEventRecord(buf, op, pkids[i], vb, inc, 4*dim, includeBytesPerRow) if err != nil { t.Fatalf("EncodeEventRecord(%v, pkid=%d): %v", op, pkids[i], err) } @@ -101,14 +106,14 @@ func encodeBatch( // TestEncodeDecodeEventRecord_Delete: round-trip a DELETE record. func TestEncodeDecodeEventRecord_Delete(t *testing.T) { for _, pkid := range []int64{1, -7, math.MaxInt64, math.MinInt64, 0} { - buf, err := EncodeEventRecord(nil, CdcOpDelete, pkid, nil, nil, 4, 0) + buf, err := EncodeEventRecord(nil, CdcOpDelete, pkid, nil, nil, 16, 0) if err != nil { t.Fatalf("encode pkid=%d: %v", pkid, err) } if len(buf) != 9 { t.Fatalf("DELETE record should be 9 bytes, got %d", len(buf)) } - rec, n, ok := DecodeEventRecord(buf, 4, 0) + rec, n, ok := DecodeEventRecord(buf, 16, 0) if !ok || n != 9 { t.Fatalf("decode failed: ok=%v n=%d", ok, n) } @@ -123,7 +128,7 @@ func TestEncodeDecodeEventRecord_Insert(t *testing.T) { dim := 3 pkid := int64(42) vec := []float32{1.5, -2.25, math.MaxFloat32} - buf, err := EncodeEventRecord(nil, CdcOpInsert, pkid, vec, nil, dim, 0) + buf, err := EncodeEventRecord(nil, CdcOpInsert, pkid, util.UnsafeSliceToBytes(vec), nil, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -131,16 +136,17 @@ func TestEncodeDecodeEventRecord_Insert(t *testing.T) { if len(buf) != want { t.Fatalf("INSERT record len %d, want %d", len(buf), want) } - rec, n, ok := DecodeEventRecord(buf, dim, 0) + rec, n, ok := DecodeEventRecord(buf, 4*dim, 0) if !ok || n != want { t.Fatalf("decode: ok=%v n=%d", ok, n) } if rec.Op != CdcOpInsert || rec.Pkid != pkid { t.Fatalf("op/pkid mismatch") } + gotVec := util.UnsafeSliceCast[float32](rec.Vec) for i, v := range vec { - if math.Float32bits(rec.Vec[i]) != math.Float32bits(v) { - t.Fatalf("vec[%d]: got %v want %v", i, rec.Vec[i], v) + if math.Float32bits(gotVec[i]) != math.Float32bits(v) { + t.Fatalf("vec[%d]: got %v want %v", i, gotVec[i], v) } } if len(rec.Include) != 0 { @@ -156,7 +162,7 @@ func TestEncodeDecodeEventRecord_InsertWithInclude(t *testing.T) { binary.LittleEndian.PutUint32(include[0:4], 0xdeadbeef) binary.LittleEndian.PutUint64(include[4:12], 0x1122334455667788) include[12] = 0x02 - buf, err := EncodeEventRecord(nil, CdcOpInsert, 1, []float32{0.1, 0.2}, include, dim, includeBytesPerRow) + buf, err := EncodeEventRecord(nil, CdcOpInsert, 1, util.UnsafeSliceToBytes([]float32{0.1, 0.2}), include, 4*dim, includeBytesPerRow) if err != nil { t.Fatal(err) } @@ -164,7 +170,7 @@ func TestEncodeDecodeEventRecord_InsertWithInclude(t *testing.T) { if len(buf) != want { t.Fatalf("len %d, want %d", len(buf), want) } - rec, n, ok := DecodeEventRecord(buf, dim, includeBytesPerRow) + rec, n, ok := DecodeEventRecord(buf, 4*dim, includeBytesPerRow) if !ok || n != want { t.Fatalf("decode: ok=%v n=%d", ok, n) } @@ -181,19 +187,19 @@ func TestEncodeDecodeEventRecord_InsertWithInclude(t *testing.T) { // TestEncodeEventRecord_Rejects: encoder rejects malformed inputs. func TestEncodeEventRecord_Rejects(t *testing.T) { // DELETE with vec. - if _, err := EncodeEventRecord(nil, CdcOpDelete, 1, []float32{1}, nil, 1, 0); err == nil { + if _, err := EncodeEventRecord(nil, CdcOpDelete, 1, util.UnsafeSliceToBytes([]float32{1}), nil, 4, 0); err == nil { t.Fatal("expected error on DELETE with vec") } // INSERT with wrong dim. - if _, err := EncodeEventRecord(nil, CdcOpInsert, 1, []float32{1, 2}, nil, 4, 0); err == nil { + if _, err := EncodeEventRecord(nil, CdcOpInsert, 1, util.UnsafeSliceToBytes([]float32{1, 2}), nil, 4, 0); err == nil { t.Fatal("expected error on dim mismatch") } // INSERT with include but includeBytesPerRow=0. - if _, err := EncodeEventRecord(nil, CdcOpInsert, 1, []float32{1}, []byte{0xff}, 1, 0); err == nil { + if _, err := EncodeEventRecord(nil, CdcOpInsert, 1, util.UnsafeSliceToBytes([]float32{1}), []byte{0xff}, 4, 0); err == nil { t.Fatal("expected error on extraneous include bytes") } // Unknown op. - if _, err := EncodeEventRecord(nil, CdcOp(99), 1, nil, nil, 1, 0); err == nil { + if _, err := EncodeEventRecord(nil, CdcOp(99), 1, nil, nil, 4, 0); err == nil { t.Fatal("expected error on unknown op") } } @@ -203,21 +209,21 @@ func TestEncodeEventRecord_Rejects(t *testing.T) { // no-bytes-left case and an unknown op byte at the boundary). func TestDecodeEventRecord_StopsAtPad(t *testing.T) { // Empty buffer: decoder reports not-ok. - if _, _, ok := DecodeEventRecord(nil, 4, 0); ok { + if _, _, ok := DecodeEventRecord(nil, 16, 0); ok { t.Fatal("decoder should not accept empty input") } // 7 bytes: not enough for any record. - if _, _, ok := DecodeEventRecord(make([]byte, 7), 4, 0); ok { + if _, _, ok := DecodeEventRecord(make([]byte, 7), 16, 0); ok { t.Fatal("decoder should not accept 7 bytes") } // Bogus op byte. bogus := []byte{42, 0, 0, 0, 0, 0, 0, 0, 0} - if _, _, ok := DecodeEventRecord(bogus, 4, 0); ok { + if _, _, ok := DecodeEventRecord(bogus, 16, 0); ok { t.Fatal("decoder should reject unknown op byte") } // INSERT op but truncated payload. short := []byte{byte(CdcOpInsert), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} // missing vec bytes - if _, _, ok := DecodeEventRecord(short, 4, 0); ok { + if _, _, ok := DecodeEventRecord(short, 16, 0); ok { t.Fatal("decoder should reject truncated INSERT") } } @@ -259,7 +265,7 @@ func TestCdcAppendEventsSql_DeleteOnly(t *testing.T) { } // Round-trip via the loader path. chunks := []EventChunk{{ChunkId: 0, Data: blobs[0]}} - state, err := ReplayEventLog(chunks, 4, 0) + state, err := ReplayEventLog(chunks, 16, 0) if err != nil { t.Fatal(err) } @@ -290,7 +296,7 @@ func TestCdcAppendEventsSql_InsertOnly(t *testing.T) { } blobs := extractUnhexBlobs(t, sqls[0]) chunks := []EventChunk{{ChunkId: 0, Data: blobs[0]}} - state, err := ReplayEventLog(chunks, dim, 0) + state, err := ReplayEventLog(chunks, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -301,9 +307,10 @@ func TestCdcAppendEventsSql_InsertOnly(t *testing.T) { if e.Pkid != pkids[i] { t.Fatalf("overflow[%d].Pkid: got %d want %d", i, e.Pkid, pkids[i]) } + gotVec := util.UnsafeSliceCast[float32](e.Vec) for k, v := range vecs[i] { - if math.Float32bits(e.Vec[k]) != math.Float32bits(v) { - t.Fatalf("overflow[%d].Vec[%d]: got %v want %v", i, k, e.Vec[k], v) + if math.Float32bits(gotVec[k]) != math.Float32bits(v) { + t.Fatalf("overflow[%d].Vec[%d]: got %v want %v", i, k, gotVec[k], v) } } } @@ -327,7 +334,7 @@ func TestCdcAppendEventsSql_Mixed(t *testing.T) { } blobs := extractUnhexBlobs(t, sqls[0]) chunks := []EventChunk{{ChunkId: 0, Data: blobs[0]}} - state, err := ReplayEventLog(chunks, dim, 0) + state, err := ReplayEventLog(chunks, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -385,7 +392,7 @@ func TestCdcAppendEventsSql_ChunkPacking(t *testing.T) { {ChunkId: 5, Data: blobs[0]}, {ChunkId: 6, Data: blobs[1]}, } - state, err := ReplayEventLog(chunks, dim, 0) + state, err := ReplayEventLog(chunks, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -408,7 +415,7 @@ func TestReplayEventLog_DeleteInsertDelete(t *testing.T) { buf, _ := encodeBatch(t, dim, 0, ops, pkids, vecs, nil) chunks := []EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}} - state, err := ReplayEventLog(chunks, dim, 0) + state, err := ReplayEventLog(chunks, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -440,7 +447,7 @@ func TestReplayEventLog_InsertDeleteInsert(t *testing.T) { vecs := [][]float32{{1, 1}, {9, 9}} buf, _ := encodeBatch(t, dim, 0, ops, pkids, vecs, nil) - state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, dim, 0) + state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -451,8 +458,9 @@ func TestReplayEventLog_InsertDeleteInsert(t *testing.T) { t.Fatalf("overflow: got %v want one entry pkid=7", state.Overflow) } // Last INSERT's vec wins for the overflow entry. - if state.Overflow[0].Vec[0] != 9 || state.Overflow[0].Vec[1] != 9 { - t.Fatalf("vec: got %v want [9 9]", state.Overflow[0].Vec) + gotVec := util.UnsafeSliceCast[float32](state.Overflow[0].Vec) + if gotVec[0] != 9 || gotVec[1] != 9 { + t.Fatalf("vec: got %v want [9 9]", gotVec) } } @@ -465,7 +473,7 @@ func TestReplayEventLog_UpsertSingle(t *testing.T) { buf, _ := encodeBatch(t, dim, 0, []CdcOp{CdcOpUpsert}, []int64{7}, [][]float32{{1, 1}}, nil) - state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, dim, 0) + state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -475,8 +483,9 @@ func TestReplayEventLog_UpsertSingle(t *testing.T) { if len(state.Overflow) != 1 || state.Overflow[0].Pkid != 7 { t.Fatalf("overflow: got %v want one entry pkid=7 (UPSERT writes new vec to brute-force overflow)", state.Overflow) } - if state.Overflow[0].Vec[0] != 1 || state.Overflow[0].Vec[1] != 1 { - t.Fatalf("vec: got %v want [1 1]", state.Overflow[0].Vec) + gotVec := util.UnsafeSliceCast[float32](state.Overflow[0].Vec) + if gotVec[0] != 1 || gotVec[1] != 1 { + t.Fatalf("vec: got %v want [1 1]", gotVec) } } @@ -489,7 +498,7 @@ func TestReplayEventLog_UpsertThenDelete(t *testing.T) { buf, _ := encodeBatch(t, dim, 0, []CdcOp{CdcOpUpsert, CdcOpDelete}, []int64{7, 7}, [][]float32{{1, 1}}, nil) - state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, dim, 0) + state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -512,7 +521,7 @@ func TestReplayEventLog_UpsertReplayIdempotent(t *testing.T) { []int64{7, 7, 7}, [][]float32{{1, 1}, {1, 1}, {1, 1}}, nil) - state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, dim, 0) + state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -522,8 +531,9 @@ func TestReplayEventLog_UpsertReplayIdempotent(t *testing.T) { if len(state.Overflow) != 1 || state.Overflow[0].Pkid != 7 { t.Fatalf("overflow: got %v want one entry pkid=7", state.Overflow) } - if state.Overflow[0].Vec[0] != 1 || state.Overflow[0].Vec[1] != 1 { - t.Fatalf("vec: got %v want [1 1]", state.Overflow[0].Vec) + gotVec := util.UnsafeSliceCast[float32](state.Overflow[0].Vec) + if gotVec[0] != 1 || gotVec[1] != 1 { + t.Fatalf("vec: got %v want [1 1]", gotVec) } } @@ -537,7 +547,7 @@ func TestReplayEventLog_InsertAfterDeleteDoesNotUnfilter(t *testing.T) { buf, _ := encodeBatch(t, dim, 0, []CdcOp{CdcOpDelete, CdcOpInsert}, []int64{7, 7}, [][]float32{{9, 9}}, nil) - state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, dim, 0) + state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -547,8 +557,9 @@ func TestReplayEventLog_InsertAfterDeleteDoesNotUnfilter(t *testing.T) { if len(state.Overflow) != 1 || state.Overflow[0].Pkid != 7 { t.Fatalf("overflow: got %v want one entry pkid=7", state.Overflow) } - if state.Overflow[0].Vec[0] != 9 || state.Overflow[0].Vec[1] != 9 { - t.Fatalf("vec: got %v want [9 9]", state.Overflow[0].Vec) + gotVec := util.UnsafeSliceCast[float32](state.Overflow[0].Vec) + if gotVec[0] != 9 || gotVec[1] != 9 { + t.Fatalf("vec: got %v want [9 9]", gotVec) } } @@ -570,7 +581,7 @@ func TestReplayEventLog_MultiChunk(t *testing.T) { {ChunkId: 0, Data: FrameCdcChunk(buf0, nil, 0, 0, 0)}, } SortChunks(chunks) - state, err := ReplayEventLog(chunks, dim, 0) + state, err := ReplayEventLog(chunks, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -596,7 +607,7 @@ func TestReplayEventLog_WithInclude(t *testing.T) { [][]float32{{1, 2}}, [][]byte{include}, ) - state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, dim, includeBytesPerRow) + state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, 4*dim, includeBytesPerRow) if err != nil { t.Fatal(err) } @@ -618,7 +629,7 @@ func TestReplayEventLog_CapturesColMetaJSON(t *testing.T) { buf, _ := encodeBatch(t, dim, 0, []CdcOp{CdcOpInsert}, []int64{1}, [][]float32{{1, 2}}, nil) chunks := []EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, []byte(colMetaJSON), 0, 0, 0)}} - state, err := ReplayEventLog(chunks, dim, 0) + state, err := ReplayEventLog(chunks, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -637,7 +648,7 @@ func TestReplayEventLog_NoColMetaJSON(t *testing.T) { buf, _ := encodeBatch(t, dim, 0, []CdcOp{CdcOpInsert}, []int64{1}, [][]float32{{1, 2}}, nil) chunks := []EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}} - state, err := ReplayEventLog(chunks, dim, 0) + state, err := ReplayEventLog(chunks, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -682,14 +693,14 @@ func TestReplayEventLog_RejectsCorruptFrame(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - _, err := ReplayEventLog([]EventChunk{{ChunkId: 7, Data: tc.mut(good)}}, dim, 0) + _, err := ReplayEventLog([]EventChunk{{ChunkId: 7, Data: tc.mut(good)}}, 4*dim, 0) if err == nil { t.Fatalf("expected error for %s, got nil", tc.name) } }) } // Sanity: the unmodified frame round-trips. - state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: good}}, dim, 0) + state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: good}}, 4*dim, 0) if err != nil { t.Fatal(err) } diff --git a/pkg/vectorindex/cuvs/small_tail.go b/pkg/vectorindex/cuvs/small_tail.go index 160d1acdc4111..9c04396e0c486 100644 --- a/pkg/vectorindex/cuvs/small_tail.go +++ b/pkg/vectorindex/cuvs/small_tail.go @@ -28,7 +28,7 @@ import ( // columns. type PendingRecord struct { Pkid int64 - Vec []float32 + Vec []byte // raw native base-type bytes (vecBytesPerRow) Include []byte } @@ -55,7 +55,7 @@ type PendingRecord struct { func SaveSmallTailAsCdc( tblcfg vectorindex.IndexTableConfig, rows []PendingRecord, - dim int, + vecBytesPerRow int, includeBytesPerRow int, colMetaJSON string, ) ([]string, error) { @@ -63,16 +63,16 @@ func SaveSmallTailAsCdc( return nil, nil } - // Pre-size the buffer: 9 (op + pkid) + 4*dim + ibpr bytes per + // Pre-size the buffer: 9 (op + pkid) + vecBytesPerRow + ibpr bytes per // INSERT record. Avoids ~len(rows) reallocs in EncodeEventRecord. - perRow := 9 + 4*dim + includeBytesPerRow + perRow := 9 + vecBytesPerRow + includeBytesPerRow records := make([]byte, 0, perRow*len(rows)) sizes := make([]int, 0, len(rows)) for _, r := range rows { before := len(records) out, err := EncodeEventRecord(records, CdcOpInsert, - r.Pkid, r.Vec, r.Include, dim, includeBytesPerRow) + r.Pkid, r.Vec, r.Include, vecBytesPerRow, includeBytesPerRow) if err != nil { return nil, err } diff --git a/pkg/vectorindex/cuvs/small_tail_test.go b/pkg/vectorindex/cuvs/small_tail_test.go index 6b96b3eefe7bd..a05454c4ba21a 100644 --- a/pkg/vectorindex/cuvs/small_tail_test.go +++ b/pkg/vectorindex/cuvs/small_tail_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/vectorindex" ) @@ -46,12 +47,12 @@ func TestSaveSmallTailAsCdc_Empty(t *testing.T) { func TestSaveSmallTailAsCdc_NoInclude(t *testing.T) { const dim = 3 rows := []PendingRecord{ - {Pkid: 1, Vec: []float32{1, 2, 3}}, - {Pkid: 2, Vec: []float32{4, 5, 6}}, - {Pkid: -3, Vec: []float32{math.MaxFloat32, 0, -1}}, + {Pkid: 1, Vec: util.UnsafeSliceToBytes([]float32{1, 2, 3})}, + {Pkid: 2, Vec: util.UnsafeSliceToBytes([]float32{4, 5, 6})}, + {Pkid: -3, Vec: util.UnsafeSliceToBytes([]float32{math.MaxFloat32, 0, -1})}, } - sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, dim, 0, "") + sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 4*dim, 0, "") require.NoError(t, err) require.NotEmpty(t, sqls) @@ -69,7 +70,7 @@ func TestSaveSmallTailAsCdc_NoInclude(t *testing.T) { require.NoError(t, err) pos := 0 for pos < len(records) { - rec, n, ok := DecodeEventRecord(records[pos:], dim, 0) + rec, n, ok := DecodeEventRecord(records[pos:], 4*dim, 0) require.True(t, ok) got = append(got, rec) pos += n @@ -80,9 +81,11 @@ func TestSaveSmallTailAsCdc_NoInclude(t *testing.T) { for i, in := range rows { require.Equal(t, CdcOpInsert, got[i].Op) require.Equal(t, in.Pkid, got[i].Pkid) - require.Len(t, got[i].Vec, dim) - for j, v := range in.Vec { - require.Equal(t, math.Float32bits(v), math.Float32bits(got[i].Vec[j]), + require.Len(t, got[i].Vec, 4*dim) + inVec := util.UnsafeSliceCast[float32](in.Vec) + gotVec := util.UnsafeSliceCast[float32](got[i].Vec) + for j, v := range inVec { + require.Equal(t, math.Float32bits(v), math.Float32bits(gotVec[j]), "row %d vec[%d] mismatch", i, j) } } @@ -94,13 +97,13 @@ func TestSaveSmallTailAsCdc_WithInclude(t *testing.T) { const dim = 2 const ibpr = 8 // one int64-shaped INCLUDE col + zero-mask byte rounded rows := []PendingRecord{ - {Pkid: 10, Vec: []float32{0.1, 0.2}, Include: []byte{1, 2, 3, 4, 5, 6, 7, 8}}, - {Pkid: 11, Vec: []float32{0.3, 0.4}, Include: []byte{9, 10, 11, 12, 13, 14, 15, 16}}, + {Pkid: 10, Vec: util.UnsafeSliceToBytes([]float32{0.1, 0.2}), Include: []byte{1, 2, 3, 4, 5, 6, 7, 8}}, + {Pkid: 11, Vec: util.UnsafeSliceToBytes([]float32{0.3, 0.4}), Include: []byte{9, 10, 11, 12, 13, 14, 15, 16}}, } // Empty colMetaJSON to keep this test focused on tag=1 INSERT // round-trip; the header-emission case has its own test below. - sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, dim, ibpr, "") + sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 4*dim, ibpr, "") require.NoError(t, err) require.NotEmpty(t, sqls) @@ -116,7 +119,7 @@ func TestSaveSmallTailAsCdc_WithInclude(t *testing.T) { require.NoError(t, err) pos := 0 for pos < len(records) { - rec, n, ok := DecodeEventRecord(records[pos:], dim, ibpr) + rec, n, ok := DecodeEventRecord(records[pos:], 4*dim, ibpr) require.True(t, ok) got = append(got, rec) pos += n @@ -137,9 +140,9 @@ func TestSaveSmallTailAsCdc_IncludeMismatchErrors(t *testing.T) { const dim = 2 const ibpr = 8 rows := []PendingRecord{ - {Pkid: 1, Vec: []float32{0.1, 0.2}, Include: []byte{1, 2, 3}}, // wrong length + {Pkid: 1, Vec: util.UnsafeSliceToBytes([]float32{0.1, 0.2}), Include: []byte{1, 2, 3}}, // wrong length } - _, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, dim, ibpr, "") + _, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 4*dim, ibpr, "") require.Error(t, err) } @@ -147,8 +150,8 @@ func TestSaveSmallTailAsCdc_IncludeMismatchErrors(t *testing.T) { // must be the well-known CdcTailId sentinel so the search-side // replay finds it. func TestSaveSmallTailAsCdc_UsesCdcTailId(t *testing.T) { - rows := []PendingRecord{{Pkid: 1, Vec: []float32{1, 2, 3, 4}}} - sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 4, 0, "") + rows := []PendingRecord{{Pkid: 1, Vec: util.UnsafeSliceToBytes([]float32{1, 2, 3, 4})}} + sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 16, 0, "") require.NoError(t, err) require.NotEmpty(t, sqls) require.Contains(t, sqls[0], "'"+vectorindex.CdcTailId+"'") @@ -163,10 +166,10 @@ func TestSaveSmallTailAsCdc_EmbedsColMetaInEveryChunk(t *testing.T) { const ibpr = 8 colMetaJSON := `[{"name":"a","type":1}]` rows := []PendingRecord{ - {Pkid: 1, Vec: []float32{0.1, 0.2}, Include: []byte{1, 2, 3, 4, 5, 6, 7, 8}}, - {Pkid: 2, Vec: []float32{0.3, 0.4}, Include: []byte{9, 10, 11, 12, 13, 14, 15, 16}}, + {Pkid: 1, Vec: util.UnsafeSliceToBytes([]float32{0.1, 0.2}), Include: []byte{1, 2, 3, 4, 5, 6, 7, 8}}, + {Pkid: 2, Vec: util.UnsafeSliceToBytes([]float32{0.3, 0.4}), Include: []byte{9, 10, 11, 12, 13, 14, 15, 16}}, } - sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, dim, ibpr, colMetaJSON) + sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 4*dim, ibpr, colMetaJSON) require.NoError(t, err) require.NotEmpty(t, sqls) @@ -183,7 +186,7 @@ func TestSaveSmallTailAsCdc_EmbedsColMetaInEveryChunk(t *testing.T) { "every chunk's frame header section must carry colMetaJSON") // Records are still pure Delete/Insert event ops (no special // header record in the records section). - rec, _, ok := DecodeEventRecord(records, dim, ibpr) + rec, _, ok := DecodeEventRecord(records, 4*dim, ibpr) require.True(t, ok) require.Equal(t, CdcOpInsert, rec.Op) } @@ -193,11 +196,11 @@ func TestSaveSmallTailAsCdc_EmbedsColMetaInEveryChunk(t *testing.T) { // recovers the colMetaJSON from the chunk frame header. func TestPeekColMetaJSON_RoundTrip(t *testing.T) { colMetaJSON := `[{"name":"a","type":1},{"name":"b","type":2}]` - rows := []PendingRecord{{Pkid: 1, Vec: []float32{1, 2}}} + rows := []PendingRecord{{Pkid: 1, Vec: util.UnsafeSliceToBytes([]float32{1, 2})}} // Note: ibpr=0 here because rows[0].Include is empty; the embedded // colMetaJSON is for the search side's INCLUDE-column wiring, not // the encode-time layout in this contrived test. - sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 2, 0, colMetaJSON) + sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 8, 0, colMetaJSON) require.NoError(t, err) require.NotEmpty(t, sqls) @@ -215,8 +218,8 @@ func TestPeekColMetaJSON_RoundTrip(t *testing.T) { // TestPeekColMetaJSON_NoHeader: when colMetaJSON is empty the chunk's // frame header section is empty too — PeekColMetaJSON returns "". func TestPeekColMetaJSON_NoHeader(t *testing.T) { - rows := []PendingRecord{{Pkid: 1, Vec: []float32{1, 2, 3, 4}}} - sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 4, 0, "") + rows := []PendingRecord{{Pkid: 1, Vec: util.UnsafeSliceToBytes([]float32{1, 2, 3, 4})}} + sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 16, 0, "") require.NoError(t, err) re := regexp.MustCompile(`unhex\('([0-9a-fA-F]*)'\)`) diff --git a/pkg/vectorindex/hnsw/plugin/compile/compile.go b/pkg/vectorindex/hnsw/plugin/compile/compile.go index 37e6fe4979305..c13a5a1a0414c 100644 --- a/pkg/vectorindex/hnsw/plugin/compile/compile.go +++ b/pkg/vectorindex/hnsw/plugin/compile/compile.go @@ -171,7 +171,7 @@ func (Hooks) handleCreate(ctx compileplugin.CompileContext, indexDefs map[string // HandleReindex: same code path as create, but honors forceSync so an // ALTER REINDEX … FORCE_SYNC (e.g. restore's RestoreTable) rebuilds an // always-async HNSW index synchronously instead of deferring to CDC. -func (h Hooks) HandleReindex(ctx compileplugin.CompileContext, indexDefs map[string]*plan.IndexDef, forceSync bool) error { +func (h Hooks) HandleReindex(ctx compileplugin.CompileContext, indexDefs map[string]*plan.IndexDef, forceSync bool, _ bool) error { return h.handleCreate(ctx, indexDefs, forceSync) } @@ -200,6 +200,12 @@ func (Hooks) ValidateReindexParams(old map[string]string, alter compileplugin.Re // not covered there. func (Hooks) HandleDropIndex(_ compileplugin.CompileContext, defs map[string]*plan.IndexDef) error { logutil.Infof("[plugin] hnsw HandleDropIndex: defs=%d", len(defs)) + // Evict the cached search index so its resources are freed NOW, rather than + // lingering until the 5-min VectorIndexCacheTTL. Mirrors the create-side + // cache.Cache.Remove(storageDef.IndexTableName). + if storageDef, ok := defs[catalog.Hnsw_TblType_Storage]; ok { + cache.Cache.Remove(storageDef.IndexTableName) + } return nil } diff --git a/pkg/vectorindex/hnsw/plugin/compile/compile_smoke_test.go b/pkg/vectorindex/hnsw/plugin/compile/compile_smoke_test.go index 9afb7aa431f51..f6b4a2ed915ea 100644 --- a/pkg/vectorindex/hnsw/plugin/compile/compile_smoke_test.go +++ b/pkg/vectorindex/hnsw/plugin/compile/compile_smoke_test.go @@ -74,7 +74,7 @@ func TestHnswHandleCreateIndex_LogLine(t *testing.T) { // TestHnswHandleReindex_DelegatesToCreate — HandleReindex routes // through HandleCreateIndex, so the same log line is covered. func TestHnswHandleReindex_DelegatesToCreate(t *testing.T) { - err := Hooks{}.HandleReindex(&stubCtx{isFrontend: true}, map[string]*plan.IndexDef{}, false) + err := Hooks{}.HandleReindex(&stubCtx{isFrontend: true}, map[string]*plan.IndexDef{}, false, false) require.Error(t, err) } diff --git a/pkg/vectorindex/hnsw/plugin/runtime/runtime.go b/pkg/vectorindex/hnsw/plugin/runtime/runtime.go index f2dff8120d67d..93995bc2e7730 100644 --- a/pkg/vectorindex/hnsw/plugin/runtime/runtime.go +++ b/pkg/vectorindex/hnsw/plugin/runtime/runtime.go @@ -93,6 +93,11 @@ func (CatalogHooks) SupportedVectorTypes() []types.T { // SupportedPrimaryKeyTypes: requires an int64 primary key. func (CatalogHooks) SupportedPrimaryKeyTypes() []types.T { return []types.T{types.T_int64} } +// ValidQuantization: HNSW (usearch) validates quantization on its own +// param-build path (ParamsFromTree), and REINDEX does not accept a quantization +// change, so there is nothing to gate here. +func (CatalogHooks) ValidQuantization(_, _ string) error { return nil } + // SupportedIncludeColumnTypes: this index has no INCLUDE-column support. func (CatalogHooks) SupportedIncludeColumnTypes() []types.T { return nil } diff --git a/pkg/vectorindex/idxcron/executor.go b/pkg/vectorindex/idxcron/executor.go index 03ca5f3f94c44..7012ccf9c561e 100644 --- a/pkg/vectorindex/idxcron/executor.go +++ b/pkg/vectorindex/idxcron/executor.go @@ -17,6 +17,8 @@ package idxcron import ( "context" "fmt" + "os" + "strconv" "strings" "sync/atomic" "time" @@ -394,10 +396,39 @@ func runReindex(ctx context.Context, return err2 } } + + // SECOND is a sub-day cadence override: when set (> 0), the interval + // is that many seconds and the hour-of-day gate is bypassed (matched to + // currentHour) since it is meaningless at second granularity. Lets an + // index set a short interval in DDL (auto_update=true second=5) without + // the MO_IDXCRON_INTERVAL_SEC env override. + secondAst, err2 := sonic.Get([]byte(idx.IndexAlgoParams), catalog.Second) + if err2 == nil { + second, err2 := secondAst.Int64() + if err2 != nil { + return err2 + } + if second > 0 { + interval = time.Duration(second) * time.Second + hour = int64(currentHour) + } + } break } + // dev/test fast mode: a short interval + bypass the hour-of-day gate so a + // freshly-created index can be observed rebuilding within seconds. + if idxcronFastIntervalSec > 0 { + interval = time.Duration(idxcronFastIntervalSec) * time.Second + hour = int64(currentHour) + } + + logutil.Infof("[idxcron] eval action=%s db=%s table=%s index=%s: auto_update=%v interval=%v hour=%d currentHour=%d", + task.Action, task.DbName, task.TableName, task.IndexName, auto_update, interval, hour, currentHour) + if !auto_update || interval == 0 || currentHour != int(hour) { + logutil.Infof("[idxcron] skip index=%s: gate (auto_update=%v interval=%v hour=%d vs currentHour=%d)", + task.IndexName, auto_update, interval, hour, currentHour) reason = Reason_Skipped return } @@ -414,7 +445,7 @@ func runReindex(ctx context.Context, return } - ok, reason2, err2 := p.Idxcron().Updatable(idxcronplugin.UpdatableInput{ + upIn := idxcronplugin.UpdatableInput{ Sqlproc: sqlproc, TableDef: tableDef, IndexName: task.IndexName, @@ -422,23 +453,39 @@ func runReindex(ctx context.Context, CreatedAt: task.CreatedAt, LastUpdateAt: task.LastUpdateAt, Interval: interval, - }) + } + ok, reason2, err2 := p.Idxcron().Updatable(upIn) if err2 != nil { return } reason = reason2 if !ok { + logutil.Infof("[idxcron] skip index=%s: Updatable=false reason=%q", task.IndexName, reason2) return } - // run alter table alter reindex in force synchronous mode to make sure to build index in single transaction - sql := fmt.Sprintf("ALTER TABLE `%s`.`%s` ALTER REINDEX `%s` %s FORCE_SYNC", - task.DbName, task.TableName, task.IndexName, d.IdxcronAlgoToken) + // The reindex option is the descriptor's fixed value unless the plugin implements + // ReindexOptioner and picks one per fire (fulltext: MERGE vs REBUILD by dead fraction). + reindexOption := d.IdxcronReindexOption + if optioner, ok := p.Idxcron().(idxcronplugin.ReindexOptioner); ok { + opt, oerr := optioner.ReindexOption(upIn) + if oerr != nil { + err2 = oerr // closure's named return — NOT the outer err (clobbered by runTxn) + return + } + reindexOption = opt + } + + // run alter table alter reindex in force synchronous mode to make sure to build index in single transaction. + sql := buildReindexSql(task.DbName, task.TableName, task.IndexName, d.IdxcronAlgoToken, reindexOption) + logutil.Infof("[idxcron] reindex FIRING index=%s: %s", task.IndexName, sql) res, err2 := runReindexSql(sqlproc, sql) if err2 != nil { + logutil.Errorf("[idxcron] reindex FAILED index=%s: %v", task.IndexName, err2) return } res.Close() + logutil.Infof("[idxcron] reindex DONE index=%s", task.IndexName) // mark reindex is performed updated = true @@ -450,17 +497,23 @@ func runReindex(ctx context.Context, func (e *IndexUpdateTaskExecutor) run(ctx context.Context) (err error) { - logutil.Infof("IndexUpdateTaskExecutor START") currentHour := time.Now().Hour() + // tick-level trace: one line per cron fire, echoing the schedule + fast-mode overrides + // so a short-interval test deploy can confirm the executor is actually ticking (and how + // often) before the per-index eval/skip/FIRING lines below. + logutil.Infof("[idxcron] tick START currentHour=%d cronExpr=%q fastIntervalSec=%d", + currentHour, IndexUpdateTaskCronExpr, idxcronFastIntervalSec) + var fired, skipped int defer func() { - logutil.Infof("IndexUpdateTaskExecutor END") + logutil.Infof("[idxcron] tick END fired=%d skipped=%d err=%v", fired, skipped, err) }() tasks, err := getTasks(ctx, e.txnEngine, e.cnTxnClient, e.cnUUID) if err != nil { return err } + logutil.Infof("[idxcron] tick: %d index-update task(s) to evaluate", len(tasks)) // do the maintenance such as ivfflat re-index, fulltext batch_delete for _, t := range tasks { @@ -483,8 +536,12 @@ func (e *IndexUpdateTaskExecutor) run(ctx context.Context) (err error) { updated, reason, err2 = runReindex(ctx, e.txnEngine, e.cnTxnClient, e.cnUUID, t, currentHour, p) } + if updated { + fired++ + } if !updated && reason == Reason_Skipped { // skip save Status when update was skipped + skipped++ continue } @@ -499,11 +556,59 @@ func (e *IndexUpdateTaskExecutor) run(ctx context.Context) (err error) { return nil } -var IndexUpdateTaskCronExpr = "0 0 * * * *" // run once an hour, beginning of hour -// var IndexUpdateTaskCronExpr = "0 55 23 * * *" // 23:55:00 everyday +// IndexUpdateTaskCronExpr — how often the executor ticks. Default hourly; a dev/test +// deploy can lower it via env MO_IDXCRON_CRON (e.g. "*/20 * * * * *" every 20s) to +// observe scheduled rebuilds without waiting for the top of the hour. Read once at +// init; the cron task is registered from this value at bootstrap (predefine.go). +var IndexUpdateTaskCronExpr = func() string { + // MO_IDXCRON_TICK_SEC is an ergonomic seconds-interval knob for the bootstrap + // tick, translated to a cron descriptor (the parser enables cron.Descriptor). + // e.g. MO_IDXCRON_TICK_SEC=10 -> "@every 10s". Takes precedence over + // MO_IDXCRON_CRON so a deploy can change how often the executor checks index + // tasks without hand-writing a cron expression. Like MO_IDXCRON_CRON it is read + // once and baked into the cron task at first bootstrap (predefine.go), so it + // only takes effect on a fresh cluster — an existing cluster keeps its + // registered expr. The default stays conservative (hourly): every tick lists + // the registered index tasks (a small SQL), so a very short interval adds + // background load and should be an explicit opt-in. + if v := os.Getenv("MO_IDXCRON_TICK_SEC"); v != "" { + if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 { + return fmt.Sprintf("@every %ds", n) + } + } + if v := os.Getenv("MO_IDXCRON_CRON"); v != "" { + return v + } + return "0 0 * * * *" // run once an hour, beginning of hour +}() + +// buildReindexSql assembles the cron-triggered reindex statement. algoToken is the +// per-plugin keyword (SyncDescriptor.IdxcronAlgoToken, e.g. "IVFFLAT"/"FULLTEXT"); +// reindexOption is an optional extra keyword inserted before FORCE_SYNC +// (SyncDescriptor.IdxcronReindexOption, e.g. "MERGE" so fulltext retrieval runs +// incremental fold+tiered compaction), omitted when empty. FORCE_SYNC always runs the +// rebuild synchronously inside the txn. +func buildReindexSql(dbName, tableName, indexName, algoToken, reindexOption string) string { + reindexOpt := "" + if reindexOption != "" { + reindexOpt = " " + reindexOption + } + return fmt.Sprintf("ALTER TABLE `%s`.`%s` ALTER REINDEX `%s` %s%s FORCE_SYNC", + dbName, tableName, indexName, algoToken, reindexOpt) +} -// var IndexUpdateTaskCronExpr = "0 */5 * * * *" // every 5 minutes -// var IndexUpdateTaskCronExpr = "*/15 * * * * *" // every 15 seconds +// idxcronFastIntervalSec — dev/test override (env MO_IDXCRON_INTERVAL_SEC, in seconds). +// When > 0, runReindex uses it as the cadence interval AND bypasses the hour-of-day +// gate, so a freshly-created index can be observed rebuilding within seconds instead of +// waiting a day (the DAY param's minimum) at a specific HOUR. Unset (0) in production. +var idxcronFastIntervalSec = func() int64 { + if v := os.Getenv("MO_IDXCRON_INTERVAL_SEC"); v != "" { + if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 { + return n + } + } + return 0 +}() const ParamSeparator = " " diff --git a/pkg/vectorindex/idxcron/executor_test.go b/pkg/vectorindex/idxcron/executor_test.go index 8c4f3e40d0a1c..5b3539527b912 100644 --- a/pkg/vectorindex/idxcron/executor_test.go +++ b/pkg/vectorindex/idxcron/executor_test.go @@ -265,6 +265,7 @@ func (m mockCatalogHooks) SupportedOpTypes() map[string]string func (m mockCatalogHooks) SupportedVectorTypes() []types.T { return nil } func (m mockCatalogHooks) SupportedPrimaryKeyTypes() []types.T { return nil } func (m mockCatalogHooks) SupportedIncludeColumnTypes() []types.T { return nil } +func (m mockCatalogHooks) ValidQuantization(_, _ string) error { return nil } func (m mockCatalogHooks) ExperimentalFlag() string { return "" } func (m mockCatalogHooks) AlterTableCloneBehavior() catalogplugin.AlterTableCloneBehavior { return catalogplugin.AlterTableCloneBehavior{} diff --git a/pkg/vectorindex/ivfflat/kmeans/device/gpu.go b/pkg/vectorindex/ivfflat/kmeans/device/gpu.go index 7bf0eafbef676..2d81157f7e5f0 100644 --- a/pkg/vectorindex/ivfflat/kmeans/device/gpu.go +++ b/pkg/vectorindex/ivfflat/kmeans/device/gpu.go @@ -75,23 +75,6 @@ func (c *GpuClusterer[T]) Close() error { return nil } -func resolveCuvsDistanceForDense(distance metric.MetricType) cuvs.DistanceType { - switch distance { - case metric.Metric_L2sqDistance: - return cuvs.L2Expanded - case metric.Metric_L2Distance: - return cuvs.L2Expanded - case metric.Metric_InnerProduct: - return cuvs.InnerProduct - case metric.Metric_CosineDistance: - return cuvs.CosineSimilarity - case metric.Metric_L1Distance: - return cuvs.L1 - default: - return cuvs.L2Expanded - } -} - func NewKMeans[T types.RealNumbers](vectors [][]T, clusterCnt, maxIterations int, deltaThreshold float64, distanceType metric.MetricType, _ kmeans.InitType, @@ -122,7 +105,13 @@ func NewKMeans[T types.RealNumbers](vectors [][]T, clusterCnt, deviceID := 0 nthread := uint32(1) - km, err := cuvs.NewGpuKMeans[float32](uint32(clusterCnt), uint32(dim), resolveCuvsDistanceForDense(distanceType), maxIterations, deviceID, nthread) + // Dense centroid clustering always uses L2, independent of the index's + // search metric (matches the CPU path, ResolveKmeansDistanceFnForDense, + // which forces L2 for every metric). cuVS kmeans_balanced only supports + // L2/InnerProduct — forwarding the search metric (e.g. L1, cosine) makes + // cuVS abort with "distance metric not supported". The search metric is + // applied later at query time (centroid scan + re-rank), not in clustering. + km, err := cuvs.NewGpuKMeans[float32](uint32(clusterCnt), uint32(dim), cuvs.L2Expanded, maxIterations, deviceID, nthread) if err != nil { return nil, err } diff --git a/pkg/vectorindex/ivfflat/kmeans/device/issue_test.go b/pkg/vectorindex/ivfflat/kmeans/device/issue_test.go index 8202874c783f0..c7b7ab92db133 100644 --- a/pkg/vectorindex/ivfflat/kmeans/device/issue_test.go +++ b/pkg/vectorindex/ivfflat/kmeans/device/issue_test.go @@ -85,7 +85,7 @@ func Search(datasetvec [][]float32, queriesvec [][]float32, limit uint, distance deviceID := 0 nthread := uint32(1) - bf, err := cuvs.NewGpuBruteForce[float32](flattenedDataset, uint64(len(datasetvec)), uint32(dim), distanceType, nthread, deviceID) + bf, err := cuvs.NewGpuBruteForce[float32, float32](flattenedDataset, uint64(len(datasetvec)), uint32(dim), distanceType, nthread, deviceID) if err != nil { return nil, nil, err } diff --git a/pkg/vectorindex/ivfflat/plugin/compile/compile.go b/pkg/vectorindex/ivfflat/plugin/compile/compile.go index 81b3f240a3504..e3309ced129a2 100644 --- a/pkg/vectorindex/ivfflat/plugin/compile/compile.go +++ b/pkg/vectorindex/ivfflat/plugin/compile/compile.go @@ -34,6 +34,7 @@ import ( "github.com/bytedance/sonic" "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" compileplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/compile" "github.com/matrixorigin/matrixone/pkg/logutil" @@ -41,6 +42,8 @@ import ( "github.com/matrixorigin/matrixone/pkg/util/executor" "github.com/matrixorigin/matrixone/pkg/vectorindex" "github.com/matrixorigin/matrixone/pkg/vectorindex/cache" + ivfflatruntime "github.com/matrixorigin/matrixone/pkg/vectorindex/ivfflat/plugin/runtime" + "github.com/matrixorigin/matrixone/pkg/vectorindex/quantizer" ) // actionIvfflatReindex mirrors idxcron.Action_Ivfflat_Reindex. Inlined @@ -66,7 +69,7 @@ func (h Hooks) HandleCreateIndex(ctx compileplugin.CompileContext, indexDefs map // HandleReindex runs the same body as HandleCreateIndex with forceSync // threaded into centroid building. Matches ddl.go:980-987 dispatch. -func (h Hooks) HandleReindex(ctx compileplugin.CompileContext, indexDefs map[string]*plan.IndexDef, forceSync bool) error { +func (h Hooks) HandleReindex(ctx compileplugin.CompileContext, indexDefs map[string]*plan.IndexDef, forceSync bool, _ bool) error { return runCreateOrReindex(ctx, indexDefs, forceSync) } @@ -87,11 +90,24 @@ func (Hooks) RestoreInitSQL(ctx compileplugin.CompileContext, indexDefs map[stri // inline — that persistence stays at the SQL-layer call site, so this // hook only performs the map merge. func (Hooks) ValidateReindexParams(old map[string]string, alter compileplugin.ReindexParamUpdate) (map[string]string, error) { - return compileplugin.MergeReindexParams(old, alter, "ivfflat", + // Merge first, then validate the EFFECTIVE quantization via the per-algo + // catalog hook (the single home shared with CREATE; the value the reindex + // set, or the index's stored value when the statement omitted it — e.g. the + // idxcron-issued rebuild). + merged, err := compileplugin.MergeReindexParams(old, alter, "ivfflat", catalog.IndexAlgoParamLists, catalog.IndexAlgoParamKmeansTrainPercent, catalog.IndexAlgoParamKmeansMaxIteration, + catalog.Quantization, ) + if err != nil { + return nil, err + } + if err := (ivfflatruntime.CatalogHooks{}).ValidQuantization( + merged[catalog.Quantization], merged[catalog.IndexAlgoParamOpType]); err != nil { + return nil, err + } + return merged, nil } // HandleDropIndex: IVF-FLAT generic hidden-table deletion is performed @@ -100,6 +116,12 @@ func (Hooks) ValidateReindexParams(old map[string]string, alter compileplugin.Re // (pkg/sql/compile/ddl.go DropIndex path). No additional cleanup here. func (Hooks) HandleDropIndex(_ compileplugin.CompileContext, defs map[string]*plan.IndexDef) error { logutil.Infof("[plugin] ivfflat HandleDropIndex: defs=%d", len(defs)) + // Evict the cached search index immediately rather than waiting for the + // 5-min VectorIndexCacheTTL. Mirrors the create-side + // cache.Cache.Remove(fmt.Sprintf("%s:0", centroidsDef.IndexTableName)). + if centroidsDef, ok := defs[catalog.SystemSI_IVFFLAT_TblType_Centroids]; ok { + cache.Cache.Remove(fmt.Sprintf("%s:0", centroidsDef.IndexTableName)) + } return nil } @@ -195,18 +217,33 @@ func runCreateOrReindex(ctx compileplugin.CompileContext, indexDefs map[string]* return err } - // 4.b populate centroids table - if err = ivfIndexCentroidsTable(ctx, centroidsDef, qryDatabase, originalTableDef, - totalCnt, metaDef.IndexTableName, forceSync); err != nil { - return err - } - - if !async || forceSync { - // 4.c populate entries table - if err = ivfIndexEntriesTable(ctx, entriesDef, qryDatabase, originalTableDef, - metaDef.IndexTableName, centroidsDef.IndexTableName); err != nil { + // 4.b + 4.c: build the index. Both kmeans (4.b) and entry assignment (4.c) + // scan the source table, but queries never re-read it (re-rank fetches only a + // handful of rows), so run the build's reads with SkipMemoryCacheWrites — this + // one-shot source scan must not evict the index-entry working set the queries + // actually hit from the fileservice cache. The optional-interface keeps the + // CompileContext interface (and its plugin mocks) untouched; non-supporting + // contexts just build directly. + buildIndex := func() error { + if err := ivfIndexCentroidsTable(ctx, centroidsDef, qryDatabase, originalTableDef, + totalCnt, metaDef.IndexTableName, forceSync); err != nil { return err } + if !async || forceSync { + if err := ivfIndexEntriesTable(ctx, entriesDef, qryDatabase, originalTableDef, + metaDef.IndexTableName, centroidsDef.IndexTableName); err != nil { + return err + } + } + return nil + } + if r, ok := ctx.(interface{ RunWithSourceReadCacheSkip(func() error) error }); ok { + err = r.RunWithSourceReadCacheSkip(buildIndex) + } else { + err = buildIndex() + } + if err != nil { + return err } // 4.d delete older entries in index table. @@ -245,6 +282,33 @@ func indexColCount(ctx compileplugin.CompileContext, indexDef *plan.IndexDef, return n, nil } +// readQuantizeBound reads a scalar DOUBLE metadata value (e.g. quantize_min / +// quantize_max) by key. found=false when the row is absent (e.g. a pre-quantizer +// index), in which case the caller falls back to a raw cast. +func readQuantizeBound(ctx compileplugin.CompileContext, qryDatabase, metaTbl, key string) (val float64, found bool, err error) { + sql := fmt.Sprintf("SELECT CAST(`%s` AS DOUBLE) FROM `%s`.`%s` WHERE `%s` = '%s'", + catalog.SystemSI_IVFFLAT_TblCol_Metadata_val, qryDatabase, metaTbl, + catalog.SystemSI_IVFFLAT_TblCol_Metadata_key, key) + rs, err := ctx.RunSqlWithResult(sql) + if err != nil { + return 0, false, err + } + defer rs.Close() + rs.ReadRows(func(_ int, cols []*vector.Vector) bool { + if len(cols) == 0 { + return false + } + rows := executor.GetFixedRows[float64](cols[0]) + if len(rows) == 0 { + return false + } + val = rows[0] + found = true + return false + }) + return val, found, nil +} + // ivfIndexMetaTable is lifted from Scope.handleIvfIndexMetaTable // (pkg/sql/compile/ddl_index_algo.go:221). func ivfIndexMetaTable(ctx compileplugin.CompileContext, indexDef *plan.IndexDef, qryDatabase string) error { @@ -395,6 +459,51 @@ func ivfIndexEntriesTable( return err } + // QUANTIZATION: if set, entries are stored as the quantization type (the entry + // column was created with that type in schema.go), so the SELECT casts the + // base vectors to it. The CENTROIDX assignment still uses the f32 base column. + indexColName := indexDef.Parts[0] + entrySelectExpr := fmt.Sprintf("`%s`", indexColName) + if qv, qerr := sonic.Get([]byte(indexDef.IndexAlgoParams), catalog.Quantization); qerr == nil { + if qstr, serr := qv.String(); serr == nil && qstr != "" { + if qt, ok := quantizer.ToVectorType(qstr); ok { + var dim int32 + for _, c := range originalTableDef.Cols { + if c.Name == indexColName { + dim = c.Typ.Width + break + } + } + if qt == types.T_array_int8 || qt == types.T_array_uint8 { + // cuVS-style asymmetric scalar quantizer: map the trained + // [min,max] (stored in metadata by ivf_create) onto the full int8 + // range [-128,127] (or uint8 [0,255]) via q(x)=round(x*mul+add). + // float16 needs no scale. + qmin, ok1, err := readQuantizeBound(ctx, qryDatabase, metadataTableName, catalog.SystemSI_IVFFLAT_Metadata_QuantizeMin) + if err != nil { + return err + } + qmax, ok2, err := readQuantizeBound(ctx, qryDatabase, metadataTableName, catalog.SystemSI_IVFFLAT_Metadata_QuantizeMax) + if err != nil { + return err + } + col := fmt.Sprintf("`%s`", indexColName) + if ok1 && ok2 && qt == types.T_array_int8 { + mul, add := quantizer.Int8Params(qmin, qmax) + entrySelectExpr = quantizer.Int8EntrySQL(col, mul, add, dim) + } else if ok1 && ok2 { + mul, add := quantizer.Uint8Params(qmin, qmax) + entrySelectExpr = quantizer.Uint8EntrySQL(col, mul, add, dim) + } else { + entrySelectExpr = quantizer.CastSQL(col, qt, dim) + } + } else { + entrySelectExpr = quantizer.CastSQL(fmt.Sprintf("`%s`", indexColName), qt, dim) + } + } + } + } + var originalTblPkColsCommaSeparated, originalTblPkColMaySerial string if originalTableDef.Pkey.PkeyColName == catalog.CPrimaryKeyColName { for i, part := range originalTableDef.Pkey.Names { @@ -433,14 +542,14 @@ func ivfIndexEntriesTable( indexColumnName := indexDef.Parts[0] centroidsCrossL2JoinTbl := fmt.Sprintf("%s "+ - "SELECT `%s`, `%s`, %s, `%s`"+ + "SELECT `%s`, `%s`, %s, %s"+ " FROM `%s`.`%s` CENTROIDX ('%s') join %s "+ " using (`%s`, `%s`) ", insertSQL, catalog.SystemSI_IVFFLAT_TblCol_Centroids_version, catalog.SystemSI_IVFFLAT_TblCol_Centroids_id, originalTblPkColMaySerial, - indexColumnName, + entrySelectExpr, // base column, or cast(base as ) under QUANTIZATION qryDatabase, originalTableDef.Name, optype, diff --git a/pkg/vectorindex/ivfflat/plugin/compile/compile_smoke_test.go b/pkg/vectorindex/ivfflat/plugin/compile/compile_smoke_test.go index fe543bde83de2..378f8495a433e 100644 --- a/pkg/vectorindex/ivfflat/plugin/compile/compile_smoke_test.go +++ b/pkg/vectorindex/ivfflat/plugin/compile/compile_smoke_test.go @@ -17,6 +17,7 @@ package compile import ( "testing" + "github.com/matrixorigin/matrixone/pkg/catalog" compileplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/compile" "github.com/matrixorigin/matrixone/pkg/pb/api" "github.com/matrixorigin/matrixone/pkg/pb/plan" @@ -74,7 +75,7 @@ func TestIvfflatHandleCreateIndex_LogLine(t *testing.T) { // TestIvfflatHandleReindex_LogLine — same shape via HandleReindex. func TestIvfflatHandleReindex_LogLine(t *testing.T) { - err := Hooks{}.HandleReindex(&stubCtx{}, map[string]*plan.IndexDef{}, false) + err := Hooks{}.HandleReindex(&stubCtx{}, map[string]*plan.IndexDef{}, false, false) require.Error(t, err) } @@ -89,6 +90,21 @@ func TestIvfflatValidateReindexParams_Passthrough(t *testing.T) { require.Equal(t, old, got) } +// TestIvfflatValidateReindexParams_Quantization: IVF-FLAT honors a narrow-type +// quantization on reindex (same set as CREATE) and rejects unknown values. +func TestIvfflatValidateReindexParams_Quantization(t *testing.T) { + got, err := Hooks{}.ValidateReindexParams(nil, compileplugin.ReindexParamUpdate{ + Params: map[string]string{catalog.Quantization: "int8"}, + }) + require.NoError(t, err) + require.Equal(t, "int8", got[catalog.Quantization]) + + _, err = Hooks{}.ValidateReindexParams(nil, compileplugin.ReindexParamUpdate{ + Params: map[string]string{catalog.Quantization: "garbage"}, + }) + require.Error(t, err) +} + // TestIvfflatIdxcronMetadata_BackgroundLog covers the entry log line // of IdxcronMetadata via the isFrontend=false path (which short- // circuits through BuildIdxcronMetadata's IsFrontend guard). diff --git a/pkg/vectorindex/ivfflat/plugin/plan/schema.go b/pkg/vectorindex/ivfflat/plugin/plan/schema.go index 5207c442e44d2..7d147dd89586a 100644 --- a/pkg/vectorindex/ivfflat/plugin/plan/schema.go +++ b/pkg/vectorindex/ivfflat/plugin/plan/schema.go @@ -24,6 +24,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" "github.com/matrixorigin/matrixone/pkg/sql/util" ivfflatrt "github.com/matrixorigin/matrixone/pkg/vectorindex/ivfflat/plugin/runtime" + "github.com/matrixorigin/matrixone/pkg/vectorindex/quantizer" ) // ivfflatCatalogHooks is the shared (stateless) catalog-hooks instance used for @@ -146,14 +147,31 @@ func (Hooks) BuildSecondaryIndexDefs( Typ: plan.Type{Id: int32(types.T_int64)}, Default: &plan.Default{NullAbility: false, Expr: nil, OriginString: ""}, } + // Centroid type is decoupled from the entry type. Centroids are f32 whenever + // the entries are NOT a plain f32/f64 column: i.e. for a narrow base + // (bf16/f16/int8) or for ANY base under QUANTIZATION (incl. f64). f32 gives + // accurate assignment, fast f32 search, and tiny RAM for the few centroids; + // the entries carry the memory win. A plain f32/f64 column keeps its type. + centroidTyp := plan.Type{ + Id: colMap[colName].Typ.Id, + Width: colMap[colName].Typ.Width, + Scale: colMap[colName].Typ.Scale, + } + quantized := indexInfo.IndexOption != nil && indexInfo.IndexOption.Quantization != "" + switch types.T(centroidTyp.Id) { + case types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: + centroidTyp.Id = int32(types.T_array_float32) + centroidTyp.Scale = 0 + default: + if quantized { + centroidTyp.Id = int32(types.T_array_float32) + centroidTyp.Scale = 0 + } + } tableDefs[1].Cols[2] = &plan.ColDef{ - Name: catalog.SystemSI_IVFFLAT_TblCol_Centroids_centroid, - Alg: plan.CompressType_Lz4, - Typ: plan.Type{ - Id: colMap[colName].Typ.Id, - Width: colMap[colName].Typ.Width, - Scale: colMap[colName].Typ.Scale, - }, + Name: catalog.SystemSI_IVFFLAT_TblCol_Centroids_centroid, + Alg: plan.CompressType_Lz4, + Typ: centroidTyp, Default: &plan.Default{NullAbility: true, Expr: nil, OriginString: ""}, } tableDefs[1].Cols[3] = planplugin.MakeHiddenColDefByName(catalog.CPrimaryKeyColName) @@ -218,14 +236,38 @@ func (Hooks) BuildSecondaryIndexDefs( }, Default: &plan.Default{NullAbility: false, Expr: nil, OriginString: ""}, } + // Entry type follows the QUANTIZATION option: CREATE INDEX ... USING + // ivfflat ... QUANTIZATION='int8' stores entries as vecint8 (quantized from + // the base vectors), while the base column and the f32 centroids are + // unchanged. Without QUANTIZATION the entries keep the base column type. + entryTyp := plan.Type{ + Id: colMap[colName].Typ.Id, + Width: colMap[colName].Typ.Width, + Scale: colMap[colName].Typ.Scale, + } + if indexInfo.IndexOption != nil && indexInfo.IndexOption.Quantization != "" { + if qt, ok := quantizer.ToVectorType(indexInfo.IndexOption.Quantization); ok { + // QUANTIZATION is downcast-only: the quantized entry element must be the + // same width or narrower than the base column. Upcasting (e.g. a bf16 or + // int8 base with QUANTIZATION='float32') is unsupported — it costs 2-4x the + // entry storage for no precision gain and forces the f32 distance kernel + // over narrow entries. Omit QUANTIZATION to keep the base-width entries. + baseSize := types.Type{Oid: types.T(colMap[colName].Typ.Id)}.GetArrayElementSize() + quantSize := types.Type{Oid: qt}.GetArrayElementSize() + if quantSize > baseSize { + return nil, nil, moerr.NewNotSupportedf(ctx.GetContext(), + "ivfflat QUANTIZATION '%s' (%d bytes/element) cannot upcast base column %s (%d bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type", + indexInfo.IndexOption.Quantization, quantSize, + types.T(colMap[colName].Typ.Id).String(), baseSize) + } + entryTyp.Id = int32(qt) + entryTyp.Scale = 0 + } + } tableDefs[2].Cols[3] = &plan.ColDef{ - Name: catalog.SystemSI_IVFFLAT_TblCol_Entries_entry, - Alg: plan.CompressType_Lz4, - Typ: plan.Type{ - Id: colMap[colName].Typ.Id, - Width: colMap[colName].Typ.Width, - Scale: colMap[colName].Typ.Scale, - }, + Name: catalog.SystemSI_IVFFLAT_TblCol_Entries_entry, + Alg: plan.CompressType_Lz4, + Typ: entryTyp, Default: &plan.Default{NullAbility: true, Expr: nil, OriginString: ""}, } tableDefs[2].Cols[4] = planplugin.MakeHiddenColDefByName(catalog.CPrimaryKeyColName) diff --git a/pkg/vectorindex/ivfflat/plugin/runtime/runtime.go b/pkg/vectorindex/ivfflat/plugin/runtime/runtime.go index a64904ee68cbd..0af7543fd6b41 100644 --- a/pkg/vectorindex/ivfflat/plugin/runtime/runtime.go +++ b/pkg/vectorindex/ivfflat/plugin/runtime/runtime.go @@ -30,6 +30,7 @@ import ( catalogplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/catalog" "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" "github.com/matrixorigin/matrixone/pkg/vectorindex/metric" + "github.com/matrixorigin/matrixone/pkg/vectorindex/quantizer" ) // actionIvfflatReindex mirrors idxcron.Action_Ivfflat_Reindex. Inlined @@ -127,15 +128,36 @@ func (CatalogHooks) ExperimentalFlag() string { return "" } // SupportedOpTypes returns IVF-FLAT's metric registry. IVF uses a // distinct metric table from HNSW/USearch (OpTypeToIvfMetric). -// SupportedVectorTypes: IVF-FLAT indexes f32 or f64 vectors. +// SupportedVectorTypes: IVF-FLAT indexes all vector element types. Entries are +// stored in their own (narrow) type; centroids are f32 (decoupled). kmeans runs +// in f32, narrow distances go through the float32 bridge / narrow kernels. func (CatalogHooks) SupportedVectorTypes() []types.T { - return []types.T{types.T_array_float32, types.T_array_float64} + return []types.T{ + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, + } } // SupportedPrimaryKeyTypes: IVF-FLAT imposes no PK-type constraint — the // primary key may be any type. nil = "no constraint". func (CatalogHooks) SupportedPrimaryKeyTypes() []types.T { return nil } +// ValidQuantization gates the quantization value for IVF-FLAT: it must name a +// narrow vector type IVF-FLAT supports (float32/float16/bf16/int8/uint8, via +// quantizer.ToVectorType). IVF-FLAT re-ranks on the CPU from the stored entries, +// so unlike the cuvs backends it imposes no op_type restriction; op is unused. +// One home for CREATE (plan/schema) and REINDEX (compile/ValidateReindexParams). +func (CatalogHooks) ValidQuantization(quant, _ string) error { + if quant == "" { + return nil + } + if _, ok := quantizer.ToVectorType(quant); !ok { + return moerr.NewNotSupportedNoCtxf( + "ivfflat quantization %q (supported: float32, float16, bf16, int8, uint8)", quant) + } + return nil +} + // SupportedIncludeColumnTypes: this index has no INCLUDE-column support. func (CatalogHooks) SupportedIncludeColumnTypes() []types.T { return nil } @@ -215,6 +237,9 @@ func (CatalogHooks) ParamsFromTree(idx *tree.Index) (map[string]string, error) { if idx.IndexOption.Hour > 0 { res[catalog.Hour] = strconv.FormatInt(idx.IndexOption.Hour, 10) } + if idx.IndexOption.Second > 0 { + res[catalog.Second] = strconv.FormatInt(idx.IndexOption.Second, 10) + } if idx.IndexOption.KmeansTrainPercent > 0 { res[catalog.IndexAlgoParamKmeansTrainPercent] = strconv.FormatInt(idx.IndexOption.KmeansTrainPercent, 10) @@ -222,5 +247,17 @@ func (CatalogHooks) ParamsFromTree(idx *tree.Index) (map[string]string, error) { if idx.IndexOption.KmeansMaxIteration > 0 { res[catalog.IndexAlgoParamKmeansMaxIteration] = strconv.FormatInt(idx.IndexOption.KmeansMaxIteration, 10) } + + // QUANTIZATION stores the ivfflat ENTRIES in a narrow type (float16/int8); + // the base column and f32 centroids are unchanged. Persist it in algo_params + // so the entries build (compile) and the search can read it back. Only the + // predefined names that map to a MO narrow vector type are accepted. + if q := idx.IndexOption.Quantization; q != "" { + if _, ok := quantizer.ToVectorType(q); !ok { + return nil, moerr.NewInternalErrorNoCtx(fmt.Sprintf( + "ivfflat: unsupported quantization '%s' (supported: 'float32', 'float16', 'bf16', 'int8', 'uint8')", q)) + } + res[catalog.Quantization] = catalog.ToLower(q) + } return res, nil } diff --git a/pkg/vectorindex/ivfflat/plugin/runtime/runtime_test.go b/pkg/vectorindex/ivfflat/plugin/runtime/runtime_test.go index 372d5ac2d1f62..723dd1765c113 100644 --- a/pkg/vectorindex/ivfflat/plugin/runtime/runtime_test.go +++ b/pkg/vectorindex/ivfflat/plugin/runtime/runtime_test.go @@ -137,3 +137,27 @@ func TestIvfflatParamsFromTree_InvalidOpType(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "invalid op_type") } + +func TestIvfflatParamsFromTree_Quantization(t *testing.T) { + for _, q := range []string{"int8", "uint8", "float16", "bf16", "float32", "INT8", "Bf16", "UINT8"} { + idx := &tree.Index{IndexOption: &tree.IndexOption{Quantization: q}} + got, err := CatalogHooks{}.ParamsFromTree(idx) + require.NoErrorf(t, err, "quantization %q", q) + require.Equalf(t, catalog.ToLower(q), got[catalog.Quantization], "stored quantization %q", q) + } + // omitted -> not present in params + idx := &tree.Index{IndexOption: &tree.IndexOption{}} + got, err := CatalogHooks{}.ParamsFromTree(idx) + require.NoError(t, err) + _, present := got[catalog.Quantization] + require.False(t, present) +} + +func TestIvfflatParamsFromTree_InvalidQuantization(t *testing.T) { + for _, q := range []string{"float64", "f16", "garbage"} { + idx := &tree.Index{IndexOption: &tree.IndexOption{Quantization: q}} + _, err := CatalogHooks{}.ParamsFromTree(idx) + require.Errorf(t, err, "quantization %q should be rejected", q) + require.Contains(t, err.Error(), "unsupported quantization") + } +} diff --git a/pkg/vectorindex/ivfflat/search.go b/pkg/vectorindex/ivfflat/search.go index 723fb7b544786..106b42b5ac0c7 100644 --- a/pkg/vectorindex/ivfflat/search.go +++ b/pkg/vectorindex/ivfflat/search.go @@ -30,6 +30,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/vectorindex/brute_force" "github.com/matrixorigin/matrixone/pkg/vectorindex/cache" "github.com/matrixorigin/matrixone/pkg/vectorindex/metric" + "github.com/matrixorigin/matrixone/pkg/vectorindex/quantizer" "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" ) @@ -47,6 +48,11 @@ var runSql = sqlexec.RunSql type IvfflatSearchIndex[T types.RealNumbers] struct { Version int64 Centroids cache.VectorIndexSearchIf + // QuantMul/QuantAdd are the int8 scalar-quantizer params (q(x)=round(x*mul+add)) + // derived from the trained [min,max] in metadata; the query uses the same + // transform as the entries. Defaults (1,0) = identity when not int8-quantized. + QuantMul float64 + QuantAdd float64 } // This is the Ivf search implementation that implement VectorIndexSearchIf interface @@ -127,16 +133,63 @@ func (idx *IvfflatSearchIndex[T]) LoadCentroids(proc *sqlexec.SqlProcess, idxcfg func (idx *IvfflatSearchIndex[T]) LoadIndex(proc *sqlexec.SqlProcess, idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig, nthread int64) (err error) { idx.Version = idxcfg.Ivfflat.Version + idx.QuantMul = 1.0 + idx.QuantAdd = 0.0 err = idx.LoadCentroids(proc, idxcfg, tblcfg, nthread) if err != nil { return err } + // int8/uint8 QUANTIZATION: load the trained [min,max] and derive the same + // transform the entries were quantized with, so the query maps identically. + if vt := types.T(idxcfg.Ivfflat.VectorType); vt == types.T_array_int8 || vt == types.T_array_uint8 { + if err = idx.loadQuantizeBounds(proc, tblcfg, vt); err != nil { + return err + } + } + + return nil +} + +func (idx *IvfflatSearchIndex[T]) loadQuantizeBounds(proc *sqlexec.SqlProcess, tblcfg vectorindex.IndexTableConfig, vt types.T) error { + // Fetch both trained bounds in one round-trip; the metadata table is + // small and this runs once per index load. + sql := fmt.Sprintf("SELECT `%s`, CAST(`%s` AS DOUBLE) FROM `%s`.`%s` WHERE `%s` IN ('%s', '%s')", + catalog.SystemSI_IVFFLAT_TblCol_Metadata_key, catalog.SystemSI_IVFFLAT_TblCol_Metadata_val, + tblcfg.DbName, tblcfg.MetadataTable, catalog.SystemSI_IVFFLAT_TblCol_Metadata_key, + catalog.SystemSI_IVFFLAT_Metadata_QuantizeMin, catalog.SystemSI_IVFFLAT_Metadata_QuantizeMax) + res, err := runSql(proc, sql) + if err != nil { + return err + } + defer res.Close() + + var qmin, qmax float64 + var ok1, ok2 bool + for _, bat := range res.Batches { + keyVec, valVec := bat.Vecs[0], bat.Vecs[1] + for i := 0; i < bat.RowCount(); i++ { + val := vector.GetFixedAtNoTypeCheck[float64](valVec, i) + switch keyVec.GetStringAt(i) { + case catalog.SystemSI_IVFFLAT_Metadata_QuantizeMin: + qmin, ok1 = val, true + case catalog.SystemSI_IVFFLAT_Metadata_QuantizeMax: + qmax, ok2 = val, true + } + } + } + if ok1 && ok2 { + if vt == types.T_array_uint8 { + idx.QuantMul, idx.QuantAdd = quantizer.Uint8Params(qmin, qmax) + } else { + idx.QuantMul, idx.QuantAdd = quantizer.Int8Params(qmin, qmax) + } + } return nil } -func (idx *IvfflatSearchIndex[T]) findCentroids(sqlproc *sqlexec.SqlProcess, query []T, distfn metric.DistanceFunction[T], idxcfg vectorindex.IndexConfig, probe uint, _ int64) ([]int64, error) { +func (idx *IvfflatSearchIndex[T]) findCentroids(sqlproc *sqlexec.SqlProcess, query []T, idxcfg vectorindex.IndexConfig, probe uint, _ int64) ([]int64, error) { if idx.Centroids == nil { // empty index has id = 1 @@ -244,12 +297,7 @@ func (idx *IvfflatSearchIndex[T]) Search( nthread int64, ) (keys any, distances []float64, err error) { - distfn, err := metric.ResolveDistanceFn[T](metric.MetricType(idxcfg.Ivfflat.Metric)) - if err != nil { - return - } - - centroids_ids, err := idx.findCentroids(sqlproc, query, distfn, idxcfg, rt.Probe, nthread) + centroids_ids, err := idx.findCentroids(sqlproc, query, idxcfg, rt.Probe, nthread) if err != nil { return } @@ -280,6 +328,36 @@ func (idx *IvfflatSearchIndex[T]) Search( vecFromB64Fn = "vecf64_from_base64" } + // Re-rank distance. The ENTRY must stay a plain column so the ORDER BY + // index-param pushdown (readutil.SetIndexParam) can identify it — wrapping it + // in a CAST makes Args[0] a function and panics. The query must be a CONSTANT + // vec literal of the SAME (narrow) type as the entries, or the pushdown can't + // fold it and the pushed top-limit stays 0 ("top limit must be positive"). A + // cast of vecf32_from_base64(...) does NOT fold (vector casts aren't constant- + // folded), so for narrow entries quantize the f32 query to the entry type here + // and pass it via vec{bf16,f16,int8}_from_base64 — a STRICT decode that folds + // to a narrow literal, the narrow sibling of vecf32_from_base64. f32/f64 use + // the plain f32 base64 decode. + entryCol := fmt.Sprintf("`%s`", catalog.SystemSI_IVFFLAT_TblCol_Entries_entry) + queryExpr := fmt.Sprintf("%s('%s')", vecFromB64Fn, queryB64) + if qf32, ok := any(query).([]float32); ok { + switch types.T(idxcfg.Ivfflat.VectorType) { + case types.T_array_bf16: + queryExpr = fmt.Sprintf("vecbf16_from_base64('%s')", types.ArrayToBase64(types.Float32ToBF16Slice(qf32))) + case types.T_array_float16: + queryExpr = fmt.Sprintf("vecf16_from_base64('%s')", types.ArrayToBase64(types.Float32ToFloat16Slice(qf32))) + case types.T_array_int8: + // apply the same q(x)=x*mul+add transform as the entries, then round+clamp + // to int8. (mul,add)=(1,0) falls back to the raw cast (no quantizer). + sq := quantizer.ApplyInt8(qf32, idx.QuantMul, idx.QuantAdd) + queryExpr = fmt.Sprintf("vecint8_from_base64('%s')", types.ArrayToBase64(sq)) + case types.T_array_uint8: + // same transform as int8, narrowed to the unsigned [0,255] range. + sq := quantizer.ApplyUint8(qf32, idx.QuantMul, idx.QuantAdd) + queryExpr = fmt.Sprintf("vecuint8_from_base64('%s')", types.ArrayToBase64(sq)) + } + } + if sqlproc != nil && sqlproc.ExactPkFilter != "" { // Exact PK path: WaitUniqueJoinKeys converted small key set into ExactPkFilter. // Query entries directly by pk list, skip centroid-based filtering. @@ -294,12 +372,11 @@ func (idx *IvfflatSearchIndex[T]) Search( // a plain filtered read that returns the full candidate set; the downstream // Node_SORT + LIMIT k does the ranking and truncation. sql = fmt.Sprintf( - "SELECT `%s`, %s(`%s`, %s('%s')) as vec_dist FROM `%s`.`%s` WHERE `%s` = %d AND `%s` IN (%s)", + "SELECT `%s`, %s(%s, %s) as vec_dist FROM `%s`.`%s` WHERE `%s` = %d AND `%s` IN (%s)", catalog.SystemSI_IVFFLAT_TblCol_Entries_pk, metric.MetricTypeToDistFuncName[metric.MetricType(idxcfg.Ivfflat.Metric)], - catalog.SystemSI_IVFFLAT_TblCol_Entries_entry, - vecFromB64Fn, - queryB64, + entryCol, + queryExpr, tblcfg.DbName, tblcfg.EntriesTable, catalog.SystemSI_IVFFLAT_TblCol_Entries_version, idx.Version, @@ -309,12 +386,11 @@ func (idx *IvfflatSearchIndex[T]) Search( } else { // Standard centroid-based path with optional CBloomFilter pre-filtering. sql = fmt.Sprintf( - "SELECT `%s`, %s(`%s`, %s('%s')) as vec_dist FROM `%s`.`%s` WHERE `%s` = %d AND `%s` IN (%s) ORDER BY vec_dist LIMIT %d", + "SELECT `%s`, %s(%s, %s) as vec_dist FROM `%s`.`%s` WHERE `%s` = %d AND `%s` IN (%s) ORDER BY vec_dist LIMIT %d", catalog.SystemSI_IVFFLAT_TblCol_Entries_pk, metric.MetricTypeToDistFuncName[metric.MetricType(idxcfg.Ivfflat.Metric)], - catalog.SystemSI_IVFFLAT_TblCol_Entries_entry, - vecFromB64Fn, - queryB64, + entryCol, + queryExpr, tblcfg.DbName, tblcfg.EntriesTable, catalog.SystemSI_IVFFLAT_TblCol_Entries_version, idx.Version, @@ -325,8 +401,6 @@ func (idx *IvfflatSearchIndex[T]) Search( } //fmt.Println("IVFFlat SQL: ", sql) - //os.Stderr.WriteString(sql) - //os.Stderr.WriteString("\n") res, err := runSql(sqlproc, sql) if err != nil { diff --git a/pkg/vectorindex/ivfflat/search_quantize_test.go b/pkg/vectorindex/ivfflat/search_quantize_test.go new file mode 100644 index 0000000000000..2e274429248e3 --- /dev/null +++ b/pkg/vectorindex/ivfflat/search_quantize_test.go @@ -0,0 +1,87 @@ +// Copyright 2025 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ivfflat + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/testutil" + "github.com/matrixorigin/matrixone/pkg/util/executor" + "github.com/matrixorigin/matrixone/pkg/vectorindex" + "github.com/matrixorigin/matrixone/pkg/vectorindex/quantizer" + "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" + "github.com/stretchr/testify/require" +) + +// mockQuantizeBoundsResult builds a (key, val) result mirroring the single +// `WHERE key IN ('quantize_min','quantize_max')` query loadQuantizeBounds issues. +func mockQuantizeBoundsResult(m *mpool.MPool, qmin, qmax float64) executor.Result { + bat := batch.NewWithSize(2) + keyVec := vector.NewVec(types.T_varchar.ToType()) + _ = vector.AppendBytes(keyVec, []byte(catalog.SystemSI_IVFFLAT_Metadata_QuantizeMin), false, m) + _ = vector.AppendBytes(keyVec, []byte(catalog.SystemSI_IVFFLAT_Metadata_QuantizeMax), false, m) + valVec := vector.NewVec(types.T_float64.ToType()) + _ = vector.AppendFixed(valVec, qmin, false, m) + _ = vector.AppendFixed(valVec, qmax, false, m) + bat.Vecs[0] = keyVec + bat.Vecs[1] = valVec + bat.SetRowCount(2) + return executor.Result{Mp: m, Batches: []*batch.Batch{bat}} +} + +func TestLoadQuantizeBounds(t *testing.T) { + defer func() { runSql = sqlexec.RunSql }() + + m := mpool.MustNewZero() + proc := testutil.NewProcessWithMPool(t, "", m) + sqlproc := sqlexec.NewSqlProcess(proc) + var tblcfg vectorindex.IndexTableConfig + + const qmin, qmax = -2.0, 6.0 + + // both bounds present → params derived per element type + runSql = func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + return mockQuantizeBoundsResult(m, qmin, qmax), nil + } + for _, vt := range []types.T{types.T_array_int8, types.T_array_uint8} { + idx := &IvfflatSearchIndex[float32]{QuantMul: 1, QuantAdd: 0} + require.NoError(t, idx.loadQuantizeBounds(sqlproc, tblcfg, vt)) + + wantMul, wantAdd := quantizer.Int8Params(qmin, qmax) + if vt == types.T_array_uint8 { + wantMul, wantAdd = quantizer.Uint8Params(qmin, qmax) + } + require.Equal(t, wantMul, idx.QuantMul) + require.Equal(t, wantAdd, idx.QuantAdd) + } + + // bounds absent → params left at identity (1,0) + runSql = func(_ *sqlexec.SqlProcess, _ string) (executor.Result, error) { + return executor.Result{}, nil + } + idx := &IvfflatSearchIndex[float32]{QuantMul: 1, QuantAdd: 0} + require.NoError(t, idx.loadQuantizeBounds(sqlproc, tblcfg, types.T_array_int8)) + require.Equal(t, 1.0, idx.QuantMul) + require.Equal(t, 0.0, idx.QuantAdd) + + // sql error propagates + runSql = mock_runSql_parser_error + require.Error(t, (&IvfflatSearchIndex[float32]{}).loadQuantizeBounds(sqlproc, tblcfg, types.T_array_int8)) +} diff --git a/pkg/vectorindex/ivfpq/build_gpu.go b/pkg/vectorindex/ivfpq/build_gpu.go index 1dbdbd2dac5a5..cd67f4b8066f1 100644 --- a/pkg/vectorindex/ivfpq/build_gpu.go +++ b/pkg/vectorindex/ivfpq/build_gpu.go @@ -23,6 +23,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/sqlquote" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/cuvs" "github.com/matrixorigin/matrixone/pkg/vectorindex" ) @@ -31,44 +32,57 @@ import ( // When the current sub-index reaches IndexCapacity, it is finalized (Build called) and a // new sub-index is created, mirroring the CagraBuild pattern. // +// IvfpqBuild carries two element types: base/quantizer-source B (the decoded +// source column type — f32 or f16) and storage Q (the cuVS sub-index storage +// type). For a direct index B==Q; for a quantized index (e.g. vecf16 base -> +// int8 storage) B is the base type and Q the 1-byte storage type. +// // IvfpqBuild is single-threaded; the ivfpq_create table function runs with IsSingle=true. -type IvfpqBuild[T cuvs.VectorType] struct { +type IvfpqBuild[B, Q cuvs.VectorType] struct { uid string idxcfg vectorindex.IndexConfig tblcfg vectorindex.IndexTableConfig - indexes []*IvfpqModel[T] - current *IvfpqModel[T] + indexes []*IvfpqModel[B, Q] + current *IvfpqModel[B, Q] nthread uint32 devices []int count int64 idBuf [1]int64 + // (B, Q) routing tags computed once at construction. bIsHalf: the base + // type is f16. qIsHalf: the storage type is f16 (so a half base goes + // native rather than quantized). + bIsHalf bool + qIsHalf bool + // Filter column metadata (INCLUDE columns) — see CagraBuild.filterColMetaJSON. filterColMetaJSON string } -func NewIvfpqBuild[T cuvs.VectorType]( +func NewIvfpqBuild[B, Q cuvs.VectorType]( uid string, idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig, nthread uint32, devices []int, -) (*IvfpqBuild[T], error) { - return &IvfpqBuild[T]{ +) (*IvfpqBuild[B, Q], error) { + return &IvfpqBuild[B, Q]{ uid: uid, idxcfg: idxcfg, tblcfg: tblcfg, - indexes: make([]*IvfpqModel[T], 0, 4), + indexes: make([]*IvfpqModel[B, Q], 0, 4), nthread: nthread, devices: devices, + bIsHalf: cuvs.GetQuantization[B]() == cuvs.F16, + qIsHalf: cuvs.GetQuantization[Q]() == cuvs.F16, }, nil } -func (b *IvfpqBuild[T]) createKey(n int) string { +func (b *IvfpqBuild[B, Q]) createKey(n int) string { return fmt.Sprintf("%s:%d", b.uid, n) } -func (b *IvfpqBuild[T]) getOrCreateCurrent() (*IvfpqModel[T], error) { +func (b *IvfpqBuild[B, Q]) getOrCreateCurrent() (*IvfpqModel[B, Q], error) { capacity := b.idxcfg.IndexCapacity if b.current != nil && b.count >= capacity { @@ -82,7 +96,7 @@ func (b *IvfpqBuild[T]) getOrCreateCurrent() (*IvfpqModel[T], error) { if b.current == nil { key := b.createKey(len(b.indexes)) - m, err := NewIvfpqModelForBuild[T](key, b.idxcfg, b.nthread, b.devices) + m, err := NewIvfpqModelForBuild[B, Q](key, b.idxcfg, b.nthread, b.devices) if err != nil { return nil, err } @@ -104,32 +118,46 @@ func (b *IvfpqBuild[T]) getOrCreateCurrent() (*IvfpqModel[T], error) { } // SetFilterColumns — see cagra.CagraBuild.SetFilterColumns. -func (b *IvfpqBuild[T]) SetFilterColumns(colMetaJSON string) { +func (b *IvfpqBuild[B, Q]) SetFilterColumns(colMetaJSON string) { b.filterColMetaJSON = colMetaJSON } // AddFilterChunk — see cagra.CagraBuild.AddFilterChunk. -func (b *IvfpqBuild[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { +func (b *IvfpqBuild[B, Q]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { if b.current == nil { - return moerr.NewInternalErrorNoCtx("IvfpqBuild.AddFilterChunk: no current sub-index (call AddFloat first)") + return moerr.NewInternalErrorNoCtx("IvfpqBuild.AddFilterChunk: no current sub-index (call AddRow first)") } return b.current.Index.AddFilterChunk(colIdx, data, nullBitmap, nrows) } -func (b *IvfpqBuild[T]) AddFloat(id int64, vec []float32) error { +// AddRow buffers one source row. vecBytes is the raw little-endian base-type +// bytes of one vector (4*dim for an f32 base, 2*dim for an f16 base) — the +// non-generic ivfpqBuilder interface can't name the concrete element type B, so +// the bytes are reinterpreted here with UnsafeSliceCast (zero-copy, no per-row +// heap alloc). Routing by (B, Q): +// - f16 base, f16 storage (direct, Q==B): native AddChunk([]Q). +// - otherwise (f32 base, or f16 base -> int8/uint8): AddChunkQuantize([]B), +// which converts B -> Q on device (B==Q copy, or learned/cast quantizer). +func (b *IvfpqBuild[B, Q]) AddRow(id int64, vecBytes []byte) error { idx, err := b.getOrCreateCurrent() if err != nil { return err } b.idBuf[0] = id - if err = idx.AddChunkFloat(vec, 1, b.idBuf[:]); err != nil { + + if b.bIsHalf && b.qIsHalf { + err = idx.AddChunk(util.UnsafeSliceCast[Q](vecBytes), 1, b.idBuf[:]) + } else { + err = idx.AddChunkQuantize(util.UnsafeSliceCast[B](vecBytes), 1, b.idBuf[:]) + } + if err != nil { return err } b.count++ return nil } -func (b *IvfpqBuild[T]) ToInsertSql(ts int64) ([]string, error) { +func (b *IvfpqBuild[B, Q]) ToInsertSql(ts int64) ([]string, error) { if b.current != nil && b.count > 0 { if err := b.current.Build(); err != nil { return nil, err @@ -160,7 +188,7 @@ func (b *IvfpqBuild[T]) ToInsertSql(ts int64) ([]string, error) { return sqls, nil } -func (b *IvfpqBuild[T]) Destroy() error { +func (b *IvfpqBuild[B, Q]) Destroy() error { var errs error if b.current != nil { if err := b.current.Destroy(); err != nil { @@ -177,6 +205,6 @@ func (b *IvfpqBuild[T]) Destroy() error { return errs } -func (b *IvfpqBuild[T]) GetIndexes() []*IvfpqModel[T] { +func (b *IvfpqBuild[B, Q]) GetIndexes() []*IvfpqModel[B, Q] { return b.indexes } diff --git a/pkg/vectorindex/ivfpq/cdc_load_test.go b/pkg/vectorindex/ivfpq/cdc_load_test.go index 252ae3985e330..f1810c027088f 100644 --- a/pkg/vectorindex/ivfpq/cdc_load_test.go +++ b/pkg/vectorindex/ivfpq/cdc_load_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" @@ -60,7 +61,11 @@ func encodeChunk(t *testing.T, dim, includeBytesPerRow int, ops []cuvscdc.CdcOp, } insIdx++ } - out, err := cuvscdc.EncodeEventRecord(buf, op, pkids[i], v, inc, dim, includeBytesPerRow) + var vb []byte + if v != nil { + vb = util.UnsafeSliceToBytes(v) + } + out, err := cuvscdc.EncodeEventRecord(buf, op, pkids[i], vb, inc, 4*dim, includeBytesPerRow) require.NoError(t, err) buf = out } @@ -86,7 +91,7 @@ func TestLoadCdcEventsFromDB_RoundTrip(t *testing.T) { } defer func() { runSql = orig }() - idx := &IvfpqModel[float32]{Id: "idx-1"} + idx := &IvfpqModel[float32, float32]{Id: "idx-1"} got, err := idx.loadCdcEventsFromDB(sqlproc, tblcfg) require.NoError(t, err) require.Len(t, got, 1) @@ -105,7 +110,7 @@ func TestLoadCdcEventsFromDB_Empty(t *testing.T) { } defer func() { runSql = orig }() - idx := &IvfpqModel[float32]{Id: "idx-1"} + idx := &IvfpqModel[float32, float32]{Id: "idx-1"} got, err := idx.loadCdcEventsFromDB(sqlproc, testTblcfg()) require.NoError(t, err) require.Empty(t, got) @@ -121,7 +126,7 @@ func TestReplayEventChunks_DeleteInsertDelete(t *testing.T) { ) chunks := []cuvscdc.EventChunk{{ChunkId: 0, Data: chunkBytes}} - delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks(chunks, dim, 0) + delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks[float32](chunks, dim, 0) require.NoError(t, err) require.Equal(t, []int64{1}, delPkids) require.Empty(t, ovPkids) @@ -139,7 +144,7 @@ func TestReplayEventChunks_FlattenOverflow(t *testing.T) { ) chunks := []cuvscdc.EventChunk{{ChunkId: 0, Data: chunkBytes}} - delPkids, ovPkids, ovVecs, _, err := replayEventChunks(chunks, dim, 0) + delPkids, ovPkids, ovVecs, _, err := replayEventChunks[float32](chunks, dim, 0) require.NoError(t, err) require.Empty(t, delPkids) require.Equal(t, []int64{10, 20}, ovPkids) @@ -158,7 +163,7 @@ func TestReplayEventChunks_MultiChunkOrder(t *testing.T) { {ChunkId: 1, Data: chunk1}, {ChunkId: 0, Data: chunk0}, } - delPkids, ovPkids, _, _, err := replayEventChunks(chunks, dim, 0) + delPkids, ovPkids, _, _, err := replayEventChunks[float32](chunks, dim, 0) require.NoError(t, err) require.Equal(t, []int64{5}, delPkids) require.Empty(t, ovPkids) @@ -225,7 +230,7 @@ func TestLoadIndex_WithCdcDeltas(t *testing.T) { } defer func() { runSql = origRunSql }() - models, err := LoadMetadata[float32](sqlproc, tblcfg.DbName, tblcfg.MetadataTable) + models, err := LoadMetadata[float32, float32](sqlproc, tblcfg.DbName, tblcfg.MetadataTable) require.NoError(t, err) require.Equal(t, 1, len(models)) @@ -245,7 +250,7 @@ func TestLoadIndex_WithCdcDeltas(t *testing.T) { // prefilter should drop it from the result set. data := generateTestData(testNVectors, testDim) query := data[:testDim] - keys, _, err := idx.SearchF32(query, 1, 0) + keys, _, err := idx.SearchQuantize(query, 1, 0) require.NoError(t, err) require.Equal(t, 1, len(keys)) require.NotEqual(t, ids[0], keys[0], diff --git a/pkg/vectorindex/ivfpq/model_gpu.go b/pkg/vectorindex/ivfpq/model_gpu.go index cd1ae161f8449..7e3a04ee41552 100644 --- a/pkg/vectorindex/ivfpq/model_gpu.go +++ b/pkg/vectorindex/ivfpq/model_gpu.go @@ -28,6 +28,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/sqlquote" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/cuvs" "github.com/matrixorigin/matrixone/pkg/logutil" @@ -42,9 +43,9 @@ var runSql = sqlexec.RunSql var runSql_streaming = sqlexec.RunStreamingSql // IvfpqModel wraps a GpuIvfPq index and handles load/save to secondary index tables. -type IvfpqModel[T cuvs.VectorType] struct { +type IvfpqModel[B, Q cuvs.VectorType] struct { Id string - Index *cuvs.GpuIvfPq[T] + Index *cuvs.GpuIvfPq[B, Q] Path string FileSize int64 MaxCapacity uint64 @@ -67,10 +68,10 @@ type IvfpqModel[T cuvs.VectorType] struct { // CDC insert overflow — pkids that the replay left in the brute-force // overflow (INSERT record with no later DELETE). Brute-force searched at - // query time and merged with main-index results. Always F32 regardless - // of T. + // query time and merged with main-index results. Stored in the native + // base type B (f32 or f16), matching the base-typed overflow brute force. OverflowPkids []int64 - OverflowVecs []float32 // len = len(OverflowPkids) * dim + OverflowVecs []B // len = len(OverflowPkids) * dim // INCLUDE column data carried alongside each overflow row. Layout // matches the EncodeEventRecord INSERT-record include section: @@ -89,8 +90,8 @@ type IvfpqModel[T cuvs.VectorType] struct { OverflowColMetaJSON string } -func NewIvfpqModelForBuild[T cuvs.VectorType](id string, cfg vectorindex.IndexConfig, nthread uint32, devices []int) (*IvfpqModel[T], error) { - return &IvfpqModel[T]{ +func NewIvfpqModelForBuild[B, Q cuvs.VectorType](id string, cfg vectorindex.IndexConfig, nthread uint32, devices []int) (*IvfpqModel[B, Q], error) { + return &IvfpqModel[B, Q]{ Id: id, Idxcfg: cfg, NThread: nthread, @@ -98,7 +99,7 @@ func NewIvfpqModelForBuild[T cuvs.VectorType](id string, cfg vectorindex.IndexCo }, nil } -func (idx *IvfpqModel[T]) ivfpqConfig() (cuvsMetric cuvs.DistanceType, bp cuvs.IvfPqBuildParams, mode cuvs.DistributionMode, err error) { +func (idx *IvfpqModel[B, Q]) ivfpqConfig() (cuvsMetric cuvs.DistanceType, bp cuvs.IvfPqBuildParams, mode cuvs.DistributionMode, err error) { cfg := idx.Idxcfg.CuvsIvfpq var ok bool cuvsMetric, ok = metric.MetricTypeToCuvsMetric[metric.MetricType(cfg.Metric)] @@ -124,7 +125,7 @@ func (idx *IvfpqModel[T]) ivfpqConfig() (cuvsMetric cuvs.DistanceType, bp cuvs.I } // InitEmpty allocates the GPU buffer for totalCount vectors. -func (idx *IvfpqModel[T]) InitEmpty(totalCount uint64) error { +func (idx *IvfpqModel[B, Q]) InitEmpty(totalCount uint64) error { if idx.Index != nil { return moerr.NewInternalErrorNoCtx("IvfpqModel: index already initialized") } @@ -138,7 +139,7 @@ func (idx *IvfpqModel[T]) InitEmpty(totalCount uint64) error { if buildMode == cuvs.Replicated { buildMode = cuvs.SingleGpu } - gi, err := cuvs.NewGpuIvfPqEmpty[T]( + gi, err := cuvs.NewGpuIvfPqEmpty[B, Q]( totalCount, uint32(idx.Idxcfg.CuvsIvfpq.Dimensions), cuvsMetric, @@ -159,18 +160,35 @@ func (idx *IvfpqModel[T]) InitEmpty(totalCount uint64) error { return nil } -func (idx *IvfpqModel[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []int64) error { +// AddChunk appends a chunk of native storage-type (T) vectors with no +// quantization — used when the base column type equals the storage type +// (e.g. a vecf16 base stored as half). Mirrors AddChunkFloat but raw. +func (idx *IvfpqModel[B, Q]) AddChunk(chunk []Q, chunkCount uint64, ids []int64) error { if idx.Index == nil { return moerr.NewInternalErrorNoCtx("IvfpqModel: index not initialized; call InitEmpty first") } - if err := idx.Index.AddChunkFloat(chunk, chunkCount, ids); err != nil { + if err := idx.Index.AddChunk(chunk, chunkCount, ids); err != nil { return err } idx.Len += int64(chunkCount) return nil } -func (idx *IvfpqModel[T]) Build() error { +// AddChunkQuantize appends a chunk of base-typed (B) vectors, quantizing +// natively to the 1-byte storage type Q (int8/uint8). Used for a vecf16 base +// with QUANTIZATION=int8/uint8 — no f32 detour. +func (idx *IvfpqModel[B, Q]) AddChunkQuantize(chunk []B, chunkCount uint64, ids []int64) error { + if idx.Index == nil { + return moerr.NewInternalErrorNoCtx("IvfpqModel: index not initialized; call InitEmpty first") + } + if err := idx.Index.AddChunkQuantize(chunk, chunkCount, ids); err != nil { + return err + } + idx.Len += int64(chunkCount) + return nil +} + +func (idx *IvfpqModel[B, Q]) Build() error { if idx.Index == nil { return moerr.NewInternalErrorNoCtx("IvfpqModel: index not initialized") } @@ -181,7 +199,7 @@ func (idx *IvfpqModel[T]) Build() error { return nil } -func (idx *IvfpqModel[T]) Destroy() error { +func (idx *IvfpqModel[B, Q]) Destroy() error { if idx.Index != nil { if err := idx.Index.Destroy(); err != nil { return err @@ -195,7 +213,7 @@ func (idx *IvfpqModel[T]) Destroy() error { return nil } -func (idx *IvfpqModel[T]) saveToFile() error { +func (idx *IvfpqModel[B, Q]) saveToFile() error { if idx.Index == nil { return nil } @@ -246,7 +264,7 @@ func (idx *IvfpqModel[T]) saveToFile() error { return nil } -func (idx *IvfpqModel[T]) ToSql(cfg vectorindex.IndexTableConfig) ([]string, error) { +func (idx *IvfpqModel[B, Q]) ToSql(cfg vectorindex.IndexTableConfig) ([]string, error) { if err := idx.saveToFile(); err != nil { return nil, err } @@ -306,16 +324,17 @@ func joinStrings(ss []string, sep string) string { return result } -func (idx *IvfpqModel[T]) Empty() bool { +func (idx *IvfpqModel[B, Q]) Empty() bool { return idx.Len == 0 } -func (idx *IvfpqModel[T]) Full() bool { +func (idx *IvfpqModel[B, Q]) Full() bool { return idx.MaxCapacity > 0 && uint64(idx.Len) >= idx.MaxCapacity } -// SearchF32 performs a KNN search using a float32 query vector. -func (idx *IvfpqModel[T]) SearchF32(query []float32, limit uint32, nprobes uint32) (keys []int64, distances []float32, err error) { +// SearchQuantize performs a KNN search using a base-typed (B) query vector; the +// index converts B -> its storage type Q on device (was SearchF32, f32-only). +func (idx *IvfpqModel[B, Q]) SearchQuantize(query []B, limit uint32, nprobes uint32) (keys []int64, distances []float32, err error) { if idx.Index == nil { return nil, nil, moerr.NewInternalErrorNoCtx("IvfpqModel: index not loaded") } @@ -326,14 +345,14 @@ func (idx *IvfpqModel[T]) SearchF32(query []float32, limit uint32, nprobes uint3 if sp.NProbes == 0 { sp = cuvs.DefaultIvfPqSearchParams() } - res, err := idx.Index.SearchFloat(query, 1, uint32(idx.Idxcfg.CuvsIvfpq.Dimensions), limit, sp) + res, err := idx.Index.SearchQuantize(query, 1, uint32(idx.Idxcfg.CuvsIvfpq.Dimensions), limit, sp) if err != nil { return nil, nil, err } return res.Neighbors, res.Distances, nil } -func (idx *IvfpqModel[T]) Search(query []T, limit uint32, nprobes uint32) (keys []int64, distances []float32, err error) { +func (idx *IvfpqModel[B, Q]) Search(query []Q, limit uint32, nprobes uint32) (keys []int64, distances []float32, err error) { if idx.Index == nil { return nil, nil, moerr.NewInternalErrorNoCtx("IvfpqModel: index not loaded") } @@ -351,7 +370,7 @@ func (idx *IvfpqModel[T]) Search(query []T, limit uint32, nprobes uint32) (keys return res.Neighbors, res.Distances, nil } -func (idx *IvfpqModel[T]) loadChunk(ctx context.Context, +func (idx *IvfpqModel[B, Q]) loadChunk(ctx context.Context, sqlproc *sqlexec.SqlProcess, stream_chan chan executor.Result, error_chan chan error, @@ -395,7 +414,7 @@ func (idx *IvfpqModel[T]) loadChunk(ctx context.Context, // the storage table in parallel, then unpacks the tar onto the GPU, replays // the event log to derive (deleted, overflow), and applies the deletes via // Index.DeleteIds. -func (idx *IvfpqModel[T]) LoadIndex( +func (idx *IvfpqModel[B, Q]) LoadIndex( sqlproc *sqlexec.SqlProcess, idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig, @@ -529,7 +548,7 @@ func (idx *IvfpqModel[T]) LoadIndex( return err } - gi, err := cuvs.NewGpuIvfPqEmpty[T]( + gi, err := cuvs.NewGpuIvfPqEmpty[B, Q]( uint64(idxcfg.IndexCapacity), uint32(idxcfg.CuvsIvfpq.Dimensions), cuvsMetric, @@ -565,7 +584,7 @@ func (idx *IvfpqModel[T]) LoadIndex( } includeBytesPerRow = ibpr } - delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks(eventChunks, dim, includeBytesPerRow) + delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks[B](eventChunks, dim, includeBytesPerRow) if err != nil { gi.Destroy() return err @@ -602,7 +621,7 @@ func (idx *IvfpqModel[T]) LoadIndex( // loadCdcEventsFromDB reads the tag=1 event-log rows for this index. See // pkg/vectorindex/cagra/model_gpu.go for design notes. -func (idx *IvfpqModel[T]) loadCdcEventsFromDB( +func (idx *IvfpqModel[B, Q]) loadCdcEventsFromDB( sqlproc *sqlexec.SqlProcess, tblcfg vectorindex.IndexTableConfig, ) ([]cuvscdc.EventChunk, error) { @@ -633,16 +652,20 @@ func (idx *IvfpqModel[T]) loadCdcEventsFromDB( // replayEventChunks sorts the chunks by chunk_id, replays the records, and // flattens (deleted, overflow) into the parallel slices the IvfpqModel // struct carries (the layout buildOverflow consumes). -func replayEventChunks( +func replayEventChunks[B cuvs.VectorType]( chunks []cuvscdc.EventChunk, dim int, includeBytesPerRow int, -) ([]int64, []int64, []float32, []byte, error) { +) ([]int64, []int64, []B, []byte, error) { if len(chunks) == 0 { return nil, nil, nil, nil, nil } cuvscdc.SortChunks(chunks) - state, err := cuvscdc.ReplayEventLog(chunks, dim, includeBytesPerRow) + // The codec stores vectors as opaque bytes; the per-row byte length is + // dim * sizeof(B). Reinterpret each row's bytes back to the native base + // type B for the overflow brute force — no f32 detour. + vecBytesPerRow := dim * int(util.UnsafeSizeOf[B]()) + state, err := cuvscdc.ReplayEventLog(chunks, vecBytesPerRow, includeBytesPerRow) if err != nil { return nil, nil, nil, nil, err } @@ -654,14 +677,15 @@ func replayEventChunks( return deletedPkids, nil, nil, nil, nil } ovPkids := make([]int64, len(state.Overflow)) - ovVecs := make([]float32, len(state.Overflow)*dim) + ovVecs := make([]B, len(state.Overflow)*dim) + ovVecBytes := util.UnsafeSliceToBytes(ovVecs) var ovInc []byte if includeBytesPerRow > 0 { ovInc = make([]byte, len(state.Overflow)*includeBytesPerRow) } for i, e := range state.Overflow { ovPkids[i] = e.Pkid - copy(ovVecs[i*dim:(i+1)*dim], e.Vec) + copy(ovVecBytes[i*vecBytesPerRow:(i+1)*vecBytesPerRow], e.Vec) if includeBytesPerRow > 0 { copy(ovInc[i*includeBytesPerRow:(i+1)*includeBytesPerRow], e.Include) } @@ -669,7 +693,7 @@ func replayEventChunks( return deletedPkids, ovPkids, ovVecs, ovInc, nil } -func (idx *IvfpqModel[T]) Unload() error { +func (idx *IvfpqModel[B, Q]) Unload() error { if idx.Index == nil { return nil } @@ -688,7 +712,7 @@ func (idx *IvfpqModel[T]) Unload() error { } // LoadMetadata loads IvfpqModel descriptors from the metadata table. -func LoadMetadata[T cuvs.VectorType](sqlproc *sqlexec.SqlProcess, dbname string, metatbl string) ([]*IvfpqModel[T], error) { +func LoadMetadata[B, Q cuvs.VectorType](sqlproc *sqlexec.SqlProcess, dbname string, metatbl string) ([]*IvfpqModel[B, Q], error) { sql := fmt.Sprintf("SELECT * FROM %s ORDER BY timestamp ASC", sqlquote.QualifiedIdent(dbname, metatbl)) res, err := runSql(sqlproc, sql) if err != nil { @@ -701,7 +725,7 @@ func LoadMetadata[T cuvs.VectorType](sqlproc *sqlexec.SqlProcess, dbname string, total += bat.RowCount() } - indexes := make([]*IvfpqModel[T], 0, total) + indexes := make([]*IvfpqModel[B, Q], 0, total) for _, bat := range res.Batches { idVec := bat.Vecs[0] chksumVec := bat.Vecs[1] @@ -712,7 +736,7 @@ func LoadMetadata[T cuvs.VectorType](sqlproc *sqlexec.SqlProcess, dbname string, chksum := chksumVec.GetStringAt(i) ts := vector.GetFixedAtWithTypeCheck[int64](tsVec, i) fs := vector.GetFixedAtWithTypeCheck[int64](fsVec, i) - idx := &IvfpqModel[T]{Id: id, Checksum: chksum, Timestamp: ts, FileSize: fs} + idx := &IvfpqModel[B, Q]{Id: id, Checksum: chksum, Timestamp: ts, FileSize: fs} indexes = append(indexes, idx) } } @@ -720,7 +744,7 @@ func LoadMetadata[T cuvs.VectorType](sqlproc *sqlexec.SqlProcess, dbname string, } // ToDeleteSql generates DELETE SQL for storage and metadata tables. -func (idx *IvfpqModel[T]) ToDeleteSql(cfg vectorindex.IndexTableConfig) ([]string, error) { +func (idx *IvfpqModel[B, Q]) ToDeleteSql(cfg vectorindex.IndexTableConfig) ([]string, error) { sqls := make([]string, 0, 2) sqls = append(sqls, fmt.Sprintf("DELETE FROM %s WHERE %s = %s", sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), catalog.Ivfpq_TblCol_Storage_Index_Id, sqlquote.String(idx.Id))) diff --git a/pkg/vectorindex/ivfpq/model_test.go b/pkg/vectorindex/ivfpq/model_test.go index 8b41fb7e8251b..f2b9bd66859fb 100644 --- a/pkg/vectorindex/ivfpq/model_test.go +++ b/pkg/vectorindex/ivfpq/model_test.go @@ -128,7 +128,7 @@ func makeIndexBatch(proc *process.Process, tarPath string) *batch.Batch { // buildTestModel builds an IvfpqModel, calls Build, and saves via ToSql. // Index is nil after ToSql (GPU memory freed). Path/Checksum/FileSize are set. -func buildTestModel(t *testing.T, id string, ids []int64) *IvfpqModel[float32] { +func buildTestModel(t *testing.T, id string, ids []int64) *IvfpqModel[float32, float32] { t.Helper() idxcfg := testIdxcfg() @@ -141,13 +141,13 @@ func buildTestModel(t *testing.T, id string, ids []int64) *IvfpqModel[float32] { } } - m, err := NewIvfpqModelForBuild[float32](id, idxcfg, 1, []int{0}) + m, err := NewIvfpqModelForBuild[float32, float32](id, idxcfg, 1, []int{0}) require.NoError(t, err) err = m.InitEmpty(testNVectors) require.NoError(t, err) - err = m.AddChunkFloat(data, testNVectors, ids) + err = m.AddChunkQuantize(data, testNVectors, ids) require.NoError(t, err) err = m.Build() @@ -185,7 +185,7 @@ func TestModelStreamError(t *testing.T) { } defer func() { runSql = origRunSql }() - idx := &IvfpqModel[float32]{ + idx := &IvfpqModel[float32, float32]{ Id: "test-stream-err", FileSize: 1024, Checksum: "fake-checksum", @@ -212,13 +212,13 @@ func TestModelBuildAndLoad(t *testing.T) { } // ---- Build ---- - built, err := NewIvfpqModelForBuild[float32]("test-build", idxcfg, 1, []int{0}) + built, err := NewIvfpqModelForBuild[float32, float32]("test-build", idxcfg, 1, []int{0}) require.NoError(t, err) err = built.InitEmpty(testNVectors) require.NoError(t, err) - err = built.AddChunkFloat(data, testNVectors, ids) + err = built.AddChunkQuantize(data, testNVectors, ids) require.NoError(t, err) err = built.Build() @@ -248,7 +248,7 @@ func TestModelBuildAndLoad(t *testing.T) { defer func() { runSql = origRunSql }() // ---- Load from local tar ---- - loader := &IvfpqModel[float32]{ + loader := &IvfpqModel[float32, float32]{ Id: "test-build", Path: tarPath, Checksum: checksum, @@ -267,7 +267,7 @@ func TestModelBuildAndLoad(t *testing.T) { // ---- Search ---- query := data[:testDim] - keys, dists, err := loader.SearchF32(query, 1, 0) + keys, dists, err := loader.SearchQuantize(query, 1, 0) require.NoError(t, err) require.Equal(t, 1, len(keys)) require.Equal(t, 1, len(dists)) @@ -335,7 +335,7 @@ func TestModelLoadFromDB(t *testing.T) { } defer func() { runSql = origRunSql }() - models, err := LoadMetadata[float32](sqlproc, tblcfg.DbName, tblcfg.MetadataTable) + models, err := LoadMetadata[float32, float32](sqlproc, tblcfg.DbName, tblcfg.MetadataTable) require.NoError(t, err) require.Equal(t, 1, len(models)) @@ -350,7 +350,7 @@ func TestModelLoadFromDB(t *testing.T) { data := generateTestData(testNVectors, testDim) query := data[:testDim] - keys, dists, err := idx.SearchF32(query, 1, 0) + keys, dists, err := idx.SearchQuantize(query, 1, 0) require.NoError(t, err) require.Equal(t, 1, len(keys)) fmt.Printf("LoadFromDB SearchF32: keys=%v dists=%v\n", keys, dists) @@ -360,7 +360,7 @@ func TestModelLoadFromDB(t *testing.T) { func TestModelNil(t *testing.T) { var tblcfg vectorindex.IndexTableConfig - idx := &IvfpqModel[float32]{} + idx := &IvfpqModel[float32, float32]{} // InitEmpty fails because Devices is empty. err := idx.InitEmpty(10) @@ -372,16 +372,16 @@ func TestModelNil(t *testing.T) { require.NotNil(t, err) // AddChunkFloat fails because Index is nil. - err = idx.AddChunkFloat([]float32{1, 2, 3, 4}, 1, []int64{1}) + err = idx.AddChunkQuantize([]float32{1, 2, 3, 4}, 1, []int64{1}) require.NotNil(t, err) // SearchF32 fails because Index is nil. - _, _, err = idx.SearchF32([]float32{0, 0, 0, 0}, 1, 0) + _, _, err = idx.SearchQuantize([]float32{0, 0, 0, 0}, 1, 0) require.NotNil(t, err) // SearchF32 with nil query fails. - idx2 := &IvfpqModel[float32]{} - _, _, err = idx2.SearchF32(nil, 1, 0) + idx2 := &IvfpqModel[float32, float32]{} + _, _, err = idx2.SearchQuantize(nil, 1, 0) require.NotNil(t, err) // ToSql on a never-built model returns empty. @@ -411,7 +411,7 @@ func TestModelEmptyBuild(t *testing.T) { idxcfg := testIdxcfg() tblcfg := testTblcfg() - built, err := NewIvfpqModelForBuild[float32]("test-empty", idxcfg, 1, []int{0}) + built, err := NewIvfpqModelForBuild[float32, float32]("test-empty", idxcfg, 1, []int{0}) require.NoError(t, err) // Not dirty → ToSql returns empty slice. diff --git a/pkg/vectorindex/ivfpq/plugin/compile/compile.go b/pkg/vectorindex/ivfpq/plugin/compile/compile.go index a8695affe0e81..a6c25b739d153 100644 --- a/pkg/vectorindex/ivfpq/plugin/compile/compile.go +++ b/pkg/vectorindex/ivfpq/plugin/compile/compile.go @@ -120,7 +120,7 @@ func (h Hooks) HandleCreateIndex(ctx compileplugin.CompileContext, indexDefs map // branch builds ivfpq_create synchronously inside the txn so the new // tag=0 model lands before subsequent steps observe the index — mirrors // IVF-FLAT and CAGRA. -func (h Hooks) HandleReindex(ctx compileplugin.CompileContext, indexDefs map[string]*plan.IndexDef, forceSync bool) error { +func (h Hooks) HandleReindex(ctx compileplugin.CompileContext, indexDefs map[string]*plan.IndexDef, forceSync bool, _ bool) error { return h.handleCreate(ctx, indexDefs, forceSync) } @@ -279,14 +279,30 @@ func registerIdxcronUpdate( // IVF-PQ supports updating `lists` at REINDEX time — mirrors IVF-FLAT // since both algorithms key on the inverted-list count for their build. func (Hooks) ValidateReindexParams(old map[string]string, alter compileplugin.ReindexParamUpdate) (map[string]string, error) { - return compileplugin.MergeReindexParams(old, alter, "ivfpq", + // Merge first, then validate the EFFECTIVE quantization via the per-algo + // catalog hook (the single home shared with CREATE). The merged map is the + // index's actual post-reindex config: the value the reindex set, or — when + // the reindex omitted QUANTIZATION (e.g. the idxcron-issued rebuild) — the + // value already stored on the index. Validating the merge (not the raw alter + // delta) means the check is never skipped just because the statement omitted + // quantization, and quantization and op_type come from one consistent source. + merged, err := compileplugin.MergeReindexParams(old, alter, "ivfpq", catalog.IndexAlgoParamLists, catalog.IndexAlgoParamKmeansTrainPercent, catalog.IndexAlgoParamKmeansMaxIteration, catalog.IndexAlgoParamMaxIndexCapacity, catalog.HnswM, catalog.BitsPerCode, + catalog.Quantization, ) + if err != nil { + return nil, err + } + if err := (ivfpqruntime.CatalogHooks{}).ValidQuantization( + merged[catalog.Quantization], merged[catalog.IndexAlgoParamOpType]); err != nil { + return nil, err + } + return merged, nil } // HandleDropIndex runs algorithm-specific cleanup beyond the generic @@ -299,6 +315,12 @@ func (Hooks) ValidateReindexParams(old map[string]string, alter compileplugin.Re // so this is a no-op. Compare HNSW, which does maintain CDC tasks. func (Hooks) HandleDropIndex(_ compileplugin.CompileContext, defs map[string]*plan.IndexDef) error { logutil.Infof("[plugin] ivfpq HandleDropIndex: defs=%d", len(defs)) + // Evict the cached search index so its GPU resources are freed NOW, rather + // than lingering until the 5-min VectorIndexCacheTTL housekeeping reaps it. + // Mirrors the create-side cache.Cache.Remove(storageDef.IndexTableName). + if storageDef, ok := defs[catalog.Ivfpq_TblType_Storage]; ok { + cache.Cache.Remove(storageDef.IndexTableName) + } return nil } diff --git a/pkg/vectorindex/ivfpq/plugin/compile/compile_test.go b/pkg/vectorindex/ivfpq/plugin/compile/compile_test.go index 94bad4e94640f..96ab01f85aebd 100644 --- a/pkg/vectorindex/ivfpq/plugin/compile/compile_test.go +++ b/pkg/vectorindex/ivfpq/plugin/compile/compile_test.go @@ -254,6 +254,36 @@ func TestIvfpqValidateReindexParams_Unsupported(t *testing.T) { require.Contains(t, err.Error(), catalog.HnswEfConstruction) } +// TestIvfpqValidateReindexParams_Quantization: IVF-PQ (cuvs) accepts the +// cuvs-supported quantization names and rejects others (e.g. bf16). +func TestIvfpqValidateReindexParams_Quantization(t *testing.T) { + got, err := Hooks{}.ValidateReindexParams(nil, compileplugin.ReindexParamUpdate{ + Params: map[string]string{catalog.Quantization: "int8"}, + }) + require.NoError(t, err) + require.Equal(t, "int8", got[catalog.Quantization]) + + _, err = Hooks{}.ValidateReindexParams(nil, compileplugin.ReindexParamUpdate{ + Params: map[string]string{catalog.Quantization: "bf16"}, + }) + require.Error(t, err) + + // int8/uint8 on a non-L2 (inner-product) index IS rejected at REINDEX via the + // ValidQuantization hook: the merged op_type is inner-product and the + // int8/uint8 affine quantizer only preserves L2 geometry. + _, err = Hooks{}.ValidateReindexParams( + map[string]string{catalog.IndexAlgoParamOpType: "vector_ip_ops"}, + compileplugin.ReindexParamUpdate{Params: map[string]string{catalog.Quantization: "uint8"}}) + require.Error(t, err) + + // ...but uint8 with L2 (the merged op_type) is accepted. + got, err = Hooks{}.ValidateReindexParams( + map[string]string{catalog.IndexAlgoParamOpType: "vector_l2_ops"}, + compileplugin.ReindexParamUpdate{Params: map[string]string{catalog.Quantization: "uint8"}}) + require.NoError(t, err) + require.Equal(t, "uint8", got[catalog.Quantization]) +} + func TestIvfpqHandleDropIndex(t *testing.T) { require.NoError(t, Hooks{}.HandleDropIndex(nil, nil)) } @@ -390,7 +420,7 @@ func TestIvfpqHandleCreateIndex_AsyncFalseExplicit(t *testing.T) { func TestIvfpqHandleCreateIndex_BackgroundReentry(t *testing.T) { ctx := newHandleCtx(true) ctx.stubCompileContext.isFrontend = false - err := Hooks{}.HandleReindex(ctx, ivfpqIndexDefs(), true) + err := Hooks{}.HandleReindex(ctx, ivfpqIndexDefs(), true, false) require.NoError(t, err) require.True(t, ctx.stubCompileContext.lastCdcTask.called, "background re-entry still drives the CDC task") require.False(t, ctx.stubCompileContext.lastIdxcronUpdate.called, "background re-entry must NOT rewrite mo_index_update") @@ -400,6 +430,6 @@ func TestIvfpqHandleReindex_DelegatesToCreate(t *testing.T) { // HandleReindex is a thin pass-through to handleCreate; honors // the forceSync arg directly (unlike HandleCreateIndex, which // now reads catalog.IsIndexAsync). - err := Hooks{}.HandleReindex(newHandleCtx(true), ivfpqIndexDefs(), false) + err := Hooks{}.HandleReindex(newHandleCtx(true), ivfpqIndexDefs(), false, false) require.NoError(t, err) } diff --git a/pkg/vectorindex/ivfpq/plugin/iscp/iscp.go b/pkg/vectorindex/ivfpq/plugin/iscp/iscp.go index 24adcae0a1103..c444f71779131 100644 --- a/pkg/vectorindex/ivfpq/plugin/iscp/iscp.go +++ b/pkg/vectorindex/ivfpq/plugin/iscp/iscp.go @@ -49,6 +49,6 @@ func (Hooks) Run(c *iscppkg.IndexConsumer, ctx context.Context, errch chan error iscppkg.RunCuvs(c, ctx, errch, r, func(sqlproc *sqlexec.SqlProcess) (iscppkg.CuvsSync, error) { w := c.SqlWriter().(*iscppkg.CuvsCdcWriter) return ivfpq.NewIvfpqSync(sqlproc, w.DbName(), w.TblName(), w.IndexName(), - w.IndexDef(), w.Dimension(), w.ColMetaJSON()) + w.IndexDef(), w.Dimension(), w.BaseVectorType(), w.ColMetaJSON()) }) } diff --git a/pkg/vectorindex/ivfpq/plugin/plan/plan_test.go b/pkg/vectorindex/ivfpq/plugin/plan/plan_test.go index 9eabac4a943e4..45e2f7b9bd268 100644 --- a/pkg/vectorindex/ivfpq/plugin/plan/plan_test.go +++ b/pkg/vectorindex/ivfpq/plugin/plan/plan_test.go @@ -168,6 +168,62 @@ func TestBuildSecondaryIndexDefs_OK(t *testing.T) { require.NotNil(t, tblDefs[1].Pkey) } +// indexOnQuant builds a single-column *tree.Index over colName with a +// QUANTIZATION option. +func indexOnQuant(colName, quant string) *tree.Index { + idx := indexOn(colName) + idx.IndexOption = &tree.IndexOption{Quantization: quant} + return idx +} + +// f16ColMap returns a colMap with an int64 pk and a vecf16 base column. +func f16ColMap(pkName, vecName string) map[string]*plan.ColDef { + m := vecColMap(pkName, vecName) + m[vecName].Typ.Id = int32(types.T_array_float16) + return m +} + +// TestBuildSecondaryIndexDefs_F16Base: a vecf16 base column is accepted. +func TestBuildSecondaryIndexDefs_F16Base(t *testing.T) { + idxDefs, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOn("vec"), f16ColMap("id", "vec"), nil, "id") + require.NoError(t, err) + require.Len(t, idxDefs, 2) +} + +// TestBuildSecondaryIndexDefs_UnsupportedBase: only vecf32 / vecf16 are valid +// base columns; vecf64 / vecbf16 / vecint8 / vecuint8 are rejected. +func TestBuildSecondaryIndexDefs_UnsupportedBase(t *testing.T) { + for _, oid := range []types.T{ + types.T_array_float64, types.T_array_bf16, types.T_array_int8, types.T_array_uint8, + } { + colMap := vecColMap("id", "vec") + colMap["vec"].Typ.Id = int32(oid) + _, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOn("vec"), colMap, nil, "id") + require.Error(t, err, "base type %s must be rejected", oid) + } +} + +// TestBuildSecondaryIndexDefs_F16UpcastRejected: vecf16 base + QUANTIZATION +// float32 is an upcast (4 > 2 bytes) and must be rejected by the downcast +// guard. (The accepted downcast path — f16 -> int8/uint8 — is exercised +// end-to-end by the GPU functional BVT, since the full def build past the +// guard needs a richer compiler context than this stub provides.) +func TestBuildSecondaryIndexDefs_F16UpcastRejected(t *testing.T) { + _, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOnQuant("vec", "float32"), f16ColMap("id", "vec"), nil, "id") + require.Error(t, err) +} + +// TestBuildSecondaryIndexDefs_BF16QuantRejected: QUANTIZATION 'bf16' has no GPU +// bfloat16 storage (cuVS has no bfloat16 index/quantizer), so it must be rejected +// rather than silently falling back to f32 storage — even though it passes the +// downcast width guard (bf16 is 2 bytes). Rejected on both f32 and f16 bases. +func TestBuildSecondaryIndexDefs_BF16QuantRejected(t *testing.T) { + _, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOnQuant("vec", "bf16"), vecColMap("id", "vec"), nil, "id") + require.Error(t, err, "f32 base + bf16 quant must be rejected") + _, _, err = Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOnQuant("vec", "bf16"), f16ColMap("id", "vec"), nil, "id") + require.Error(t, err, "f16 base + bf16 quant must be rejected") +} + // --- schema.go: BuildFullTextIndexDefs ------------------------------------- func TestBuildFullTextIndexDefs_Unsupported(t *testing.T) { diff --git a/pkg/vectorindex/ivfpq/plugin/plan/schema.go b/pkg/vectorindex/ivfpq/plugin/plan/schema.go index f4b291ba03264..4c24e61b4c4b7 100644 --- a/pkg/vectorindex/ivfpq/plugin/plan/schema.go +++ b/pkg/vectorindex/ivfpq/plugin/plan/schema.go @@ -24,6 +24,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" "github.com/matrixorigin/matrixone/pkg/sql/util" ivfpqrt "github.com/matrixorigin/matrixone/pkg/vectorindex/ivfpq/plugin/runtime" + "github.com/matrixorigin/matrixone/pkg/vectorindex/quantizer" ) // ivfpqCatalogHooks is the shared (stateless) catalog-hooks instance used for @@ -88,7 +89,40 @@ func (Hooks) BuildSecondaryIndexDefs( return nil, nil, moerr.NewInvalidInputf(ctx.GetContext(), "column '%s' is not exist", indexInfo.KeyParts[0].ColName.ColNameOrigin()) } if !catalogplugin.SupportsVectorType(ivfpqCatalogHooks, types.T(colMap[name].Typ.Id)) { - return nil, nil, moerr.NewNotSupported(ctx.GetContext(), "IvfPQ only supports VECF32 column types") + return nil, nil, moerr.NewNotSupported(ctx.GetContext(), "IvfPQ only supports VECF32 / VECF16 base column types") + } + // QUANTIZATION is downcast-only: the storage element must be the same width + // or narrower than the base column (f16 base -> int8/uint8 OK; f16 base -> + // float32 is an upcast and rejected). Mirrors ivfflat's guard. + if indexInfo.IndexOption != nil && indexInfo.IndexOption.Quantization != "" { + if qt, ok := quantizer.ToVectorType(indexInfo.IndexOption.Quantization); ok { + // bf16 storage does not exist on the GPU (cuVS/cgo has no bfloat16 + // index or quantizer), so reject it explicitly rather than silently + // falling back to f32 storage. Supported cuvs storage = f16/int8/uint8. + if qt == types.T_array_bf16 { + return nil, nil, moerr.NewNotSupportedf(ctx.GetContext(), + "IvfPQ does not support '%s' quantization (no GPU bfloat16 storage); use 'float16', 'int8', or 'uint8'", + indexInfo.IndexOption.Quantization) + } + baseSize := types.Type{Oid: types.T(colMap[name].Typ.Id)}.GetArrayElementSize() + quantSize := types.Type{Oid: qt}.GetArrayElementSize() + if quantSize > baseSize { + return nil, nil, moerr.NewNotSupportedf(ctx.GetContext(), + "IvfPQ QUANTIZATION '%s' (%d bytes/element) cannot upcast base column %s (%d bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type", + indexInfo.IndexOption.Quantization, quantSize, + types.T(colMap[name].Typ.Id).String(), baseSize) + } + // int8/uint8 quantization is L2-only (the affine quantizer breaks + // inner-product / cosine geometry). Gated by the per-algo catalog + // hook — the single home shared with REINDEX + // (compile/ValidateReindexParams) — so CREATE and REINDEX cannot + // drift. (bf16 and width/upcast are rejected above with base-column- + // aware messages before reaching here.) + if err := ivfpqCatalogHooks.ValidQuantization( + indexInfo.IndexOption.Quantization, indexInfo.IndexOption.AlgoParamVectorOpType); err != nil { + return nil, nil, err + } + } } for _, existedIndex := range existedIndexes { if existedIndex.IndexAlgo == catalog.MoIndexIvfpqAlgo.ToString() && existedIndex.Parts[0] == name { diff --git a/pkg/vectorindex/ivfpq/plugin/runtime/runtime.go b/pkg/vectorindex/ivfpq/plugin/runtime/runtime.go index ea8b5144ff34f..70df636622bf6 100644 --- a/pkg/vectorindex/ivfpq/plugin/runtime/runtime.go +++ b/pkg/vectorindex/ivfpq/plugin/runtime/runtime.go @@ -126,12 +126,43 @@ func (CatalogHooks) ExperimentalFlag() string { return IvfpqIndexFlag } // "vector_l2_ops") to a stable internal identifier. Used by plan-side // op_type validation when matching an ORDER BY distance function against // the index's declared op_type. -// SupportedVectorTypes: IVF-PQ (cuvs) indexes f32 vectors only. -func (CatalogHooks) SupportedVectorTypes() []types.T { return []types.T{types.T_array_float32} } +// SupportedVectorTypes: IVF-PQ (cuvs) accepts f32 and f16 base columns. f16 is +// stored natively as half, or downcast-quantized to int8/uint8 via QUANTIZATION. +// int8/uint8 base columns are unsupported (the CDC overflow brute force is f32/f16-only). +func (CatalogHooks) SupportedVectorTypes() []types.T { + return []types.T{types.T_array_float32, types.T_array_float16} +} // SupportedPrimaryKeyTypes: requires an int64 primary key. func (CatalogHooks) SupportedPrimaryKeyTypes() []types.T { return []types.T{types.T_int64} } +// ValidQuantization gates the (quantization, op_type) pair for IVF-PQ (cuvs): +// the value must be a cuvs storage type (float32/float16/int8/uint8 — bf16 and +// float64 are absent from CuvsQuantizationNameToType), and the 1-byte int8/uint8 +// scalar quantizer is L2-only (its affine map q(x)=scalar*x+offset preserves L2 +// ordering — the offset cancels in a difference — but biases inner-product and +// rotates cosine angles). One home for CREATE (plan/schema) and REINDEX +// (compile/ValidateReindexParams). quant=="" => no quantization (valid); op=="" +// => value rule only. +func (CatalogHooks) ValidQuantization(quant, op string) error { + if quant == "" { + return nil + } + quant = strings.ToLower(quant) + if !metric.ValidQuantization(quant) { + return moerr.NewNotSupportedNoCtxf( + "ivfpq quantization %q (supported: float32, float16, int8, uint8)", quant) + } + if quant == metric.Quantization_INT8_Str || quant == metric.Quantization_UINT8_Str { + switch strings.ToLower(op) { + case metric.OpType_InnerProduct, metric.OpType_CosineDistance: + return moerr.NewNotSupportedNoCtxf( + "ivfpq quantization %q is only supported with L2 (op_type 'vector_l2_ops'); the int8/uint8 affine quantizer does not preserve inner-product / cosine geometry", quant) + } + } + return nil +} + // SupportedIncludeColumnTypes: cuvs INCLUDE (pre-filter) columns accept // int32/int64/float32/float64 scalars. func (CatalogHooks) SupportedIncludeColumnTypes() []types.T { @@ -234,6 +265,9 @@ func (CatalogHooks) ParamsFromTree(idx *tree.Index) (map[string]string, error) { if idx.IndexOption.Hour > 0 { res[catalog.Hour] = strconv.FormatInt(idx.IndexOption.Hour, 10) } + if idx.IndexOption.Second > 0 { + res[catalog.Second] = strconv.FormatInt(idx.IndexOption.Second, 10) + } if idx.IndexOption.KmeansTrainPercent > 0 { res[catalog.IndexAlgoParamKmeansTrainPercent] = strconv.FormatInt(idx.IndexOption.KmeansTrainPercent, 10) diff --git a/pkg/vectorindex/ivfpq/plugin/runtime/runtime_test.go b/pkg/vectorindex/ivfpq/plugin/runtime/runtime_test.go index 7aef203e7c144..20adf3852562f 100644 --- a/pkg/vectorindex/ivfpq/plugin/runtime/runtime_test.go +++ b/pkg/vectorindex/ivfpq/plugin/runtime/runtime_test.go @@ -189,3 +189,15 @@ func TestIvfpqJoinIncludeColumns(t *testing.T) { } require.Equal(t, "a,b", joinIncludeColumns(cols)) } + +// TestIvfpqValidQuantization exercises the per-algo (quant, op) catalog hook +// shared by CREATE and REINDEX. +func TestIvfpqValidQuantization(t *testing.T) { + h := CatalogHooks{} + require.NoError(t, h.ValidQuantization("", "vector_ip_ops")) + require.NoError(t, h.ValidQuantization("float16", "vector_ip_ops")) + require.NoError(t, h.ValidQuantization("uint8", "vector_l2_ops")) + require.Error(t, h.ValidQuantization("int8", "vector_ip_ops")) + require.Error(t, h.ValidQuantization("uint8", "vector_cosine_ops")) + require.Error(t, h.ValidQuantization("bf16", "vector_l2_ops")) +} diff --git a/pkg/vectorindex/ivfpq/search_gpu.go b/pkg/vectorindex/ivfpq/search_gpu.go index 03f645d4b47cf..3eb2af627eded 100644 --- a/pkg/vectorindex/ivfpq/search_gpu.go +++ b/pkg/vectorindex/ivfpq/search_gpu.go @@ -29,19 +29,19 @@ import ( ) // IvfpqSearch implements cache.VectorIndexSearchIf for GPU IVF-PQ indexes. -type IvfpqSearch[T cuvs.VectorType] struct { +type IvfpqSearch[B, Q cuvs.VectorType] struct { Idxcfg vectorindex.IndexConfig Tblcfg vectorindex.IndexTableConfig - Indexes []*IvfpqModel[T] - MultiIndex *cuvs.MultiGpuIvfPq[T] - Overflow *cuvs.GpuBruteForce[T] // CDC insert overflow; nil when no overflow records exist + Indexes []*IvfpqModel[B, Q] + MultiIndex *cuvs.MultiGpuIvfPq[B, Q] + Overflow cuvs.BruteForceOverflow[B] // CDC insert overflow; nil when no overflow records exist Devices []int ThreadsSearch int64 } -func NewIvfpqSearch[T cuvs.VectorType](idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig, devices []int) *IvfpqSearch[T] { +func NewIvfpqSearch[B, Q cuvs.VectorType](idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig, devices []int) *IvfpqSearch[B, Q] { nthread := vectorindex.GetConcurrency(tblcfg.ThreadsSearch) - return &IvfpqSearch[T]{ + return &IvfpqSearch[B, Q]{ Idxcfg: idxcfg, Tblcfg: tblcfg, Devices: devices, @@ -50,11 +50,7 @@ func NewIvfpqSearch[T cuvs.VectorType](idxcfg vectorindex.IndexConfig, tblcfg ve } // Search implements cache.VectorIndexSearchIf. -func (s *IvfpqSearch[T]) Search(sqlproc *sqlexec.SqlProcess, anyquery any, rt vectorindex.RuntimeConfig) (keys any, distances []float64, err error) { - query, ok := anyquery.([]float32) - if !ok { - return nil, nil, moerr.NewInternalErrorNoCtx("IvfpqSearch: query type mismatch") - } +func (s *IvfpqSearch[B, Q]) Search(sqlproc *sqlexec.SqlProcess, anyquery any, rt vectorindex.RuntimeConfig) (keys any, distances []float64, err error) { if s.MultiIndex == nil { return []int64{}, []float64{}, nil @@ -90,10 +86,18 @@ func (s *IvfpqSearch[T]) Search(sqlproc *sqlexec.SqlProcess, anyquery any, rt ve neighbors64 []int64 dists32 []float32 ) + // Any base (f32 or vecf16) routes its native base-typed (B) query through the + // const-B* search_quantize path — cuVS converts B to storage Q on device (B==Q + // copy for a direct index, learned/cast quantizer for a compressed one). The + // query asserts to []B for both float32 (B==float) and Float16 (B==half) base. + qB, ok := anyquery.([]B) + if !ok { + return nil, nil, moerr.NewInternalErrorNoCtx("IvfpqSearch: query type mismatch") + } if rt.FilterJSON != "" { - neighbors64, dists32, err = s.MultiIndex.SearchFloat32WithFilter(query, 1, dim, uint32(limit), sp, rt.FilterJSON) + neighbors64, dists32, err = s.MultiIndex.SearchQuantizeWithFilter(qB, 1, dim, uint32(limit), sp, rt.FilterJSON) } else { - neighbors64, dists32, err = s.MultiIndex.SearchFloat32(query, 1, dim, uint32(limit), sp) + neighbors64, dists32, err = s.MultiIndex.SearchQuantize(qB, 1, dim, uint32(limit), sp) } if err != nil { return nil, nil, err @@ -118,7 +122,7 @@ func (s *IvfpqSearch[T]) Search(sqlproc *sqlexec.SqlProcess, anyquery any, rt ve } // SearchFloat32 implements cache.VectorIndexSearchIf. -func (s *IvfpqSearch[T]) SearchFloat32(proc *sqlexec.SqlProcess, query any, rt vectorindex.RuntimeConfig, outKeys []int64, outDists []float32) error { +func (s *IvfpqSearch[B, Q]) SearchFloat32(proc *sqlexec.SqlProcess, query any, rt vectorindex.RuntimeConfig, outKeys []int64, outDists []float32) error { keys, dists, err := s.Search(proc, query, rt) if err != nil { return err @@ -138,8 +142,8 @@ func (s *IvfpqSearch[T]) SearchFloat32(proc *sqlexec.SqlProcess, query any, rt v } // Load implements cache.VectorIndexSearchIf. -func (s *IvfpqSearch[T]) Load(sqlproc *sqlexec.SqlProcess) (err error) { - indexes, err := LoadMetadata[T](sqlproc, s.Tblcfg.DbName, s.Tblcfg.MetadataTable) +func (s *IvfpqSearch[B, Q]) Load(sqlproc *sqlexec.SqlProcess) (err error) { + indexes, err := LoadMetadata[B, Q](sqlproc, s.Tblcfg.DbName, s.Tblcfg.MetadataTable) if err != nil { return err } @@ -172,7 +176,7 @@ func (s *IvfpqSearch[T]) Load(sqlproc *sqlexec.SqlProcess) (err error) { // loadCdcTail mirrors cagra.CagraSearch.loadCdcTail — see that for the // architectural commentary. Differs only in the IndexConfig type slot and // the GpuIvfPq element type. -func (s *IvfpqSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { +func (s *IvfpqSearch[B, Q]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { var ( includeBytesPerRow int colMetaJSON string @@ -189,7 +193,7 @@ func (s *IvfpqSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { } } - stub := &IvfpqModel[T]{Id: vectorindex.CdcTailId} + stub := &IvfpqModel[B, Q]{Id: vectorindex.CdcTailId} chunks, err := stub.loadCdcEventsFromDB(sqlproc, s.Tblcfg) if err != nil { return err @@ -213,7 +217,7 @@ func (s *IvfpqSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { } dim := int(s.Idxcfg.CuvsIvfpq.Dimensions) - delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks(chunks, dim, includeBytesPerRow) + delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks[B](chunks, dim, includeBytesPerRow) if err != nil { return err } @@ -229,7 +233,7 @@ func (s *IvfpqSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { } } - s.Indexes = append(s.Indexes, &IvfpqModel[T]{ + s.Indexes = append(s.Indexes, &IvfpqModel[B, Q]{ Id: vectorindex.CdcTailId, DeletedPkids: delPkids, OverflowPkids: ovPkids, @@ -242,8 +246,8 @@ func (s *IvfpqSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { } // addOverflowFilterChunks — see cagra/search_gpu.go for docs. -func addOverflowFilterChunks[T cuvs.VectorType]( - bf *cuvs.GpuBruteForce[T], +func addOverflowFilterChunks[B, OB cuvs.VectorType]( + bf *cuvs.GpuBruteForce[B, OB], colMetaJSON string, includeBytes []byte, nrows uint64, @@ -265,7 +269,7 @@ func addOverflowFilterChunks[T cuvs.VectorType]( // loaded model's CDC insert overflow. When the underlying index has INCLUDE // columns, the brute-force is set up with the matching FilterStore so a // filtered query can prefilter overflow rows. -func (s *IvfpqSearch[T]) buildOverflow() error { +func (s *IvfpqSearch[B, Q]) buildOverflow() error { total := uint64(0) for _, m := range s.Indexes { total += uint64(len(m.OverflowPkids)) @@ -286,25 +290,46 @@ func (s *IvfpqSearch[T]) buildOverflow() error { device = s.Devices[0] } - bf, err := cuvs.NewGpuBruteForceEmpty[T]( - total, dim, cuvsMetric, uint32(s.ThreadsSearch), device) + // cuVS brute force can only store float/half. Pick the overflow storage type + // OB from the index storage Q: keep Q when float/half, else fall back to base + // B (which is always float/half). The type-erased BruteForceOverflow[B] holds + // either concrete *GpuBruteForce[B, OB]. + var ( + ov cuvs.BruteForceOverflow[B] + err error + ) + switch cuvs.GetQuantization[Q]() { + case cuvs.F32, cuvs.F16: + ov, err = buildOverflowBF[B, Q](s.Indexes, total, dim, cuvsMetric, device, uint32(s.ThreadsSearch)) + default: // INT8/UINT8: brute force can't store these → store base B. + ov, err = buildOverflowBF[B, B](s.Indexes, total, dim, cuvsMetric, device, uint32(s.ThreadsSearch)) + } if err != nil { return err } + s.Overflow = ov + return nil +} + +// buildOverflowBF — see cagra/search_gpu.go. +func buildOverflowBF[B, OB cuvs.VectorType, Q cuvs.VectorType]( + indexes []*IvfpqModel[B, Q], + total uint64, dim uint32, cuvsMetric cuvs.DistanceType, device int, threads uint32, +) (cuvs.BruteForceOverflow[B], error) { + bf, err := cuvs.NewGpuBruteForceEmpty[B, OB](total, dim, cuvsMetric, threads, device) + if err != nil { + return nil, err + } if err = bf.Start(); err != nil { bf.Destroy() - return err + return nil, err } - // INCLUDE-column wiring — pull the col-meta JSON from the first loaded - // model (every shard agrees by construction). For small-data-only - // indexes (no tag=0 sub-index ever built) the synthetic CDC-tail model - // carries the colMetaJSON recovered from the CdcOpHeader record. var ( colMetaJSON string includeBytesPerRow int ) - for _, m := range s.Indexes { + for _, m := range indexes { if m.Index != nil { colMetaJSON = m.Index.GetFilterColMetaJSON() includeBytesPerRow = m.IncludeBytesPerRow @@ -312,7 +337,7 @@ func (s *IvfpqSearch[T]) buildOverflow() error { } } if colMetaJSON == "" { - for _, m := range s.Indexes { + for _, m := range indexes { if m.OverflowColMetaJSON != "" { colMetaJSON = m.OverflowColMetaJSON includeBytesPerRow = m.IncludeBytesPerRow @@ -323,32 +348,33 @@ func (s *IvfpqSearch[T]) buildOverflow() error { if colMetaJSON != "" && includeBytesPerRow > 0 { if err = bf.SetFilterColumns(colMetaJSON, total); err != nil { bf.Destroy() - return err + return nil, err } } - for _, m := range s.Indexes { + for _, m := range indexes { if len(m.OverflowPkids) == 0 { continue } count := uint64(len(m.OverflowPkids)) - if err = bf.AddChunkFloat(m.OverflowVecs, count, m.OverflowPkids); err != nil { + // Base-typed (B) overflow vectors; AddChunkQuantize converts B -> Q storage + // on the C++ side (native store when B==Q, f32->f16 cast otherwise). + if err = bf.AddChunkQuantize(m.OverflowVecs, count, m.OverflowPkids); err != nil { bf.Destroy() - return err + return nil, err } if colMetaJSON != "" && includeBytesPerRow > 0 { if err = addOverflowFilterChunks(bf, colMetaJSON, m.OverflowIncludeBytes, count, includeBytesPerRow); err != nil { bf.Destroy() - return err + return nil, err } } } if err = bf.Build(); err != nil { bf.Destroy() - return err + return nil, err } - s.Overflow = bf - return nil + return bf, nil } // buildMultiIndex assembles a MultiGpuIvfPq from the loaded indexes. @@ -358,14 +384,14 @@ func (s *IvfpqSearch[T]) buildOverflow() error { // s.MultiIndex == nil — that's the load-bearing path for "no main // index + no brute-force → empty result". Any future regression here // will fail TestIvfpqSearchEmpty. -func (s *IvfpqSearch[T]) buildMultiIndex() (*cuvs.MultiGpuIvfPq[T], error) { +func (s *IvfpqSearch[B, Q]) buildMultiIndex() (*cuvs.MultiGpuIvfPq[B, Q], error) { cuvsMetric, ok := metric.MetricTypeToCuvsMetric[metric.MetricType(s.Idxcfg.CuvsIvfpq.Metric)] if !ok { // Unsupported metric is a real error — surface it rather than returning a // nil index, which Search would treat as an (empty) success. return nil, moerr.NewInternalErrorNoCtxf("IvfpqSearch: unsupported metric type %v", s.Idxcfg.CuvsIvfpq.Metric) } - gpuIndices := make([]*cuvs.GpuIvfPq[T], 0, len(s.Indexes)) + gpuIndices := make([]*cuvs.GpuIvfPq[B, Q], 0, len(s.Indexes)) for _, model := range s.Indexes { if model.Index != nil { gpuIndices = append(gpuIndices, model.Index) @@ -381,7 +407,7 @@ func (s *IvfpqSearch[T]) buildMultiIndex() (*cuvs.MultiGpuIvfPq[T], error) { } // loadIndexes loads each model's index data from the database. -func (s *IvfpqSearch[T]) loadIndexes(sqlproc *sqlexec.SqlProcess, indexes []*IvfpqModel[T]) ([]*IvfpqModel[T], error) { +func (s *IvfpqSearch[B, Q]) loadIndexes(sqlproc *sqlexec.SqlProcess, indexes []*IvfpqModel[B, Q]) ([]*IvfpqModel[B, Q], error) { for _, idx := range indexes { idx.Devices = s.Devices if err := idx.LoadIndex(sqlproc, s.Idxcfg, s.Tblcfg, s.ThreadsSearch, true); err != nil { @@ -395,7 +421,7 @@ func (s *IvfpqSearch[T]) loadIndexes(sqlproc *sqlexec.SqlProcess, indexes []*Ivf } // Destroy implements cache.VectorIndexSearchIf. -func (s *IvfpqSearch[T]) Destroy() { +func (s *IvfpqSearch[B, Q]) Destroy() { s.MultiIndex = nil if s.Overflow != nil { s.Overflow.Destroy() @@ -408,6 +434,6 @@ func (s *IvfpqSearch[T]) Destroy() { } // UpdateConfig implements cache.VectorIndexSearchIf. -func (s *IvfpqSearch[T]) UpdateConfig(newalgo cache.VectorIndexSearchIf) error { +func (s *IvfpqSearch[B, Q]) UpdateConfig(newalgo cache.VectorIndexSearchIf) error { return nil } diff --git a/pkg/vectorindex/ivfpq/search_test.go b/pkg/vectorindex/ivfpq/search_test.go index 28532a76597aa..222d636fb07d8 100644 --- a/pkg/vectorindex/ivfpq/search_test.go +++ b/pkg/vectorindex/ivfpq/search_test.go @@ -34,7 +34,7 @@ import ( // loadedModel builds an index, saves it, then reloads it into GPU memory from // the local tar file. Returns the model with Index != nil. -func loadedModel(t *testing.T, id string) *IvfpqModel[float32] { +func loadedModel(t *testing.T, id string) *IvfpqModel[float32, float32] { t.Helper() built := buildTestModel(t, id, nil) tarPath := built.Path @@ -53,7 +53,7 @@ func loadedModel(t *testing.T, id string) *IvfpqModel[float32] { } defer func() { runSql = origRunSql }() - loader := &IvfpqModel[float32]{ + loader := &IvfpqModel[float32, float32]{ Id: id, Path: tarPath, Checksum: built.Checksum, @@ -72,7 +72,7 @@ func TestIvfpqSearchEmpty(t *testing.T) { proc := testutil.NewProcessWithMPool(t, "", m) sqlproc := sqlexec.NewSqlProcess(proc) - s := NewIvfpqSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) + s := NewIvfpqSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) require.Empty(t, s.Indexes) rt := vectorindex.RuntimeConfig{Limit: 4} @@ -98,8 +98,8 @@ func TestIvfpqSearchTypeMismatch(t *testing.T) { idx := loadedModel(t, "type-mismatch") defer idx.Destroy() - s := NewIvfpqSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) - s.Indexes = []*IvfpqModel[float32]{idx} + s := NewIvfpqSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) + s.Indexes = []*IvfpqModel[float32, float32]{idx} s.MultiIndex, _ = s.buildMultiIndex() rt := vectorindex.RuntimeConfig{Limit: 4} @@ -118,8 +118,8 @@ func TestIvfpqSearchAndSearchFloat32(t *testing.T) { idx := loadedModel(t, "search-single") defer idx.Destroy() - s := NewIvfpqSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) - s.Indexes = []*IvfpqModel[float32]{idx} + s := NewIvfpqSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) + s.Indexes = []*IvfpqModel[float32, float32]{idx} s.MultiIndex, _ = s.buildMultiIndex() data := generateTestData(testNVectors, testDim) @@ -156,8 +156,8 @@ func TestIvfpqSearchMultipleIndexes(t *testing.T) { idx1 := loadedModel(t, "multi-1") defer idx1.Destroy() - s := NewIvfpqSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) - s.Indexes = []*IvfpqModel[float32]{idx0, idx1} + s := NewIvfpqSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) + s.Indexes = []*IvfpqModel[float32, float32]{idx0, idx1} s.MultiIndex, _ = s.buildMultiIndex() data := generateTestData(testNVectors, testDim) @@ -206,7 +206,7 @@ func TestIvfpqSearchLoad(t *testing.T) { } defer func() { runSql_streaming = origStream }() - s := NewIvfpqSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) + s := NewIvfpqSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) err := s.Load(sqlproc) require.NoError(t, err) require.Equal(t, 1, len(s.Indexes)) diff --git a/pkg/vectorindex/ivfpq/sync.go b/pkg/vectorindex/ivfpq/sync.go index 54144bd66f418..1c20aaff04803 100644 --- a/pkg/vectorindex/ivfpq/sync.go +++ b/pkg/vectorindex/ivfpq/sync.go @@ -28,6 +28,8 @@ import ( "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/util" + "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/pb/plan" @@ -48,6 +50,7 @@ type IvfpqSync struct { activeIndexId string dim int + vecBytesPerRow int // dim * base element size (4*dim for f32, 2*dim for f16) includeBytesPerRow int colMetaJSON string @@ -66,11 +69,19 @@ func NewIvfpqSync( idxname string, idxdefs []*plan.IndexDef, dimension int32, + baseType types.T, colMetaJSON string, ) (*IvfpqSync, error) { if dimension <= 0 { return nil, moerr.NewInternalErrorNoCtx("IvfpqSync: invalid dimension") } + // CDC records carry the vector as raw native base-type bytes: 2*dim for a + // vecf16 base, 4*dim otherwise. Must match the iscp writer's encode width + // and the search-side replayEventChunks[B] read width. + elemSize := 4 + if baseType == types.T_array_float16 { + elemSize = 2 + } var idxtblcfg vectorindex.IndexTableConfig idxtblcfg.DbName = db @@ -102,6 +113,7 @@ func NewIvfpqSync( tblcfg: idxtblcfg, idxname: idxname, dim: int(dimension), + vecBytesPerRow: int(dimension) * elemSize, includeBytesPerRow: includeBytesPerRow, colMetaJSON: colMetaJSON, activeIndexId: vectorindex.CdcTailId, @@ -176,7 +188,7 @@ func (s *IvfpqSync) AppendRecords(_ *sqlexec.SqlProcess, recordBytes []byte) err n = 9 // op (1) + pkid (8) case cuvscdc.CdcOpInsert, cuvscdc.CdcOpUpsert: // UPSERT shares INSERT's payload shape; only the op byte differs. - n = 9 + 4*s.dim + s.includeBytesPerRow + n = 9 + s.vecBytesPerRow + s.includeBytesPerRow default: return moerr.NewInternalErrorNoCtx(fmt.Sprintf( "IvfpqSync.AppendRecords: unknown op %d at offset %d", op, pos)) @@ -212,7 +224,10 @@ func (s *IvfpqSync) appendRecord(op cuvscdc.CdcOp, pkid int64, vec []float32, in } } before := len(s.pendingRecords) - out, err := cuvscdc.EncodeEventRecord(s.pendingRecords, op, pkid, vec, include, s.dim, s.includeBytesPerRow) + // This synchronous VectorIndexCdc[float32] path is f32-only (vec is + // []float32). vecf16 ongoing ingestion flows through the iscp writer → + // AppendRecords byte path instead, which honors s.vecBytesPerRow. + out, err := cuvscdc.EncodeEventRecord(s.pendingRecords, op, pkid, util.UnsafeSliceToBytes(vec), include, 4*s.dim, s.includeBytesPerRow) if err != nil { return err } diff --git a/pkg/vectorindex/ivfpq/sync_test.go b/pkg/vectorindex/ivfpq/sync_test.go index eda8956df5219..5f72650799d89 100644 --- a/pkg/vectorindex/ivfpq/sync_test.go +++ b/pkg/vectorindex/ivfpq/sync_test.go @@ -24,6 +24,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" @@ -112,7 +113,7 @@ func TestIvfpqSync_Update_AllInsert(t *testing.T) { defer rec.install(t)() s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) require.Equal(t, vectorindex.CdcTailId, s.activeIndexId) @@ -129,7 +130,7 @@ func TestIvfpqSync_Update_AllInsert(t *testing.T) { require.Len(t, rec.statements, 1) require.Contains(t, rec.statements[0], "'cdc_tail', 0,") - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 0) require.NoError(t, err) require.Empty(t, state.Deleted) require.Len(t, state.Overflow, 2) @@ -145,7 +146,7 @@ func TestIvfpqSync_Update_DeleteAndInsert(t *testing.T) { defer rec.install(t)() s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -159,7 +160,7 @@ func TestIvfpqSync_Update_DeleteAndInsert(t *testing.T) { require.Len(t, rec.statements, 1) require.Contains(t, rec.statements[0], "'cdc_tail', 7,") - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 7), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 7), 16, 0) require.NoError(t, err) require.Equal(t, []int64{42}, state.Deleted) require.Len(t, state.Overflow, 1) @@ -176,7 +177,7 @@ func TestIvfpqSync_Update_DeleteInsertDelete(t *testing.T) { defer rec.install(t)() s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -191,7 +192,7 @@ func TestIvfpqSync_Update_DeleteInsertDelete(t *testing.T) { require.NoError(t, s.Save(sqlproc)) - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 0) require.NoError(t, err) require.Equal(t, []int64{1}, state.Deleted) require.Empty(t, state.Overflow) @@ -207,7 +208,7 @@ func TestIvfpqSync_Update_DeleteIdempotent(t *testing.T) { defer rec.install(t)() s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -220,7 +221,7 @@ func TestIvfpqSync_Update_DeleteIdempotent(t *testing.T) { require.NoError(t, s.Update(sqlproc, cdc)) require.NoError(t, s.Save(sqlproc)) - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 0) require.NoError(t, err) require.ElementsMatch(t, []int64{5, 7}, state.Deleted) } @@ -235,7 +236,7 @@ func TestIvfpqSync_Update_Upsert(t *testing.T) { defer rec.install(t)() s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -248,11 +249,11 @@ func TestIvfpqSync_Update_Upsert(t *testing.T) { require.Len(t, s.pendingSizes, 2) require.NoError(t, s.Save(sqlproc)) - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 0) require.NoError(t, err) require.ElementsMatch(t, []int64{100}, state.Deleted) require.Len(t, state.Overflow, 1) - require.Equal(t, []float32{9, 9, 9, 9}, state.Overflow[0].Vec) + require.Equal(t, []float32{9, 9, 9, 9}, util.UnsafeSliceCast[float32](state.Overflow[0].Vec)) } func TestIvfpqSync_Update_DimMismatch(t *testing.T) { @@ -261,7 +262,7 @@ func TestIvfpqSync_Update_DimMismatch(t *testing.T) { sqlproc := sqlexec.NewSqlProcess(proc) s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -289,7 +290,7 @@ func TestIvfpqSync_Update_WithIncludeBytes(t *testing.T) { require.Equal(t, 9, expectedIBPR) s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, colMetaJSON) + idxdefs("__meta", "__storage"), 4, types.T_array_float32, colMetaJSON) require.NoError(t, err) require.Equal(t, 9, s.includeBytesPerRow) @@ -303,7 +304,7 @@ func TestIvfpqSync_Update_WithIncludeBytes(t *testing.T) { require.NoError(t, s.Update(sqlproc, cdc)) require.NoError(t, s.Save(sqlproc)) - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 9) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 9) require.NoError(t, err) require.Len(t, state.Overflow, 1) require.Equal(t, include, state.Overflow[0].Include) @@ -330,7 +331,7 @@ func TestIvfpqSync_Update_NoOpSaveSkipsSql(t *testing.T) { defer func() { runSql = origRun }() s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{} @@ -354,7 +355,7 @@ func TestIvfpqSync_NewSync_Stateless(t *testing.T) { defer func() { runSql = origRun }() s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) require.Equal(t, vectorindex.CdcTailId, s.activeIndexId) require.Equal(t, 0, called) @@ -370,7 +371,7 @@ func TestIvfpqSync_RunOnce(t *testing.T) { defer rec.install(t)() s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ diff --git a/pkg/vectorindex/metric/cpu.go b/pkg/vectorindex/metric/cpu.go index 605d10c408fe2..3ca9222647712 100644 --- a/pkg/vectorindex/metric/cpu.go +++ b/pkg/vectorindex/metric/cpu.go @@ -29,7 +29,7 @@ const GPUThresholdSQL = GPUThresholdSync / 4 // gpuMode is accepted-but-ignored in non-gpu builds — CPU is the only // option here. The signature matches the gpu.go variant so callers // pass the flag uniformly. -func PairWiseDistance[T types.RealNumbers]( +func PairWiseDistance[T types.ArrayElement]( x [][]T, y [][]T, metric MetricType, @@ -38,7 +38,7 @@ func PairWiseDistance[T types.RealNumbers]( return GoPairWiseDistance(x, y, metric) } -func PairwiseDistanceLaunch[T types.RealNumbers]( +func PairwiseDistanceLaunch[T types.ArrayElement]( x [][]T, y [][]T, metric MetricType, diff --git a/pkg/vectorindex/metric/distance_func.go b/pkg/vectorindex/metric/distance_func.go index ba91d5f5dad48..ce4748adb2162 100644 --- a/pkg/vectorindex/metric/distance_func.go +++ b/pkg/vectorindex/metric/distance_func.go @@ -1,3 +1,5 @@ +//go:build !(amd64 && go1.26 && goexperiment.simd) + // Copyright 2023 Matrix Origin // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -438,122 +440,3 @@ func ScaleInPlace[T types.RealNumbers](v []T, scale T) { v[i] *= scale } } - -// IMPORTANT: Elkans Kmeans always use L2Distance for dense vector or images. After getting the centroids, we can use other distance function -// specified by user to assign vector to corresponding centroids (CENTROIDX JOIN / ProductL2). - -func ResolveKmeansDistanceFn[T types.RealNumbers](metric MetricType, spherical bool) (DistanceFunction[T], bool, error) { - if spherical { - return ResolveKmeansDistanceFnForSparse[T](metric) - } - return ResolveKmeansDistanceFnForDense[T](metric) -} - -func ResolveKmeansDistanceFnForDense[T types.RealNumbers](metric MetricType) (DistanceFunction[T], bool, error) { - var distanceFunction DistanceFunction[T] - normalize := false - switch metric { - case Metric_L2Distance: - distanceFunction = L2Distance[T] - normalize = false - case Metric_L2sqDistance: - // Elkans Kmeans always uses true L2Distance regardless of user metric. - distanceFunction = L2Distance[T] - normalize = false - case Metric_InnerProduct: - distanceFunction = L2Distance[T] - normalize = false - case Metric_CosineDistance: - distanceFunction = L2Distance[T] - normalize = false - case Metric_L1Distance: - distanceFunction = L2Distance[T] - normalize = false - default: - return nil, normalize, moerr.NewInternalErrorNoCtx("invalid distance type") - } - return distanceFunction, normalize, nil -} - -// IMPORTANT: Spherical Kmeans always use Spherical Distance / Cosine Similarity for Sparse vector or text embedding (TD-IDF). -// After getting the centroids, we can use other distance function -// specified by user to assign vector to corresponding centroids (CENTROIDX JOIN / ProductL2). -func ResolveKmeansDistanceFnForSparse[T types.RealNumbers](metric MetricType) (DistanceFunction[T], bool, error) { - var distanceFunction DistanceFunction[T] - normalize := false - switch metric { - case Metric_L2Distance: - distanceFunction = L2Distance[T] - normalize = false - case Metric_L2sqDistance: - distanceFunction = L2Distance[T] - normalize = false - case Metric_InnerProduct: - distanceFunction = SphericalDistance[T] - normalize = true - case Metric_CosineDistance: - distanceFunction = SphericalDistance[T] - normalize = true - case Metric_L1Distance: - distanceFunction = L2Distance[T] - normalize = false - default: - return nil, normalize, moerr.NewInternalErrorNoCtx("invalid distance type") - } - return distanceFunction, normalize, nil -} - -// ResolveDistanceFn is used for similarity score for search and assign vector to centroids (CENTROIDX JOIN / ProductL2). -// IMPORTANT: Don't use it for Elkans Kmeans. -// NOTE: Metric_L2Distance returns L2DistanceSq (squared distance). Callers that need true L2 -// must apply sqrt to each result afterwards (as GoPairWiseDistance does). -func ResolveDistanceFn[T types.RealNumbers](metric MetricType) (DistanceFunction[T], error) { - var distanceFunction DistanceFunction[T] - switch metric { - case Metric_L2Distance: - distanceFunction = L2DistanceSq[T] // caller must sqrt; see function doc above - case Metric_L2sqDistance: - distanceFunction = L2DistanceSq[T] - case Metric_InnerProduct: - distanceFunction = InnerProduct[T] - case Metric_CosineDistance: - distanceFunction = CosineDistance[T] - case Metric_L1Distance: - distanceFunction = L1Distance[T] - default: - return nil, moerr.NewInternalErrorNoCtx("invalid distance type") - } - return distanceFunction, nil -} - -func GoPairWiseDistance[T types.RealNumbers]( - x [][]T, - y [][]T, - metric MetricType, -) ([]float32, error) { - distFn, err := ResolveDistanceFn[T](metric) - if err != nil { - return nil, err - } - - nX := len(x) - nY := len(y) - res := make([]float32, nX*nY) - for i := 0; i < nX; i++ { - for j := 0; j < nY; j++ { - d, err := distFn(x[i], y[j]) - if err != nil { - return nil, err - } - res[i*nY+j] = float32(d) - } - } - - if metric == Metric_L2Distance { - for i := range res { - res[i] = float32(math.Sqrt(float64(res[i]))) - } - } - - return res, nil -} diff --git a/pkg/vectorindex/metric/distance_func_amd64.go b/pkg/vectorindex/metric/distance_func_amd64.go new file mode 100644 index 0000000000000..07cf2ae3438f6 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_amd64.go @@ -0,0 +1,655 @@ +//go:build amd64 && go1.26 && goexperiment.simd + +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metric + +import ( + "math" + "os" + "simd/archsimd" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +// hasAVX512 gates the top kernel tier. TESTING-ONLY override: set +// MO_METRIC_NO_AVX512=1 to force AVX512-capable CPUs down to the AVX2 (and +// then scalar) path, so the lower tiers get real coverage on this hardware. +// Package-level var initializers run before every init() selector, so the +// override is seen by all kernel-selection init() funcs in this package. +var ( + hasAVX512 = archsimd.X86.AVX512() && os.Getenv("MO_METRIC_NO_AVX512") == "" +) + +// Reduction Helpers - Simple Store and Tree Sum for maximum throughput +func sumF32x16(v archsimd.Float32x16) float32 { + var a [16]float32 + v.Store(&a) + s0 := (a[0] + a[1]) + (a[2] + a[3]) + s1 := (a[4] + a[5]) + (a[6] + a[7]) + s2 := (a[8] + a[9]) + (a[10] + a[11]) + s3 := (a[12] + a[13]) + (a[14] + a[15]) + return (s0 + s1) + (s2 + s3) +} + +func sumF64x8(v archsimd.Float64x8) float64 { + var a [8]float64 + v.Store(&a) + return (a[0] + a[1] + a[2] + a[3]) + (a[4] + a[5] + a[6] + a[7]) +} + +// L2 Distance Squared kernels +func L2DistanceSqFloat32(a, b []float32) (float32, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + + var sum float32 + i := 0 + + if hasAVX512 && n >= 64 { + acc0, acc1, acc2, acc3 := archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{} + for i <= n-64 { + as, bs := a[i:i+64:i+64], b[i:i+64:i+64] + d0 := archsimd.LoadFloat32x16Slice(as[0:16]).Sub(archsimd.LoadFloat32x16Slice(bs[0:16])) + d1 := archsimd.LoadFloat32x16Slice(as[16:32]).Sub(archsimd.LoadFloat32x16Slice(bs[16:32])) + d2 := archsimd.LoadFloat32x16Slice(as[32:48]).Sub(archsimd.LoadFloat32x16Slice(bs[32:48])) + d3 := archsimd.LoadFloat32x16Slice(as[48:64]).Sub(archsimd.LoadFloat32x16Slice(bs[48:64])) + + acc0 = d0.MulAdd(d0, acc0) + acc1 = d1.MulAdd(d1, acc1) + acc2 = d2.MulAdd(d2, acc2) + acc3 = d3.MulAdd(d3, acc3) + i += 64 + } + sum += sumF32x16(acc0.Add(acc1).Add(acc2.Add(acc3))) + } + + for i <= n-8 { + // BCE Hint + as := a[i : i+8 : i+8] + bs := b[i : i+8 : i+8] + d0 := as[0] - bs[0] + d1 := as[1] - bs[1] + d2 := as[2] - bs[2] + d3 := as[3] - bs[3] + d4 := as[4] - bs[4] + d5 := as[5] - bs[5] + d6 := as[6] - bs[6] + d7 := as[7] - bs[7] + sum += (d0*d0 + d1*d1) + (d2*d2 + d3*d3) + (d4*d4 + d5*d5) + (d6*d6 + d7*d7) + i += 8 + } + + for ; i < n; i++ { + diff := a[i] - b[i] + sum += diff * diff + } + return sum, nil +} + +func InnerProductFloat32(a, b []float32) (float32, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + + var total float32 + i := 0 + + if hasAVX512 && n >= 64 { + acc0, acc1, acc2, acc3 := archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{} + for i <= n-64 { + as, bs := a[i:i+64:i+64], b[i:i+64:i+64] + acc0 = archsimd.LoadFloat32x16Slice(as[0:16]).MulAdd(archsimd.LoadFloat32x16Slice(bs[0:16]), acc0) + acc1 = archsimd.LoadFloat32x16Slice(as[16:32]).MulAdd(archsimd.LoadFloat32x16Slice(bs[16:32]), acc1) + acc2 = archsimd.LoadFloat32x16Slice(as[32:48]).MulAdd(archsimd.LoadFloat32x16Slice(bs[32:48]), acc2) + acc3 = archsimd.LoadFloat32x16Slice(as[48:64]).MulAdd(archsimd.LoadFloat32x16Slice(bs[48:64]), acc3) + i += 64 + } + total += sumF32x16(acc0.Add(acc1).Add(acc2.Add(acc3))) + } + + for i <= n-8 { + // BCE Hint + as := a[i : i+8 : i+8] + bs := b[i : i+8 : i+8] + total += as[0]*bs[0] + as[1]*bs[1] + as[2]*bs[2] + as[3]*bs[3] + + as[4]*bs[4] + as[5]*bs[5] + as[6]*bs[6] + as[7]*bs[7] + i += 8 + } + + for ; i < n; i++ { + total += a[i] * b[i] + } + return -total, nil +} + +func L2Distance[T types.RealNumbers](v1, v2 []T) (T, error) { + if pf32, ok := any(v1).([]float32); ok { + dist, err := L2DistanceSqFloat32(pf32, any(v2).([]float32)) + if err != nil { + return 0, err + } + return T(math.Sqrt(float64(dist))), nil + } + if pf64, ok := any(v1).([]float64); ok { + dist, err := L2DistanceSqFloat64(pf64, any(v2).([]float64)) + if err != nil { + return 0, err + } + return T(math.Sqrt(dist)), nil + } + return 0, moerr.NewInternalErrorNoCtx("vector type not supported") +} + +func L2DistanceSqFloat64(a, b []float64) (float64, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum float64 + i := 0 + if hasAVX512 && n >= 32 { + acc0, acc1, acc2, acc3 := archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{} + for i <= n-32 { + as, bs := a[i:i+32:i+32], b[i:i+32:i+32] + d0 := archsimd.LoadFloat64x8Slice(as[0:8]).Sub(archsimd.LoadFloat64x8Slice(bs[0:8])) + d1 := archsimd.LoadFloat64x8Slice(as[8:16]).Sub(archsimd.LoadFloat64x8Slice(bs[8:16])) + d2 := archsimd.LoadFloat64x8Slice(as[16:24]).Sub(archsimd.LoadFloat64x8Slice(bs[16:24])) + d3 := archsimd.LoadFloat64x8Slice(as[24:32]).Sub(archsimd.LoadFloat64x8Slice(bs[24:32])) + acc0 = d0.MulAdd(d0, acc0) + acc1 = d1.MulAdd(d1, acc1) + acc2 = d2.MulAdd(d2, acc2) + acc3 = d3.MulAdd(d3, acc3) + i += 32 + } + sum += sumF64x8(acc0.Add(acc1).Add(acc2.Add(acc3))) + } + + for i <= n-8 { + // BCE Hint + as := a[i : i+8 : i+8] + bs := b[i : i+8 : i+8] + d0 := as[0] - bs[0] + d1 := as[1] - bs[1] + d2 := as[2] - bs[2] + d3 := as[3] - bs[3] + d4 := as[4] - bs[4] + d5 := as[5] - bs[5] + d6 := as[6] - bs[6] + d7 := as[7] - bs[7] + sum += (d0*d0 + d1*d1) + (d2*d2 + d3*d3) + (d4*d4 + d5*d5) + (d6*d6 + d7*d7) + i += 8 + } + + for ; i < n; i++ { + diff := a[i] - b[i] + sum += diff * diff + } + return sum, nil +} + +func InnerProductFloat64(a, b []float64) (float64, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var total float64 + i := 0 + if hasAVX512 && n >= 32 { + acc0, acc1, acc2, acc3 := archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{} + for i <= n-32 { + as, bs := a[i:i+32:i+32], b[i:i+32:i+32] + acc0 = archsimd.LoadFloat64x8Slice(as[0:8]).MulAdd(archsimd.LoadFloat64x8Slice(bs[0:8]), acc0) + acc1 = archsimd.LoadFloat64x8Slice(as[8:16]).MulAdd(archsimd.LoadFloat64x8Slice(bs[8:16]), acc1) + acc2 = archsimd.LoadFloat64x8Slice(as[16:24]).MulAdd(archsimd.LoadFloat64x8Slice(bs[16:24]), acc2) + acc3 = archsimd.LoadFloat64x8Slice(as[24:32]).MulAdd(archsimd.LoadFloat64x8Slice(bs[24:32]), acc3) + i += 32 + } + total += sumF64x8(acc0.Add(acc1).Add(acc2.Add(acc3))) + } + + for i <= n-8 { + // BCE Hint + as := a[i : i+8 : i+8] + bs := b[i : i+8 : i+8] + total += as[0]*bs[0] + as[1]*bs[1] + as[2]*bs[2] + as[3]*bs[3] + + as[4]*bs[4] + as[5]*bs[5] + as[6]*bs[6] + as[7]*bs[7] + i += 8 + } + + for ; i < n; i++ { + total += a[i] * b[i] + } + return -total, nil +} + +func L2DistanceSq[T types.RealNumbers](p, q []T) (T, error) { + if pf32, ok := any(p).([]float32); ok { + res, err := L2DistanceSqFloat32(pf32, any(q).([]float32)) + return T(res), err + } + if pf64, ok := any(p).([]float64); ok { + res, err := L2DistanceSqFloat64(pf64, any(q).([]float64)) + return T(res), err + } + return 0, moerr.NewInternalErrorNoCtx("vector type not supported") +} + +func InnerProduct[T types.RealNumbers](p, q []T) (T, error) { + if pf32, ok := any(p).([]float32); ok { + res, err := InnerProductFloat32(pf32, any(q).([]float32)) + return T(res), err + } + if pf64, ok := any(p).([]float64); ok { + res, err := InnerProductFloat64(pf64, any(q).([]float64)) + return T(res), err + } + return 0, moerr.NewInternalErrorNoCtx("vector type not supported") +} + +func L1DistanceFloat32(a, b []float32) (float32, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension mismatch") + } + var sum float32 + i := 0 + if hasAVX512 && n >= 64 { + acc0, acc1, acc2, acc3 := archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{} + for i <= n-64 { + as, bs := a[i:i+64:i+64], b[i:i+64:i+64] + acc0 = acc0.Add(archsimd.LoadFloat32x16Slice(as[0:16]).Sub(archsimd.LoadFloat32x16Slice(bs[0:16])).Max(archsimd.LoadFloat32x16Slice(bs[0:16]).Sub(archsimd.LoadFloat32x16Slice(as[0:16])))) + acc1 = acc1.Add(archsimd.LoadFloat32x16Slice(as[16:32]).Sub(archsimd.LoadFloat32x16Slice(bs[16:32])).Max(archsimd.LoadFloat32x16Slice(bs[16:32]).Sub(archsimd.LoadFloat32x16Slice(as[16:32])))) + acc2 = acc2.Add(archsimd.LoadFloat32x16Slice(as[32:48]).Sub(archsimd.LoadFloat32x16Slice(bs[32:48])).Max(archsimd.LoadFloat32x16Slice(bs[32:48]).Sub(archsimd.LoadFloat32x16Slice(as[32:48])))) + acc3 = acc3.Add(archsimd.LoadFloat32x16Slice(as[48:64]).Sub(archsimd.LoadFloat32x16Slice(bs[48:64])).Max(archsimd.LoadFloat32x16Slice(bs[48:64]).Sub(archsimd.LoadFloat32x16Slice(as[48:64])))) + i += 64 + } + sum += sumF32x16(acc0.Add(acc1).Add(acc2.Add(acc3))) + } + + abs := func(x float32) float32 { + return math.Float32frombits(math.Float32bits(x) &^ (1 << 31)) + } + for i <= n-8 { + // BCE Hint + as := a[i : i+8 : i+8] + bs := b[i : i+8 : i+8] + sum += abs(as[0]-bs[0]) + abs(as[1]-bs[1]) + abs(as[2]-bs[2]) + abs(as[3]-bs[3]) + + abs(as[4]-bs[4]) + abs(as[5]-bs[5]) + abs(as[6]-bs[6]) + abs(as[7]-bs[7]) + i += 8 + } + + for ; i < n; i++ { + sum += abs(a[i] - b[i]) + } + return sum, nil +} + +func L1DistanceFloat64(a, b []float64) (float64, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension mismatch") + } + var sum float64 + i := 0 + if hasAVX512 && n >= 32 { + acc0, acc1, acc2, acc3 := archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{} + for i <= n-32 { + as, bs := a[i:i+32:i+32], b[i:i+32:i+32] + acc0 = acc0.Add(archsimd.LoadFloat64x8Slice(as[0:8]).Sub(archsimd.LoadFloat64x8Slice(bs[0:8])).Max(archsimd.LoadFloat64x8Slice(bs[0:8]).Sub(archsimd.LoadFloat64x8Slice(as[0:8])))) + acc1 = acc1.Add(archsimd.LoadFloat64x8Slice(as[8:16]).Sub(archsimd.LoadFloat64x8Slice(bs[8:16])).Max(archsimd.LoadFloat64x8Slice(bs[8:16]).Sub(archsimd.LoadFloat64x8Slice(as[8:16])))) + acc2 = acc2.Add(archsimd.LoadFloat64x8Slice(as[16:24]).Sub(archsimd.LoadFloat64x8Slice(bs[16:24])).Max(archsimd.LoadFloat64x8Slice(bs[16:24]).Sub(archsimd.LoadFloat64x8Slice(as[16:24])))) + acc3 = acc3.Add(archsimd.LoadFloat64x8Slice(as[24:32]).Sub(archsimd.LoadFloat64x8Slice(bs[24:32])).Max(archsimd.LoadFloat64x8Slice(bs[24:32]).Sub(archsimd.LoadFloat64x8Slice(as[24:32])))) + i += 32 + } + sum += sumF64x8(acc0.Add(acc1).Add(acc2.Add(acc3))) + } + + abs := func(x float64) float64 { + return math.Abs(x) + } + for i <= n-8 { + // BCE Hint + as := a[i : i+8 : i+8] + bs := b[i : i+8 : i+8] + sum += abs(as[0]-bs[0]) + abs(as[1]-bs[1]) + abs(as[2]-bs[2]) + abs(as[3]-bs[3]) + + abs(as[4]-bs[4]) + abs(as[5]-bs[5]) + abs(as[6]-bs[6]) + abs(as[7]-bs[7]) + i += 8 + } + + for ; i < n; i++ { + sum += abs(a[i] - b[i]) + } + return sum, nil +} + +func L1Distance[T types.RealNumbers](p, q []T) (T, error) { + if pf32, ok := any(p).([]float32); ok { + res, err := L1DistanceFloat32(pf32, any(q).([]float32)) + return T(res), err + } + if pf64, ok := any(p).([]float64); ok { + res, err := L1DistanceFloat64(pf64, any(q).([]float64)) + return T(res), err + } + return 0, moerr.NewInternalErrorNoCtx("vector type not supported") +} + +func CosineDistanceF32(a, b []float32) (float32, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension mismatch") + } + var dot, normA, normB float32 + i := 0 + if n >= 16 && hasAVX512 { + accD, accA, accB := archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{} + for i <= n-16 { + va, vb := archsimd.LoadFloat32x16Slice(a[i:i+16]), archsimd.LoadFloat32x16Slice(b[i:i+16]) + accD = va.MulAdd(vb, accD) + accA = va.MulAdd(va, accA) + accB = vb.MulAdd(vb, accB) + i += 16 + } + dot, normA, normB = sumF32x16(accD), sumF32x16(accA), sumF32x16(accB) + } + + for i <= n-4 { + // BCE Hint + va := a[i : i+4 : i+4] + vb := b[i : i+4 : i+4] + dot += va[0]*vb[0] + va[1]*vb[1] + va[2]*vb[2] + va[3]*vb[3] + normA += va[0]*va[0] + va[1]*va[1] + va[2]*va[2] + va[3]*va[3] + normB += vb[0]*vb[0] + vb[1]*vb[1] + vb[2]*vb[2] + vb[3]*vb[3] + i += 4 + } + + for ; i < n; i++ { + dot, normA, normB = dot+a[i]*b[i], normA+a[i]*a[i], normB+b[i]*b[i] + } + den := math.Sqrt(float64(normA)) * math.Sqrt(float64(normB)) + if den == 0 { + return 1.0, nil + } + return float32(cosineDistClamped(float64(dot), den)), nil +} + +func CosineDistanceF64(a, b []float64) (float64, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension mismatch") + } + var dot, normA, normB float64 + i := 0 + if n >= 8 && hasAVX512 { + accD, accA, accB := archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{} + for i <= n-8 { + va, vb := archsimd.LoadFloat64x8Slice(a[i:i+8]), archsimd.LoadFloat64x8Slice(b[i:i+8]) + accD = va.MulAdd(vb, accD) + accA = va.MulAdd(va, accA) + accB = vb.MulAdd(vb, accB) + i += 8 + } + dot, normA, normB = sumF64x8(accD), sumF64x8(accA), sumF64x8(accB) + } + + for i <= n-4 { + // BCE Hint + va := a[i : i+4 : i+4] + vb := b[i : i+4 : i+4] + dot += va[0]*vb[0] + va[1]*vb[1] + va[2]*vb[2] + va[3]*vb[3] + normA += va[0]*va[0] + va[1]*va[1] + va[2]*va[2] + va[3]*va[3] + normB += vb[0]*vb[0] + vb[1]*vb[1] + vb[2]*vb[2] + vb[3]*vb[3] + i += 4 + } + + for ; i < n; i++ { + dot, normA, normB = dot+a[i]*b[i], normA+a[i]*a[i], normB+b[i]*b[i] + } + den := math.Sqrt(normA) * math.Sqrt(normB) + if den == 0 { + return 1.0, nil + } + return cosineDistClamped(dot, den), nil +} + +func CosineDistance[T types.RealNumbers](p, q []T) (T, error) { + if pf32, ok := any(p).([]float32); ok { + res, err := CosineDistanceF32(pf32, any(q).([]float32)) + return T(res), err + } + if pf64, ok := any(p).([]float64); ok { + res, err := CosineDistanceF64(pf64, any(q).([]float64)) + return T(res), err + } + return 0, moerr.NewInternalErrorNoCtx("vector type not supported") +} + +func CosineSimilarityF32(a, b []float32) (float32, error) { + n := len(a) + if n == 0 { + return 0, nil + } + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension mismatch") + } + var dot, normA, normB float32 + i := 0 + if n >= 16 && hasAVX512 { + accD, accA, accB := archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{} + for i <= n-16 { + va, vb := archsimd.LoadFloat32x16Slice(a[i:i+16]), archsimd.LoadFloat32x16Slice(b[i:i+16]) + accD = va.MulAdd(vb, accD) + accA = va.MulAdd(va, accA) + accB = vb.MulAdd(vb, accB) + i += 16 + } + dot, normA, normB = sumF32x16(accD), sumF32x16(accA), sumF32x16(accB) + } + + for i <= n-4 { + // BCE Hint + va := a[i : i+4 : i+4] + vb := b[i : i+4 : i+4] + dot += va[0]*vb[0] + va[1]*vb[1] + va[2]*vb[2] + va[3]*vb[3] + normA += va[0]*va[0] + va[1]*va[1] + va[2]*va[2] + va[3]*va[3] + normB += vb[0]*vb[0] + vb[1]*vb[1] + vb[2]*vb[2] + vb[3]*vb[3] + i += 4 + } + + for ; i < n; i++ { + dot, normA, normB = dot+a[i]*b[i], normA+a[i]*a[i], normB+b[i]*b[i] + } + den := math.Sqrt(float64(normA)) * math.Sqrt(float64(normB)) + if den == 0 { + return 0, moerr.NewInternalErrorNoCtx("cosine similarity zero denominator") + } + return float32(float64(dot) / den), nil +} + +func CosineSimilarityF64(a, b []float64) (float64, error) { + n := len(a) + if n == 0 { + return 0, nil + } + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension mismatch") + } + var dot, normA, normB float64 + i := 0 + if n >= 8 && hasAVX512 { + accD, accA, accB := archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{} + for i <= n-8 { + va, vb := archsimd.LoadFloat64x8Slice(a[i:i+8]), archsimd.LoadFloat64x8Slice(b[i:i+8]) + accD = va.MulAdd(vb, accD) + accA = va.MulAdd(va, accA) + accB = vb.MulAdd(vb, accB) + i += 8 + } + dot, normA, normB = sumF64x8(accD), sumF64x8(accA), sumF64x8(accB) + } + + for i <= n-4 { + // BCE Hint + va := a[i : i+4 : i+4] + vb := b[i : i+4 : i+4] + dot += va[0]*vb[0] + va[1]*vb[1] + va[2]*vb[2] + va[3]*vb[3] + normA += va[0]*va[0] + va[1]*va[1] + va[2]*va[2] + va[3]*va[3] + normB += vb[0]*vb[0] + vb[1]*vb[1] + vb[2]*vb[2] + vb[3]*vb[3] + i += 4 + } + + for ; i < n; i++ { + dot, normA, normB = dot+a[i]*b[i], normA+a[i]*a[i], normB+b[i]*b[i] + } + den := math.Sqrt(normA) * math.Sqrt(normB) + if den == 0 { + return 0, moerr.NewInternalErrorNoCtx("cosine similarity zero denominator") + } + return dot / den, nil +} + +func CosineSimilarity[T types.RealNumbers](p, q []T) (T, error) { + if pf32, ok := any(p).([]float32); ok { + res, err := CosineSimilarityF32(pf32, any(q).([]float32)) + return T(res), err + } + if pf64, ok := any(p).([]float64); ok { + res, err := CosineSimilarityF64(pf64, any(q).([]float64)) + return T(res), err + } + return 0, moerr.NewInternalErrorNoCtx("vector type not supported") +} + +func SphericalDistanceFloat32(a, b []float32) (float32, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var total float32 + i := 0 + if hasAVX512 && n >= 64 { + acc0, acc1, acc2, acc3 := archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{} + for i <= n-64 { + as, bs := a[i:i+64:i+64], b[i:i+64:i+64] + acc0 = archsimd.LoadFloat32x16Slice(as[0:16]).MulAdd(archsimd.LoadFloat32x16Slice(bs[0:16]), acc0) + acc1 = archsimd.LoadFloat32x16Slice(as[16:32]).MulAdd(archsimd.LoadFloat32x16Slice(bs[16:32]), acc1) + acc2 = archsimd.LoadFloat32x16Slice(as[32:48]).MulAdd(archsimd.LoadFloat32x16Slice(bs[32:48]), acc2) + acc3 = archsimd.LoadFloat32x16Slice(as[48:64]).MulAdd(archsimd.LoadFloat32x16Slice(bs[48:64]), acc3) + i += 64 + } + total += sumF32x16(acc0.Add(acc1).Add(acc2.Add(acc3))) + } + + for i <= n-8 { + // BCE Hint + as := a[i : i+8 : i+8] + bs := b[i : i+8 : i+8] + total += as[0]*bs[0] + as[1]*bs[1] + as[2]*bs[2] + as[3]*bs[3] + + as[4]*bs[4] + as[5]*bs[5] + as[6]*bs[6] + as[7]*bs[7] + i += 8 + } + + for ; i < n; i++ { + total += a[i] * b[i] + } + if total > 1.0 { + total = 1.0 + } else if total < -1.0 { + total = -1.0 + } + return float32(math.Acos(float64(total)) / math.Pi), nil +} + +func SphericalDistanceFloat64(a, b []float64) (float64, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var total float64 + i := 0 + if hasAVX512 && n >= 32 { + acc0, acc1, acc2, acc3 := archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{} + for i <= n-32 { + as, bs := a[i:i+32:i+32], b[i:i+32:i+32] + acc0 = archsimd.LoadFloat64x8Slice(as[0:8]).MulAdd(archsimd.LoadFloat64x8Slice(bs[0:8]), acc0) + acc1 = archsimd.LoadFloat64x8Slice(as[8:16]).MulAdd(archsimd.LoadFloat64x8Slice(bs[8:16]), acc1) + acc2 = archsimd.LoadFloat64x8Slice(as[16:24]).MulAdd(archsimd.LoadFloat64x8Slice(bs[16:24]), acc2) + acc3 = archsimd.LoadFloat64x8Slice(as[24:32]).MulAdd(archsimd.LoadFloat64x8Slice(bs[24:32]), acc3) + i += 32 + } + total += sumF64x8(acc0.Add(acc1).Add(acc2.Add(acc3))) + } + + for i <= n-8 { + // BCE Hint + as := a[i : i+8 : i+8] + bs := b[i : i+8 : i+8] + total += as[0]*bs[0] + as[1]*bs[1] + as[2]*bs[2] + as[3]*bs[3] + + as[4]*bs[4] + as[5]*bs[5] + as[6]*bs[6] + as[7]*bs[7] + i += 8 + } + + for ; i < n; i++ { + total += a[i] * b[i] + } + if total > 1.0 { + total = 1.0 + } else if total < -1.0 { + total = -1.0 + } + return math.Acos(total) / math.Pi, nil +} + +func SphericalDistance[T types.RealNumbers](p, q []T) (T, error) { + if pf32, ok := any(p).([]float32); ok { + res, err := SphericalDistanceFloat32(pf32, any(q).([]float32)) + return T(res), err + } + if pf64, ok := any(p).([]float64); ok { + res, err := SphericalDistanceFloat64(pf64, any(q).([]float64)) + return T(res), err + } + return 0, moerr.NewInternalErrorNoCtx("vector type not supported") +} + +func NormalizeL2[T types.RealNumbers](v1 []T, normalized []T) error { + if len(v1) == 0 { + return moerr.NewInternalErrorNoCtx("cannot normalize empty vector") + } + var sumSquares float64 + for _, val := range v1 { + sumSquares += float64(val) * float64(val) + } + norm := math.Sqrt(sumSquares) + if norm == 0 { + copy(normalized, v1) + return nil + } + for i, val := range v1 { + normalized[i] = T(float64(val) / norm) + } + return nil +} + +func ScaleInPlace[T types.RealNumbers](v []T, scale T) { + for i := range v { + v[i] *= scale + } +} diff --git a/pkg/vectorindex/metric/distance_func_amd64_cover_test.go b/pkg/vectorindex/metric/distance_func_amd64_cover_test.go new file mode 100644 index 0000000000000..2b156745a0a41 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_amd64_cover_test.go @@ -0,0 +1,245 @@ +//go:build amd64 && go1.26 && goexperiment.simd + +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Branch-coverage tests for the f32/f64 amd64 distance kernels: dimension +// mismatch guards, the AVX-512 block + unrolled loop + scalar tail across many +// dims, cosine/spherical clamp + zero-denominator edges, and the +// NormalizeL2 / ScaleInPlace helpers (previously 0%). + +package metric + +import ( + "math" + "math/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +func clampUnit(x float64) float64 { + if x > 1 { + return 1 + } + if x < -1 { + return -1 + } + return x +} + +// TestAMD64KernelsAcrossDims drives every f32/f64 kernel over dims that hit the +// AVX-512 block (>=64 f32 / >=32 f64), the 8/4-lane unrolled loop, and the scalar +// remainder, checking each against a plain float64 oracle. +func TestAMD64KernelsAcrossDims(t *testing.T) { + r := rand.New(rand.NewSource(7)) + dims := []int{1, 3, 4, 7, 8, 9, 15, 16, 17, 32, 33, 64, 65, 100, 105} + for _, n := range dims { + a32, b32 := make([]float32, n), make([]float32, n) + a64, b64 := make([]float64, n), make([]float64, n) + var l2, dot, l1, na, nb float64 + for i := 0; i < n; i++ { + a32[i], b32[i] = float32(r.NormFloat64()), float32(r.NormFloat64()) + a64[i], b64[i] = float64(a32[i]), float64(b32[i]) + d := a64[i] - b64[i] + l2 += d * d + dot += a64[i] * b64[i] + if d < 0 { + l1 -= d + } else { + l1 += d + } + na += a64[i] * a64[i] + nb += b64[i] * b64[i] + } + den := math.Sqrt(na) * math.Sqrt(nb) + rel := func(want float64) float64 { return 1e-2 * (1 + math.Abs(want)) } + relTight := func(want float64) float64 { return 1e-6 * (1 + math.Abs(want)) } + + g32, err := L2DistanceSqFloat32(a32, b32) + require.NoError(t, err) + require.InDelta(t, l2, float64(g32), rel(l2), "L2sqF32 n=%d", n) + g64, err := L2DistanceSqFloat64(a64, b64) + require.NoError(t, err) + require.InDelta(t, l2, g64, relTight(l2), "L2sqF64 n=%d", n) + + // L2Distance = sqrt(L2sq) via the generic dispatcher (both type arms). + gd32, err := L2Distance(a32, b32) + require.NoError(t, err) + require.InDelta(t, math.Sqrt(l2), float64(gd32), rel(math.Sqrt(l2)), "L2F32 n=%d", n) + gd64, err := L2Distance(a64, b64) + require.NoError(t, err) + require.InDelta(t, math.Sqrt(l2), gd64, relTight(math.Sqrt(l2)), "L2F64 n=%d", n) + + // InnerProduct returns the negated dot product. + ip32, err := InnerProductFloat32(a32, b32) + require.NoError(t, err) + require.InDelta(t, -dot, float64(ip32), rel(dot), "IPF32 n=%d", n) + ip64, err := InnerProductFloat64(a64, b64) + require.NoError(t, err) + require.InDelta(t, -dot, ip64, relTight(dot), "IPF64 n=%d", n) + + l1a, err := L1DistanceFloat32(a32, b32) + require.NoError(t, err) + require.InDelta(t, l1, float64(l1a), rel(l1), "L1F32 n=%d", n) + l1b, err := L1DistanceFloat64(a64, b64) + require.NoError(t, err) + require.InDelta(t, l1, l1b, relTight(l1), "L1F64 n=%d", n) + + cd32, err := CosineDistanceF32(a32, b32) + require.NoError(t, err) + require.InDelta(t, 1.0-clampUnit(dot/den), float64(cd32), rel(1), "CosDistF32 n=%d", n) + cd64, err := CosineDistanceF64(a64, b64) + require.NoError(t, err) + require.InDelta(t, 1.0-clampUnit(dot/den), cd64, relTight(1), "CosDistF64 n=%d", n) + + cs32, err := CosineSimilarityF32(a32, b32) + require.NoError(t, err) + require.InDelta(t, dot/den, float64(cs32), rel(1), "CosSimF32 n=%d", n) + cs64, err := CosineSimilarityF64(a64, b64) + require.NoError(t, err) + require.InDelta(t, dot/den, cs64, relTight(1), "CosSimF64 n=%d", n) + + sp32, err := SphericalDistanceFloat32(a32, b32) + require.NoError(t, err) + require.InDelta(t, math.Acos(clampUnit(dot))/math.Pi, float64(sp32), rel(1), "SphF32 n=%d", n) + sp64, err := SphericalDistanceFloat64(a64, b64) + require.NoError(t, err) + require.InDelta(t, math.Acos(clampUnit(dot))/math.Pi, sp64, rel(1), "SphF64 n=%d", n) + } +} + +// TestAMD64DimensionMismatch covers the length-guard error branch of every kernel. +func TestAMD64DimensionMismatch(t *testing.T) { + x32, y32 := make([]float32, 8), make([]float32, 7) + x64, y64 := make([]float64, 8), make([]float64, 7) + for name, fn := range map[string]func() error{ + "L2sqF32": func() error { _, e := L2DistanceSqFloat32(x32, y32); return e }, + "L2sqF64": func() error { _, e := L2DistanceSqFloat64(x64, y64); return e }, + "IPF32": func() error { _, e := InnerProductFloat32(x32, y32); return e }, + "IPF64": func() error { _, e := InnerProductFloat64(x64, y64); return e }, + "L1F32": func() error { _, e := L1DistanceFloat32(x32, y32); return e }, + "L1F64": func() error { _, e := L1DistanceFloat64(x64, y64); return e }, + "CosDistF32": func() error { _, e := CosineDistanceF32(x32, y32); return e }, + "CosDistF64": func() error { _, e := CosineDistanceF64(x64, y64); return e }, + "CosSimF32": func() error { _, e := CosineSimilarityF32(x32, y32); return e }, + "CosSimF64": func() error { _, e := CosineSimilarityF64(x64, y64); return e }, + "SphF32": func() error { _, e := SphericalDistanceFloat32(x32, y32); return e }, + "SphF64": func() error { _, e := SphericalDistanceFloat64(x64, y64); return e }, + } { + require.Error(t, fn(), name) + } +} + +// TestAMD64ClampAndZeroEdges covers the spherical <-1 clamp, cosine zero-denom +// (distance 1.0), cosine-similarity zero-denom (error) and empty-input branches. +func TestAMD64ClampAndZeroEdges(t *testing.T) { + // Anti-correlated unit-ish vectors -> dot < -1 -> spherical low clamp. + anti32 := []float32{1, 1, 1, 1} + negs32 := []float32{-1, -1, -1, -1} + sp, err := SphericalDistanceFloat32(anti32, negs32) + require.NoError(t, err) + require.InDelta(t, 1.0, float64(sp), 1e-6) // acos(-1)/pi == 1 + anti64 := []float64{1, 1, 1, 1} + negs64 := []float64{-1, -1, -1, -1} + sp64, err := SphericalDistanceFloat64(anti64, negs64) + require.NoError(t, err) + require.InDelta(t, 1.0, sp64, 1e-9) + + // Zero vector -> zero denominator. + z32, nz32 := make([]float32, 8), []float32{1, 2, 3, 4, 5, 6, 7, 8} + z64, nz64 := make([]float64, 8), []float64{1, 2, 3, 4, 5, 6, 7, 8} + + d32, err := CosineDistanceF32(z32, nz32) + require.NoError(t, err) + require.Equal(t, float32(1.0), d32) + d64, err := CosineDistanceF64(z64, nz64) + require.NoError(t, err) + require.Equal(t, 1.0, d64) + + _, err = CosineSimilarityF32(z32, nz32) + require.Error(t, err) + _, err = CosineSimilarityF64(z64, nz64) + require.Error(t, err) + + // Empty input -> cosine similarity returns (0, nil) before the length check. + e32, err := CosineSimilarityF32(nil, nil) + require.NoError(t, err) + require.Equal(t, float32(0), e32) + e64, err := CosineSimilarityF64(nil, nil) + require.NoError(t, err) + require.Equal(t, 0.0, e64) +} + +// TestAMD64GenericDispatchers covers the generic RealNumbers wrappers (both the +// float32 and float64 type arms) and L2Distance's error-propagation branches. +// The trailing "type not supported" returns are unreachable: RealNumbers is +// constrained to float32|float64. +func TestAMD64GenericDispatchers(t *testing.T) { + a32, b32 := []float32{1, 2, 3, 4}, []float32{4, 3, 2, 1} + a64, b64 := []float64{1, 2, 3, 4}, []float64{4, 3, 2, 1} + for name, fn := range map[string]func() error{ + "L2sq/32": func() error { _, e := L2DistanceSq(a32, b32); return e }, + "L2sq/64": func() error { _, e := L2DistanceSq(a64, b64); return e }, + "L2/32": func() error { _, e := L2Distance(a32, b32); return e }, + "L2/64": func() error { _, e := L2Distance(a64, b64); return e }, + "IP/32": func() error { _, e := InnerProduct(a32, b32); return e }, + "IP/64": func() error { _, e := InnerProduct(a64, b64); return e }, + "L1/32": func() error { _, e := L1Distance(a32, b32); return e }, + "L1/64": func() error { _, e := L1Distance(a64, b64); return e }, + "Cos/32": func() error { _, e := CosineDistance(a32, b32); return e }, + "Cos/64": func() error { _, e := CosineDistance(a64, b64); return e }, + "CosSim/32": func() error { _, e := CosineSimilarity(a32, b32); return e }, + "CosSim/64": func() error { _, e := CosineSimilarity(a64, b64); return e }, + "Sph/32": func() error { _, e := SphericalDistance(a32, b32); return e }, + "Sph/64": func() error { _, e := SphericalDistance(a64, b64); return e }, + } { + require.NoError(t, fn(), name) + } + + // L2Distance propagates the underlying mismatch error on both type arms. + _, err := L2Distance([]float32{1, 2}, []float32{1}) + require.Error(t, err) + _, err = L2Distance([]float64{1, 2}, []float64{1}) + require.Error(t, err) +} + +// TestNormalizeL2AndScaleInPlace covers both helpers (previously 0%). +func TestNormalizeL2AndScaleInPlace(t *testing.T) { + // Empty -> error. + require.Error(t, NormalizeL2([]float32{}, []float32{})) + + // Zero vector -> copy through, norm stays zero. + zin := []float32{0, 0, 0} + zout := make([]float32, 3) + require.NoError(t, NormalizeL2(zin, zout)) + require.Equal(t, zin, zout) + + // Normal vector -> unit L2 norm. + in := []float64{3, 4} + out := make([]float64, 2) + require.NoError(t, NormalizeL2(in, out)) + require.InDelta(t, 0.6, out[0], 1e-12) + require.InDelta(t, 0.8, out[1], 1e-12) + var norm float64 + for _, v := range out { + norm += v * v + } + require.InDelta(t, 1.0, norm, 1e-12) + + // ScaleInPlace mutates in place. + v := []float32{1, 2, 3} + ScaleInPlace(v, 2) + require.Equal(t, []float32{2, 4, 6}, v) +} diff --git a/pkg/vectorindex/metric/distance_func_bench_test.go b/pkg/vectorindex/metric/distance_func_bench_test.go index 506d602d116cd..9a81b4acb6a28 100644 --- a/pkg/vectorindex/metric/distance_func_bench_test.go +++ b/pkg/vectorindex/metric/distance_func_bench_test.go @@ -25,10 +25,10 @@ Benchmark_L2Distance/Normalize_L2-10 1277733 1 Benchmark_L2Distance/L2_Distance(v1,_NormalizeL2)-10 589376 1883 ns/op */ func Benchmark_L2Distance(b *testing.B) { - dim := 128 + dim := 1024 - b.Run("L2 Distance", func(b *testing.B) { - v1, v2 := randomVectors(b.N, dim), randomVectors(b.N, dim) + b.Run("L2 Distance float64", func(b *testing.B) { + v1, v2 := randomVectors[float64](b.N, dim), randomVectors[float64](b.N, dim) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -36,34 +36,211 @@ func Benchmark_L2Distance(b *testing.B) { } }) - b.Run("Normalize L2", func(b *testing.B) { - v1 := randomVectors(b.N, dim) + b.Run("L2 Distance float32", func(b *testing.B) { + v1, v2 := randomVectors[float32](b.N, dim), randomVectors[float32](b.N, dim) b.ResetTimer() for i := 0; i < b.N; i++ { - res := make([]float64, dim) - _ = NormalizeL2[float64](v1[i], res) + _, _ = L2Distance[float32](v1[i], v2[i]) } }) - b.Run("L2 Distance(v1, NormalizeL2)", func(b *testing.B) { - v1, v2 := randomVectors(b.N, dim), randomVectors(b.N, dim) + /* + b.Run("Normalize L2 float64", func(b *testing.B) { + v1 := randomVectors[float64](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + res := make([]float64, dim) + _ = NormalizeL2[float64](v1[i], res) + } + }) + + b.Run("Normalize L2 float32", func(b *testing.B) { + v1 := randomVectors[float32](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + res := make([]float32, dim) + _ = NormalizeL2[float32](v1[i], res) + } + }) + + b.Run("L2 Distance(v1, NormalizeL2) float64", func(b *testing.B) { + v1, v2 := randomVectors[float64](b.N, dim), randomVectors[float64](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + res := make([]float64, dim) + _ = NormalizeL2[float64](v2[i], res) + _, _ = L2Distance[float64](v1[i], res) + } + }) + */ +} + +func Benchmark_L2DistanceSq(b *testing.B) { + dim := 1024 + + b.Run("L2 DistanceSq float64", func(b *testing.B) { + v1, v2 := randomVectors[float64](b.N, dim), randomVectors[float64](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = L2DistanceSq[float64](v1[i], v2[i]) + } + }) + + b.Run("L2 DistanceSq float32", func(b *testing.B) { + v1, v2 := randomVectors[float32](b.N, dim), randomVectors[float32](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = L2DistanceSq[float32](v1[i], v2[i]) + } + }) +} + +func Benchmark_L1Distance(b *testing.B) { + dim := 1024 + + b.Run("L1 Distance float64", func(b *testing.B) { + v1, v2 := randomVectors[float64](b.N, dim), randomVectors[float64](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = L1Distance[float64](v1[i], v2[i]) + } + }) + + b.Run("L1 Distance float32", func(b *testing.B) { + v1, v2 := randomVectors[float32](b.N, dim), randomVectors[float32](b.N, dim) b.ResetTimer() for i := 0; i < b.N; i++ { - res := make([]float64, dim) - _ = NormalizeL2[float64](v2[i], res) - _, _ = L2Distance[float64](v1[i], res) + _, _ = L1Distance[float32](v1[i], v2[i]) } }) +} + +func Benchmark_InnerProduct(b *testing.B) { + dim := 1024 + + b.Run("Inner Product float64", func(b *testing.B) { + v1, v2 := randomVectors[float64](b.N, dim), randomVectors[float64](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = InnerProduct[float64](v1[i], v2[i]) + } + }) + + b.Run("Inner Product float32", func(b *testing.B) { + v1, v2 := randomVectors[float32](b.N, dim), randomVectors[float32](b.N, dim) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = InnerProduct[float32](v1[i], v2[i]) + } + }) } -func randomVectors(size, dim int) [][]float64 { - vectors := make([][]float64, size) +func Benchmark_CosineDistance(b *testing.B) { + dim := 1024 + + b.Run("Cosine Distance float64", func(b *testing.B) { + v1, v2 := randomVectors[float64](b.N, dim), randomVectors[float64](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = CosineDistance[float64](v1[i], v2[i]) + } + }) + + b.Run("Cosine Distance float32", func(b *testing.B) { + v1, v2 := randomVectors[float32](b.N, dim), randomVectors[float32](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = CosineDistance[float32](v1[i], v2[i]) + } + }) +} + +func Benchmark_CosineSimilarity(b *testing.B) { + dim := 1024 + + b.Run("Cosine Similarity float64", func(b *testing.B) { + v1, v2 := randomVectors[float64](b.N, dim), randomVectors[float64](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = CosineSimilarity[float64](v1[i], v2[i]) + } + }) + + b.Run("Cosine Similarity float32", func(b *testing.B) { + v1, v2 := randomVectors[float32](b.N, dim), randomVectors[float32](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = CosineSimilarity[float32](v1[i], v2[i]) + } + }) +} + +func Benchmark_SphericalDistance(b *testing.B) { + dim := 1024 + + b.Run("Spherical Distance float64", func(b *testing.B) { + v1, v2 := randomVectors[float64](b.N, dim), randomVectors[float64](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = SphericalDistance[float64](v1[i], v2[i]) + } + }) + + b.Run("Spherical Distance float32", func(b *testing.B) { + v1, v2 := randomVectors[float32](b.N, dim), randomVectors[float32](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = SphericalDistance[float32](v1[i], v2[i]) + } + }) +} + +/* +func Benchmark_ScaleInPlace(b *testing.B) { + dim := 1024 + + b.Run("ScaleInPlace float64", func(b *testing.B) { + v1 := randomVectors[float64](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + ScaleInPlace[float64](v1[i], 0.5) + } + }) + + b.Run("ScaleInPlace float32", func(b *testing.B) { + v1 := randomVectors[float32](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + ScaleInPlace[float32](v1[i], 0.5) + } + }) +} +*/ + +func randomVectors[T float32 | float64](size, dim int) [][]T { + vectors := make([][]T, size) for i := range vectors { + vectors[i] = make([]T, dim) for j := 0; j < dim; j++ { - vectors[i] = append(vectors[i], rand.Float64()) + vectors[i][j] = T(rand.Float64()) } } return vectors diff --git a/pkg/vectorindex/metric/distance_func_f32_test.go b/pkg/vectorindex/metric/distance_func_f32_test.go new file mode 100644 index 0000000000000..098ab6134add8 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_f32_test.go @@ -0,0 +1,583 @@ +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metric + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/common/assertx" +) + +func Test_L2Distance_F32(t *testing.T) { + type args struct { + v1 []float32 + v2 []float32 + } + tests := []struct { + name string + args args + want float32 + }{ + { + name: "Test 1", + args: args{ + v1: []float32{1, 2, 3, 4}, + v2: []float32{1, 2, 4, 5}, + }, + want: 1.4142135623730951, + }, + { + name: "Test 2", + args: args{ + v1: []float32{10, 20, 30, 40}, + v2: []float32{10.5, 21.5, 31.5, 43.5}, + }, + want: 4.123105625617661, + }, + { + name: "Test 3.a", + args: args{ + v1: []float32{1, 1}, + v2: []float32{4, 1}, + }, + want: 3, + }, + { + name: "Test 3.b", + args: args{ + v1: []float32{4, 1}, + v2: []float32{1, 4}, + }, + want: 4.242640687119285, + }, + { + name: "Test 3.c", + args: args{ + v1: []float32{1, 4}, + v2: []float32{1, 1}, + }, + want: 3, + }, + { + name: "Test 4", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + }, + want: 3.1622776601683795, + }, + { + name: "Test 5", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 5.196152422706632, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, err := L2Distance[float32](tt.args.v1, tt.args.v2); err != nil || got != tt.want { + t.Errorf("L2Distance() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_L1Distance_F32(t *testing.T) { + type args struct { + v1 []float32 + v2 []float32 + } + tests := []struct { + name string + args args + want float32 + }{ + { + name: "Test 1", + args: args{ + v1: []float32{1, 2, 3, 4}, + v2: []float32{1, 2, 4, 5}, + }, + want: 2, + }, + { + name: "Test 2", + args: args{ + v1: []float32{10, 20, 30, 40}, + v2: []float32{10.5, 21.5, 31.5, 43.5}, + }, + want: 7, + }, + { + name: "Test 3.a", + args: args{ + v1: []float32{1, 1}, + v2: []float32{4, 1}, + }, + want: 3, + }, + { + name: "Test 3.b", + args: args{ + v1: []float32{4, 1}, + v2: []float32{1, 4}, + }, + want: 6, + }, + { + name: "Test 3.c", + args: args{ + v1: []float32{1, 4}, + v2: []float32{1, 1}, + }, + want: 3, + }, + { + name: "Test 4", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + }, + want: 10, + }, + { + name: "Test 5", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 27, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, err := L1Distance[float32](tt.args.v1, tt.args.v2); err != nil || got != tt.want { + t.Errorf("L1Distance() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_CosineDistance_F32(t *testing.T) { + type args struct { + v1 []float32 + v2 []float32 + } + tests := []struct { + name string + args args + want float32 + }{ + { + name: "Test 1", + args: args{ + v1: []float32{1, 2, 3, 4}, + v2: []float32{1, 2, 4, 5}, + }, + want: 0.003993481192393733, + }, + { + name: "Test 2", + args: args{ + v1: []float32{10, 20, 30, 40}, + v2: []float32{10.5, 21.5, 31.5, 43.5}, + }, + want: 0.0001253573895874105, + }, + { + name: "Test 3.a", + args: args{ + v1: []float32{1, 1}, + v2: []float32{4, 1}, + }, + want: 0.1425070742874559, + }, + { + name: "Test 3.b", + args: args{ + v1: []float32{4, 1}, + v2: []float32{1, 4}, + }, + want: 0.5294117647058824, + }, + { + name: "Test 3.c", + args: args{ + v1: []float32{1, 4}, + v2: []float32{1, 1}, + }, + want: 0.1425070742874559, + }, + { + name: "Test 4", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + }, + want: 0.0021238962030426523, + }, + { + name: "Test 5", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 0.0025062434610066964, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, err := CosineDistance[float32](tt.args.v1, tt.args.v2); err != nil || got != tt.want { + t.Errorf("CosineDistance() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_CosineSimilarity_F32(t *testing.T) { + type args struct { + v1 []float32 + v2 []float32 + } + tests := []struct { + name string + args args + want float32 + }{ + { + name: "Test 1", + args: args{ + v1: []float32{1, 2, 3, 4}, + v2: []float32{1, 2, 4, 5}, + }, + want: 0.9960065188076063, + }, + { + name: "Test 2", + args: args{ + v1: []float32{10, 20, 30, 40}, + v2: []float32{10.5, 21.5, 31.5, 43.5}, + }, + want: 0.9998746426104126, + }, + { + name: "Test 3.a", + args: args{ + v1: []float32{1, 1}, + v2: []float32{4, 1}, + }, + want: 0.8574929257125441, + }, + { + name: "Test 3.b", + args: args{ + v1: []float32{4, 1}, + v2: []float32{1, 4}, + }, + want: 0.47058823529411764, + }, + { + name: "Test 3.c", + args: args{ + v1: []float32{1, 4}, + v2: []float32{1, 1}, + }, + want: 0.8574929257125441, + }, + { + name: "Test 4", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + }, + want: 0.9978761037969573, + }, + { + name: "Test 5", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 0.9974937565389933, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, err := CosineSimilarity[float32](tt.args.v1, tt.args.v2); err != nil || got != tt.want { + t.Errorf("CosineSimilarity() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_InnerProduct_F32(t *testing.T) { + type args struct { + v1 []float32 + v2 []float32 + } + tests := []struct { + name string + args args + want float32 + }{ + { + name: "Test 1", + args: args{ + v1: []float32{1, 2, 3, 4}, + v2: []float32{1, 2, 4, 5}, + }, + want: -37, + }, + { + name: "Test 2", + args: args{ + v1: []float32{10, 20, 30, 40}, + v2: []float32{10.5, 21.5, 31.5, 43.5}, + }, + want: -3220, + }, + { + name: "Test 3.a", + args: args{ + v1: []float32{1, 1}, + v2: []float32{4, 1}, + }, + want: -5, + }, + { + name: "Test 3.b", + args: args{ + v1: []float32{4, 1}, + v2: []float32{1, 4}, + }, + want: -8, + }, + { + name: "Test 3.c", + args: args{ + v1: []float32{1, 4}, + v2: []float32{1, 1}, + }, + want: -5, + }, + { + name: "Test 4", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + }, + want: -440, + }, + { + name: "Test 5", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: -1048, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, err := InnerProduct[float32](tt.args.v1, tt.args.v2); err != nil || got != tt.want { + t.Errorf("InnerProduct() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_L2DistanceSq_F32(t *testing.T) { + type args struct { + v1 []float32 + v2 []float32 + } + tests := []struct { + name string + args args + want float32 + }{ + { + name: "Test 1", + args: args{ + v1: []float32{1, 2, 3, 4}, + v2: []float32{1, 2, 4, 5}, + }, + want: 2, + }, + { + name: "Test 2", + args: args{ + v1: []float32{10, 20, 30, 40}, + v2: []float32{10.5, 21.5, 31.5, 43.5}, + }, + want: 17, + }, + { + name: "Test 3.a", + args: args{ + v1: []float32{1, 1}, + v2: []float32{4, 1}, + }, + want: 9, + }, + { + name: "Test 3.b", + args: args{ + v1: []float32{4, 1}, + v2: []float32{1, 4}, + }, + want: 18, + }, + { + name: "Test 3.c", + args: args{ + v1: []float32{1, 4}, + v2: []float32{1, 1}, + }, + want: 9, + }, + { + name: "Test 4", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + }, + want: 10, + }, + { + name: "Test 5", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 27, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, err := L2DistanceSq[float32](tt.args.v1, tt.args.v2); err != nil || got != tt.want { + t.Errorf("L2DistanceSq() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_AngularDistance_F32(t *testing.T) { + type args struct { + v1 []float32 + v2 []float32 + } + tests := []struct { + name string + args args + want float32 + }{ + { + name: "Test 1", + args: args{ + v1: []float32{1, 2, 3, 4}, + v2: []float32{1, 2, 4, 5}, + }, + want: 0, + }, + { + name: "Test 2", + args: args{ + v1: []float32{10, 20, 30, 40}, + v2: []float32{10.5, 21.5, 31.5, 43.5}, + }, + want: 0, + }, + // Test 3: Triangle Inequality check on **un-normalized** vector + // A(1,0),B(2,2), C(0,1) => AB + AC !>= BC => 0 + 0 !>= 0.5 + { + name: "Test 3.a", + args: args{ + v1: []float32{1, 0}, + v2: []float32{2, 2}, + }, + want: 0, + }, + { + name: "Test 3.b", + args: args{ + v1: []float32{2, 2}, + v2: []float32{0, 1}, + }, + want: 0, + }, + { + name: "Test 3.c", + args: args{ + v1: []float32{0, 1}, + v2: []float32{1, 0}, + }, + want: 0.5, + }, + { + name: "Test 4", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + }, + want: 0, + }, + { + name: "Test 5", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 0, + }, + + // Test 4: Triangle Inequality check on **normalized** vector + // A(1,0),B(2,2), C(0,1) => AB + AC >= BC => 0.25 + 0.25 >= 0.5 + //{ + // name: "Test 4.a", + // args: args{ + // v1: moarray.NormalizeMoVecf64([]float32{1, 0}), + // v2: moarray.NormalizeMoVecf64([]float32{2, 2}), + // }, + // want: 0.25000000000000006, + //}, + //{ + // name: "Test 4.b", + // args: args{ + // v1: moarray.NormalizeMoVecf64([]float32{2, 2}), + // v2: moarray.NormalizeMoVecf64([]float32{0, 1}), + // }, + // want: 0.25000000000000006, + //}, + //{ + // name: "Test 4.c", + // args: args{ + // v1: moarray.NormalizeMoVecf64([]float32{0, 1}), + // v2: moarray.NormalizeMoVecf64([]float32{1, 0}), + // }, + // want: 0.5, + //}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + if got, err := SphericalDistance[float32](tt.args.v1, tt.args.v2); err != nil || !assertx.InEpsilonF64(float64(got), float64(tt.want)) { + t.Errorf("SphericalDistance() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/vectorindex/metric/distance_func_narrow.go b/pkg/vectorindex/metric/distance_func_narrow.go new file mode 100644 index 0000000000000..663c6b6d4d0f4 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow.go @@ -0,0 +1,450 @@ +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Distance kernels for the narrow vector element types: vecbf16 (types.BF16), +// vecf16 (types.Float16), vecint8 (int8). Pure Go, loop-unrolled. Untagged so +// it compiles in both the scalar and SIMD builds (a narrow SIMD variant can be +// split out later as distance_func_narrow_amd64.go, with this as the fallback + +// equivalence oracle). +// +// Semantics MATCH ResolveDistanceFn (the float32 path): +// - Metric_L2Distance / Metric_L2sqDistance -> squared L2 (caller sqrts L2) +// - Metric_InnerProduct -> -dot +// - Metric_CosineDistance -> 1 - similarity +// - Metric_L1Distance -> sum|a-b| +// +// bf16/f16 decode to float32 and reuse the float32 kernels (Go has no native +// fp16 arithmetic). int8 uses INTEGER (int64-accumulated) kernels — no float +// upcast in the inner loop — with only the cosine denominator going through +// float for the sqrt/divide. + +package metric + +import ( + "math" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +// The native narrow kernels return float32 — bf16/f16 accumulate in float32, and +// int8/uint8 cast their int64 accumulator down at the end. The merged +// ResolveDistanceFn[T, R] in resolve.go dispatches to resolveBF16Kernel / +// resolveF16Kernel / resolveInt8Kernel / resolveUint8Kernel and casts to R. + +// ---------------------------------------------------------------------------- +// bf16 / f16 CONCRETE fused kernels (unroll-8). NOT generic: a generic +// A generic [T] would share the uint16 gcshape, so .ToFloat32() would become a +// dictionary (virtual) call per element and never inlines. Concrete types let +// .ToFloat32() inline (bf16 = one shift). Decode inline, accumulate in float32, +// no slice materialized -> zero alloc. The multiply/add cannot be done without a +// float (bf16/f16 are floating-point; Go has no 16-bit-float ALU). +// ---------------------------------------------------------------------------- + +// bf16 kernel selection. The SIMD build (distance_func_narrow_amd64.go) swaps +// these to its archsimd implementations in init() when AVX-512 is available; +// otherwise they stay the pure-Go fallbacks defined below (which also remain the +// equivalence oracle the SIMD tests compare against). +var ( + bf16L2sqFn = l2sqBF16 + bf16IPFn = innerProductBF16 + bf16CosineFn = cosineDistanceBF16 + bf16L1Fn = l1DistanceBF16 +) + +func resolveBF16Kernel(metric MetricType) (func(a, b []types.BF16) (float64, error), error) { + switch metric { + case Metric_L2Distance, Metric_L2sqDistance: + return bf16L2sqFn, nil + case Metric_InnerProduct: + return bf16IPFn, nil + case Metric_CosineDistance: + return bf16CosineFn, nil + case Metric_L1Distance: + return bf16L1Fn, nil + default: + return nil, moerr.NewInternalErrorNoCtx("invalid distance type") + } +} + +func l2sqBF16(a, b []types.BF16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum float32 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + d0 := aa[0].ToFloat32() - bb[0].ToFloat32() + d1 := aa[1].ToFloat32() - bb[1].ToFloat32() + d2 := aa[2].ToFloat32() - bb[2].ToFloat32() + d3 := aa[3].ToFloat32() - bb[3].ToFloat32() + d4 := aa[4].ToFloat32() - bb[4].ToFloat32() + d5 := aa[5].ToFloat32() - bb[5].ToFloat32() + d6 := aa[6].ToFloat32() - bb[6].ToFloat32() + d7 := aa[7].ToFloat32() - bb[7].ToFloat32() + sum += (d0*d0 + d1*d1) + (d2*d2 + d3*d3) + (d4*d4 + d5*d5) + (d6*d6 + d7*d7) + } + for ; i < n; i++ { + d := a[i].ToFloat32() - b[i].ToFloat32() + sum += d * d + } + return float64(sum), nil +} + +func innerProductBF16(a, b []types.BF16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum float32 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + sum += (aa[0].ToFloat32()*bb[0].ToFloat32() + aa[1].ToFloat32()*bb[1].ToFloat32()) + + (aa[2].ToFloat32()*bb[2].ToFloat32() + aa[3].ToFloat32()*bb[3].ToFloat32()) + + (aa[4].ToFloat32()*bb[4].ToFloat32() + aa[5].ToFloat32()*bb[5].ToFloat32()) + + (aa[6].ToFloat32()*bb[6].ToFloat32() + aa[7].ToFloat32()*bb[7].ToFloat32()) + } + for ; i < n; i++ { + sum += a[i].ToFloat32() * b[i].ToFloat32() + } + return float64(-sum), nil +} + +func l1DistanceBF16(a, b []types.BF16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum float32 + for i := range a { + d := a[i].ToFloat32() - b[i].ToFloat32() + if d < 0 { + d = -d + } + sum += d + } + return float64(sum), nil +} + +// cosineDistClamped mirrors metric.CosineDistance: clamp the cosine similarity to +// [-1,1] (float32 accumulation / fp16 decode can push it a hair outside) before +// distance = 1 - sim, so a near-parallel pair never yields a tiny negative +// distance that would mis-sort in a top-k scan. +func cosineDistClamped(dot, denom float64) float64 { + sim := dot / denom + if sim > 1 { + sim = 1 + } else if sim < -1 { + sim = -1 + } + return 1.0 - sim +} + +func cosineDistanceBF16(a, b []types.BF16) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var dot, na2, nb2 float32 + for i := range a { + ai := a[i].ToFloat32() + bi := b[i].ToFloat32() + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} + +// magic-multiply branchless half->float (Fabian Giesen / rygorous, +// https://gist.github.com/rygorous/2156668). No loop and no fallback call, so it +// inlines into the kernels — unlike types.Float16.ToFloat32, whose subnormal +// normalization loop blocks inlining. The magic multiply rescales the exponent +// and turns half subnormals into the right normal floats in one step; the single +// branch only fixes up Inf/NaN. Verified EXHAUSTIVELY against ToFloat32 over all +// 65536 inputs (TestF16FastExhaustive). +var ( + f16Magic = math.Float32frombits(uint32(254-15) << 23) + f16WasInfNan = math.Float32frombits(uint32(127+16) << 23) +) + +func f16fast(h types.Float16) float32 { + o := uint32(h&0x7fff) << 13 // exponent/mantissa bits, into f32 position + of := math.Float32frombits(o) * f16Magic // rescale exponent; subnormals -> normals + ou := math.Float32bits(of) + if of >= f16WasInfNan { // Inf/NaN -> max exponent + ou |= 255 << 23 + } + ou |= uint32(h&0x8000) << 16 // sign + return math.Float32frombits(ou) +} + +// f16 kernel selection — swapped to archsimd impls by distance_func_narrow_f16_amd64.go. +var ( + f16L2sqFn = l2sqF16 + f16IPFn = innerProductF16 + f16CosineFn = cosineDistanceF16 + f16L1Fn = l1DistanceF16 +) + +func resolveF16Kernel(metric MetricType) (func(a, b []types.Float16) (float64, error), error) { + switch metric { + case Metric_L2Distance, Metric_L2sqDistance: + return f16L2sqFn, nil + case Metric_InnerProduct: + return f16IPFn, nil + case Metric_CosineDistance: + return f16CosineFn, nil + case Metric_L1Distance: + return f16L1Fn, nil + default: + return nil, moerr.NewInternalErrorNoCtx("invalid distance type") + } +} + +func l2sqF16(a, b []types.Float16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum float32 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + d0 := f16fast(aa[0]) - f16fast(bb[0]) + d1 := f16fast(aa[1]) - f16fast(bb[1]) + d2 := f16fast(aa[2]) - f16fast(bb[2]) + d3 := f16fast(aa[3]) - f16fast(bb[3]) + d4 := f16fast(aa[4]) - f16fast(bb[4]) + d5 := f16fast(aa[5]) - f16fast(bb[5]) + d6 := f16fast(aa[6]) - f16fast(bb[6]) + d7 := f16fast(aa[7]) - f16fast(bb[7]) + sum += (d0*d0 + d1*d1) + (d2*d2 + d3*d3) + (d4*d4 + d5*d5) + (d6*d6 + d7*d7) + } + for ; i < n; i++ { + d := f16fast(a[i]) - f16fast(b[i]) + sum += d * d + } + return float64(sum), nil +} + +func innerProductF16(a, b []types.Float16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum float32 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + sum += (f16fast(aa[0])*f16fast(bb[0]) + f16fast(aa[1])*f16fast(bb[1])) + + (f16fast(aa[2])*f16fast(bb[2]) + f16fast(aa[3])*f16fast(bb[3])) + + (f16fast(aa[4])*f16fast(bb[4]) + f16fast(aa[5])*f16fast(bb[5])) + + (f16fast(aa[6])*f16fast(bb[6]) + f16fast(aa[7])*f16fast(bb[7])) + } + for ; i < n; i++ { + sum += f16fast(a[i]) * f16fast(b[i]) + } + return float64(-sum), nil +} + +func l1DistanceF16(a, b []types.Float16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum float32 + for i := range a { + d := f16fast(a[i]) - f16fast(b[i]) + if d < 0 { + d = -d + } + sum += d + } + return float64(sum), nil +} + +func cosineDistanceF16(a, b []types.Float16) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var dot, na2, nb2 float32 + for i := range a { + ai := f16fast(a[i]) + bi := f16fast(b[i]) + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} + +// int8 kernel selection — swapped to archsimd impls by distance_func_narrow_int8_amd64.go. +var ( + int8L2sqFn = l2sqInt8 + int8IPFn = innerProductInt8 + int8CosineFn = cosineDistanceInt8 + int8L1Fn = l1DistanceInt8 +) + +func resolveInt8Kernel(metric MetricType) (func(a, b []int8) (float64, error), error) { + switch metric { + case Metric_L2Distance, Metric_L2sqDistance: + return int8L2sqFn, nil + case Metric_InnerProduct: + return int8IPFn, nil + case Metric_CosineDistance: + return int8CosineFn, nil + case Metric_L1Distance: + return int8L1Fn, nil + default: + return nil, moerr.NewInternalErrorNoCtx("invalid distance type") + } +} + +// ---------------------------------------------------------------------------- +// int8 integer kernels (unroll-8). Accumulate in int64: for int8 inputs the +// per-element term is bounded (|d|<=255 -> d*d<=65025, a*b in [-16384,16129]), +// and MaxArrayDimension is 65535, so int32 could overflow — int64 cannot. +// ---------------------------------------------------------------------------- + +func l2sqInt8(a, b []int8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum int64 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + d0 := int32(aa[0]) - int32(bb[0]) + d1 := int32(aa[1]) - int32(bb[1]) + d2 := int32(aa[2]) - int32(bb[2]) + d3 := int32(aa[3]) - int32(bb[3]) + d4 := int32(aa[4]) - int32(bb[4]) + d5 := int32(aa[5]) - int32(bb[5]) + d6 := int32(aa[6]) - int32(bb[6]) + d7 := int32(aa[7]) - int32(bb[7]) + sum += int64(d0*d0+d1*d1) + int64(d2*d2+d3*d3) + int64(d4*d4+d5*d5) + int64(d6*d6+d7*d7) + } + for ; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + sum += int64(d * d) + } + return float64(sum), nil +} + +func innerProductInt8(a, b []int8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum int64 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + sum += int64(int32(aa[0])*int32(bb[0])+int32(aa[1])*int32(bb[1])) + + int64(int32(aa[2])*int32(bb[2])+int32(aa[3])*int32(bb[3])) + + int64(int32(aa[4])*int32(bb[4])+int32(aa[5])*int32(bb[5])) + + int64(int32(aa[6])*int32(bb[6])+int32(aa[7])*int32(bb[7])) + } + for ; i < n; i++ { + sum += int64(int32(a[i]) * int32(b[i])) + } + // matches metric.InnerProduct: returns -dot + return float64(-sum), nil +} + +func l1DistanceInt8(a, b []int8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum int64 + n := len(a) + i := 0 + abs := func(x int32) int32 { + if x < 0 { + return -x + } + return x + } + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + sum += int64(abs(int32(aa[0])-int32(bb[0]))+abs(int32(aa[1])-int32(bb[1]))) + + int64(abs(int32(aa[2])-int32(bb[2]))+abs(int32(aa[3])-int32(bb[3]))) + + int64(abs(int32(aa[4])-int32(bb[4]))+abs(int32(aa[5])-int32(bb[5]))) + + int64(abs(int32(aa[6])-int32(bb[6]))+abs(int32(aa[7])-int32(bb[7]))) + } + for ; i < n; i++ { + sum += int64(abs(int32(a[i]) - int32(b[i]))) + } + return float64(sum), nil +} + +func cosineDistanceInt8(a, b []int8) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var dot, na2, nb2 int64 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + for k := 0; k < 8; k++ { + ai := int64(aa[k]) + bi := int64(bb[k]) + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + } + for ; i < n; i++ { + ai := int64(a[i]) + bi := int64(b[i]) + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + // matches metric.CosineDistance: denominator 0 -> distance 1.0 + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_amd64.go b/pkg/vectorindex/metric/distance_func_narrow_amd64.go new file mode 100644 index 0000000000000..144f583e57758 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_amd64.go @@ -0,0 +1,187 @@ +//go:build amd64 && go1.26 && goexperiment.simd + +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// AVX-512 SIMD distance kernels for vecbf16 (types.BF16). +// +// bf16 is the high 16 bits of an IEEE float32, so the decode bf16->f32 is a pure +// bit op: value<<16. Go's archsimd has no bf16 type and no AVX512BF16 detector +// (and the native VDPBF16PS is therefore unreachable from Go), but we don't need +// it: load the raw bf16 bytes as Uint32x16 (32 bf16 per load), split the even and +// odd 16-bit halves into two Float32x16 vectors via one shift + one and-mask + +// AsFloat32x16 bitcast, then reuse the existing AVX-512 float32 reduction +// (sumF32x16 / hasAVX512 from distance_func_amd64.go — same package + build tag). +// +// The pure-Go kernels in distance_func_narrow.go stay the fallback (non-AVX512 +// CPUs) and the equivalence oracle; init() only swaps the selection vars when +// hasAVX512 is true. + +package metric + +import ( + "math" + "unsafe" + + "simd/archsimd" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +func init() { + switch { + case hasAVX512: + bf16L2sqFn = l2sqBF16SIMD + bf16IPFn = innerProductBF16SIMD + bf16CosineFn = cosineDistanceBF16SIMD + bf16L1Fn = l1DistanceBF16SIMD + case hasAVX2: + bf16L2sqFn = l2sqBF16AVX2 + bf16IPFn = innerProductBF16AVX2 + bf16CosineFn = cosineDistanceBF16AVX2 + bf16L1Fn = l1DistanceBF16AVX2 + } +} + +// bf16AsU32 reinterprets a []types.BF16 (uint16-backed) as []uint32 viewing its +// first len/2 even-aligned pairs. x86 tolerates the unaligned load; the stored +// bf16 bytes originate from an 8-aligned []byte (BytesToArray), so in practice +// the start is 4-aligned. +func bf16AsU32(s []types.BF16) []uint32 { + if len(s) < 2 { + return nil + } + return unsafe.Slice((*uint32)(unsafe.Pointer(unsafe.SliceData(s))), len(s)/2) +} + +func l2sqBF16SIMD(a, b []types.BF16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := bf16AsU32(a), bf16AsU32(b) + hi := archsimd.BroadcastUint32x16(0xFFFF0000) + acc0, acc1 := archsimd.Float32x16{}, archsimd.Float32x16{} + np, j := len(au), 0 + for ; j <= np-16; j += 16 { + ua := archsimd.LoadUint32x16Slice(au[j : j+16]) + ub := archsimd.LoadUint32x16Slice(bu[j : j+16]) + dE := ua.ShiftAllLeft(16).AsFloat32x16().Sub(ub.ShiftAllLeft(16).AsFloat32x16()) + dO := ua.And(hi).AsFloat32x16().Sub(ub.And(hi).AsFloat32x16()) + acc0 = dE.MulAdd(dE, acc0) + acc1 = dO.MulAdd(dO, acc1) + } + sum := sumF32x16(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + d := a[i].ToFloat32() - b[i].ToFloat32() + sum += d * d + } + return float64(sum), nil +} + +func innerProductBF16SIMD(a, b []types.BF16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := bf16AsU32(a), bf16AsU32(b) + hi := archsimd.BroadcastUint32x16(0xFFFF0000) + acc0, acc1 := archsimd.Float32x16{}, archsimd.Float32x16{} + np, j := len(au), 0 + for ; j <= np-16; j += 16 { + ua := archsimd.LoadUint32x16Slice(au[j : j+16]) + ub := archsimd.LoadUint32x16Slice(bu[j : j+16]) + acc0 = ua.ShiftAllLeft(16).AsFloat32x16().MulAdd(ub.ShiftAllLeft(16).AsFloat32x16(), acc0) + acc1 = ua.And(hi).AsFloat32x16().MulAdd(ub.And(hi).AsFloat32x16(), acc1) + } + sum := sumF32x16(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + sum += a[i].ToFloat32() * b[i].ToFloat32() + } + return float64(-sum), nil +} + +func l1DistanceBF16SIMD(a, b []types.BF16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := bf16AsU32(a), bf16AsU32(b) + hi := archsimd.BroadcastUint32x16(0xFFFF0000) + absMask := archsimd.BroadcastUint32x16(0x7FFFFFFF) + acc0, acc1 := archsimd.Float32x16{}, archsimd.Float32x16{} + np, j := len(au), 0 + for ; j <= np-16; j += 16 { + ua := archsimd.LoadUint32x16Slice(au[j : j+16]) + ub := archsimd.LoadUint32x16Slice(bu[j : j+16]) + dE := ua.ShiftAllLeft(16).AsFloat32x16().Sub(ub.ShiftAllLeft(16).AsFloat32x16()) + dO := ua.And(hi).AsFloat32x16().Sub(ub.And(hi).AsFloat32x16()) + acc0 = acc0.Add(dE.AsUint32x16().And(absMask).AsFloat32x16()) + acc1 = acc1.Add(dO.AsUint32x16().And(absMask).AsFloat32x16()) + } + sum := sumF32x16(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + d := a[i].ToFloat32() - b[i].ToFloat32() + if d < 0 { + d = -d + } + sum += d + } + return float64(sum), nil +} + +func cosineDistanceBF16SIMD(a, b []types.BF16) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := bf16AsU32(a), bf16AsU32(b) + hi := archsimd.BroadcastUint32x16(0xFFFF0000) + dot0, dot1 := archsimd.Float32x16{}, archsimd.Float32x16{} + na0, na1 := archsimd.Float32x16{}, archsimd.Float32x16{} + nb0, nb1 := archsimd.Float32x16{}, archsimd.Float32x16{} + np, j := len(au), 0 + for ; j <= np-16; j += 16 { + ua := archsimd.LoadUint32x16Slice(au[j : j+16]) + ub := archsimd.LoadUint32x16Slice(bu[j : j+16]) + aE := ua.ShiftAllLeft(16).AsFloat32x16() + aO := ua.And(hi).AsFloat32x16() + bE := ub.ShiftAllLeft(16).AsFloat32x16() + bO := ub.And(hi).AsFloat32x16() + dot0 = aE.MulAdd(bE, dot0) + dot1 = aO.MulAdd(bO, dot1) + na0 = aE.MulAdd(aE, na0) + na1 = aO.MulAdd(aO, na1) + nb0 = bE.MulAdd(bE, nb0) + nb1 = bO.MulAdd(bO, nb1) + } + dot := sumF32x16(dot0.Add(dot1)) + na2 := sumF32x16(na0.Add(na1)) + nb2 := sumF32x16(nb0.Add(nb1)) + for i := j * 2; i < n; i++ { + ai, bi := a[i].ToFloat32(), b[i].ToFloat32() + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_amd64_test.go b/pkg/vectorindex/metric/distance_func_narrow_amd64_test.go new file mode 100644 index 0000000000000..ad5b04461d5c9 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_amd64_test.go @@ -0,0 +1,349 @@ +//go:build amd64 && go1.26 && goexperiment.simd + +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// SIMD-build-only tests: the narrow archsimd kernels (bf16/f16/int8) coexist with +// their pure-Go twins here, so we can (a) prove they agree and (b) benchmark them +// head to head in one binary. Only built under `GOEXPERIMENT=simd GOAMD64=v3`. + +package metric + +import ( + "math" + "math/rand" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// dims exercise the 16-lane main loop (bf16/f16: 32/iter, int8: 64/iter) plus +// every tail remainder, including odd final elements. +var narrowSIMDDims = []int{1, 2, 3, 4, 7, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 1000, 1024, 1025} + +func randF32(dim int, r *rand.Rand) []float32 { + f := make([]float32, dim) + for i := range f { + f[i] = float32(r.Float64()*16 - 8) // [-8, 8) + } + return f +} +func randBF16(dim int, r *rand.Rand) []types.BF16 { return types.Float32ToBF16Slice(randF32(dim, r)) } +func randF16(dim int, r *rand.Rand) []types.Float16 { + return types.Float32ToFloat16Slice(randF32(dim, r)) +} +func randI8(dim int, r *rand.Rand) []int8 { + v := make([]int8, dim) + for i := range v { + v[i] = int8(r.Intn(255) - 127) + } + return v +} +func randU8(dim int, r *rand.Rand) []uint8 { + v := make([]uint8, dim) + for i := range v { + v[i] = uint8(r.Intn(256)) + } + return v +} + +// checkPair asserts a SIMD kernel matches its scalar oracle. exact=true requires +// bit-equality (integer int8 L2sq/IP/L1); otherwise a magnitude-scaled tolerance +// (float reductions reorder). +func checkPair(t *testing.T, name string, dim int, got, want float64, exact bool) { + t.Helper() + if exact { + require.Equal(t, want, got, "%s dim=%d", name, dim) + return + } + require.InDelta(t, want, got, 1e-4*(1+math.Abs(want)), "%s dim=%d", name, dim) +} + +func TestBF16SIMDMatchesScalar(t *testing.T) { + if !hasAVX512 { + t.Skip("AVX-512 not available") + } + r := rand.New(rand.NewSource(42)) + type k struct { + name string + simd, scalar func(a, b []types.BF16) (float64, error) + } + for _, kn := range []k{ + {"l2sq", l2sqBF16SIMD, l2sqBF16}, + {"innerproduct", innerProductBF16SIMD, innerProductBF16}, + {"l1", l1DistanceBF16SIMD, l1DistanceBF16}, + {"cosine", cosineDistanceBF16SIMD, cosineDistanceBF16}, + } { + for _, dim := range narrowSIMDDims { + a, b := randBF16(dim, r), randBF16(dim, r) + got, err := kn.simd(a, b) + require.NoError(t, err) + want, err := kn.scalar(a, b) + require.NoError(t, err) + checkPair(t, "bf16/"+kn.name, dim, got, want, false) + } + } +} + +func TestF16SIMDMatchesScalar(t *testing.T) { + if !hasAVX512 { + t.Skip("AVX-512 not available") + } + r := rand.New(rand.NewSource(7)) + type k struct { + name string + simd, scalar func(a, b []types.Float16) (float64, error) + } + for _, kn := range []k{ + {"l2sq", l2sqF16SIMD, l2sqF16}, + {"innerproduct", innerProductF16SIMD, innerProductF16}, + {"l1", l1DistanceF16SIMD, l1DistanceF16}, + {"cosine", cosineDistanceF16SIMD, cosineDistanceF16}, + } { + for _, dim := range narrowSIMDDims { + a, b := randF16(dim, r), randF16(dim, r) + got, err := kn.simd(a, b) + require.NoError(t, err) + want, err := kn.scalar(a, b) + require.NoError(t, err) + checkPair(t, "f16/"+kn.name, dim, got, want, false) + } + } +} + +func TestInt8SIMDMatchesScalar(t *testing.T) { + if !hasAVX512 { + t.Skip("AVX-512 not available") + } + r := rand.New(rand.NewSource(9)) + type k struct { + name string + simd, scalar func(a, b []int8) (float64, error) + exact bool // integer kernels are bit-exact; cosine goes through float + } + for _, kn := range []k{ + {"l2sq", l2sqInt8SIMD, l2sqInt8, true}, + {"innerproduct", innerProductInt8SIMD, innerProductInt8, true}, + {"l1", l1DistanceInt8SIMD, l1DistanceInt8, true}, + {"cosine", cosineDistanceInt8SIMD, cosineDistanceInt8, false}, + } { + for _, dim := range narrowSIMDDims { + a, b := randI8(dim, r), randI8(dim, r) + got, err := kn.simd(a, b) + require.NoError(t, err) + want, err := kn.scalar(a, b) + require.NoError(t, err) + checkPair(t, "int8/"+kn.name, dim, got, want, kn.exact) + } + } +} + +func TestUint8SIMDMatchesScalar(t *testing.T) { + if !hasAVX2 { + t.Skip("AVX2 not available") + } + r := rand.New(rand.NewSource(11)) + type k struct { + name string + simd, scalar func(a, b []uint8) (float64, error) + exact bool // integer kernels are bit-exact; cosine goes through float + } + simdSet := func() []k { + if hasAVX512 { + return []k{ + {"l2sq", l2sqUint8SIMD, l2sqUint8, true}, + {"innerproduct", innerProductUint8SIMD, innerProductUint8, true}, + {"l1", l1DistanceUint8SIMD, l1DistanceUint8, true}, + {"cosine", cosineDistanceUint8SIMD, cosineDistanceUint8, false}, + } + } + return []k{ + {"l2sq", l2sqUint8AVX2, l2sqUint8, true}, + {"innerproduct", innerProductUint8AVX2, innerProductUint8, true}, + {"l1", l1DistanceUint8AVX2, l1DistanceUint8, true}, + {"cosine", cosineDistanceUint8AVX2, cosineDistanceUint8, false}, + } + } + for _, kn := range simdSet() { + for _, dim := range narrowSIMDDims { + a, b := randU8(dim, r), randU8(dim, r) + got, err := kn.simd(a, b) + require.NoError(t, err) + want, err := kn.scalar(a, b) + require.NoError(t, err) + checkPair(t, "uint8/"+kn.name, dim, got, want, kn.exact) + } + } +} + +// ---- head-to-head benchmarks (dim=1024, same binary) ---- +// +// GOEXPERIMENT=simd GOAMD64=v3 go test ./pkg/vectorindex/metric/ \ +// -run x -bench Benchmark_Narrow_SIMDvsScalar -benchmem + +func Benchmark_Narrow_SIMDvsScalar(b *testing.B) { + const dim = 1024 + r := rand.New(rand.NewSource(1)) + bf16a, bf16b := randBF16(dim, r), randBF16(dim, r) + f16a, f16b := randF16(dim, r), randF16(dim, r) + i8a, i8b := randI8(dim, r), randI8(dim, r) + u8a, u8b := randU8(dim, r), randU8(dim, r) + f32a, f32b := randF32(dim, r), randF32(dim, r) + f64a := make([]float64, dim) + f64b := make([]float64, dim) + for i := range f64a { + f64a[i] = float64(f32a[i]) + f64b[i] = float64(f32b[i]) + } + + runF32 := func(b *testing.B, fn func(a, c []float32) (float32, error)) { + for i := 0; i < b.N; i++ { + _, _ = fn(f32a, f32b) + } + } + runF64 := func(b *testing.B, fn func(a, c []float64) (float64, error)) { + for i := 0; i < b.N; i++ { + _, _ = fn(f64a, f64b) + } + } + runBF16 := func(b *testing.B, fn func(a, c []types.BF16) (float64, error)) { + for i := 0; i < b.N; i++ { + _, _ = fn(bf16a, bf16b) + } + } + runF16 := func(b *testing.B, fn func(a, c []types.Float16) (float64, error)) { + for i := 0; i < b.N; i++ { + _, _ = fn(f16a, f16b) + } + } + runI8 := func(b *testing.B, fn func(a, c []int8) (float64, error)) { + for i := 0; i < b.N; i++ { + _, _ = fn(i8a, i8b) + } + } + runU8 := func(b *testing.B, fn func(a, c []uint8) (float64, error)) { + for i := 0; i < b.N; i++ { + _, _ = fn(u8a, u8b) + } + } + + // avx512 sub-benchmarks call the x16 kernels directly, so they only run when + // the CPU has AVX-512 (calling them otherwise would fault). avx2 always runs + // (AVX2 is implied by the GOAMD64=v3 build). This bench bypasses the function- + // pointer selection on purpose, so a single run shows all three tiers + // side-by-side regardless of the MO_METRIC_NO_AVX512/AVX2 overrides. + // f32/f64 native baselines (no decode). These auto-dispatch to AVX-512 + // internally via hasAVX512; with MO_METRIC_NO_AVX512=1 they fall to scalar Go. + b.Run("f32", func(b *testing.B) { + b.Run("l2sq", func(b *testing.B) { runF32(b, L2DistanceSq[float32]) }) + b.Run("innerproduct", func(b *testing.B) { runF32(b, InnerProduct[float32]) }) + b.Run("l1", func(b *testing.B) { runF32(b, L1Distance[float32]) }) + b.Run("cosine", func(b *testing.B) { runF32(b, CosineDistance[float32]) }) + }) + b.Run("f64", func(b *testing.B) { + b.Run("l2sq", func(b *testing.B) { runF64(b, L2DistanceSq[float64]) }) + b.Run("innerproduct", func(b *testing.B) { runF64(b, InnerProduct[float64]) }) + b.Run("l1", func(b *testing.B) { runF64(b, L1Distance[float64]) }) + b.Run("cosine", func(b *testing.B) { runF64(b, CosineDistance[float64]) }) + }) + b.Run("bf16", func(b *testing.B) { + b.Run("l2sq/scalar", func(b *testing.B) { runBF16(b, l2sqBF16) }) + b.Run("l2sq/avx2", func(b *testing.B) { runBF16(b, l2sqBF16AVX2) }) + if hasAVX512 { + b.Run("l2sq/avx512", func(b *testing.B) { runBF16(b, l2sqBF16SIMD) }) + } + b.Run("innerproduct/scalar", func(b *testing.B) { runBF16(b, innerProductBF16) }) + b.Run("innerproduct/avx2", func(b *testing.B) { runBF16(b, innerProductBF16AVX2) }) + if hasAVX512 { + b.Run("innerproduct/avx512", func(b *testing.B) { runBF16(b, innerProductBF16SIMD) }) + } + b.Run("l1/scalar", func(b *testing.B) { runBF16(b, l1DistanceBF16) }) + b.Run("l1/avx2", func(b *testing.B) { runBF16(b, l1DistanceBF16AVX2) }) + if hasAVX512 { + b.Run("l1/avx512", func(b *testing.B) { runBF16(b, l1DistanceBF16SIMD) }) + } + b.Run("cosine/scalar", func(b *testing.B) { runBF16(b, cosineDistanceBF16) }) + b.Run("cosine/avx2", func(b *testing.B) { runBF16(b, cosineDistanceBF16AVX2) }) + if hasAVX512 { + b.Run("cosine/avx512", func(b *testing.B) { runBF16(b, cosineDistanceBF16SIMD) }) + } + }) + b.Run("f16", func(b *testing.B) { + b.Run("l2sq/scalar", func(b *testing.B) { runF16(b, l2sqF16) }) + b.Run("l2sq/avx2", func(b *testing.B) { runF16(b, l2sqF16AVX2) }) + if hasAVX512 { + b.Run("l2sq/avx512", func(b *testing.B) { runF16(b, l2sqF16SIMD) }) + } + b.Run("innerproduct/scalar", func(b *testing.B) { runF16(b, innerProductF16) }) + b.Run("innerproduct/avx2", func(b *testing.B) { runF16(b, innerProductF16AVX2) }) + if hasAVX512 { + b.Run("innerproduct/avx512", func(b *testing.B) { runF16(b, innerProductF16SIMD) }) + } + b.Run("l1/scalar", func(b *testing.B) { runF16(b, l1DistanceF16) }) + b.Run("l1/avx2", func(b *testing.B) { runF16(b, l1DistanceF16AVX2) }) + if hasAVX512 { + b.Run("l1/avx512", func(b *testing.B) { runF16(b, l1DistanceF16SIMD) }) + } + b.Run("cosine/scalar", func(b *testing.B) { runF16(b, cosineDistanceF16) }) + b.Run("cosine/avx2", func(b *testing.B) { runF16(b, cosineDistanceF16AVX2) }) + if hasAVX512 { + b.Run("cosine/avx512", func(b *testing.B) { runF16(b, cosineDistanceF16SIMD) }) + } + }) + b.Run("int8", func(b *testing.B) { + b.Run("l2sq/scalar", func(b *testing.B) { runI8(b, l2sqInt8) }) + b.Run("l2sq/avx2", func(b *testing.B) { runI8(b, l2sqInt8AVX2) }) + if hasAVX512 { + b.Run("l2sq/avx512", func(b *testing.B) { runI8(b, l2sqInt8SIMD) }) + } + b.Run("innerproduct/scalar", func(b *testing.B) { runI8(b, innerProductInt8) }) + b.Run("innerproduct/avx2", func(b *testing.B) { runI8(b, innerProductInt8AVX2) }) + if hasAVX512 { + b.Run("innerproduct/avx512", func(b *testing.B) { runI8(b, innerProductInt8SIMD) }) + } + b.Run("l1/scalar", func(b *testing.B) { runI8(b, l1DistanceInt8) }) + b.Run("l1/avx2", func(b *testing.B) { runI8(b, l1DistanceInt8AVX2) }) + if hasAVX512 { + b.Run("l1/avx512", func(b *testing.B) { runI8(b, l1DistanceInt8SIMD) }) + } + b.Run("cosine/scalar", func(b *testing.B) { runI8(b, cosineDistanceInt8) }) + b.Run("cosine/avx2", func(b *testing.B) { runI8(b, cosineDistanceInt8AVX2) }) + if hasAVX512 { + b.Run("cosine/avx512", func(b *testing.B) { runI8(b, cosineDistanceInt8SIMD) }) + } + }) + b.Run("uint8", func(b *testing.B) { + b.Run("l2sq/scalar", func(b *testing.B) { runU8(b, l2sqUint8) }) + b.Run("l2sq/avx2", func(b *testing.B) { runU8(b, l2sqUint8AVX2) }) + if hasAVX512 { + b.Run("l2sq/avx512", func(b *testing.B) { runU8(b, l2sqUint8SIMD) }) + } + b.Run("innerproduct/scalar", func(b *testing.B) { runU8(b, innerProductUint8) }) + b.Run("innerproduct/avx2", func(b *testing.B) { runU8(b, innerProductUint8AVX2) }) + if hasAVX512 { + b.Run("innerproduct/avx512", func(b *testing.B) { runU8(b, innerProductUint8SIMD) }) + } + b.Run("l1/scalar", func(b *testing.B) { runU8(b, l1DistanceUint8) }) + b.Run("l1/avx2", func(b *testing.B) { runU8(b, l1DistanceUint8AVX2) }) + if hasAVX512 { + b.Run("l1/avx512", func(b *testing.B) { runU8(b, l1DistanceUint8SIMD) }) + } + b.Run("cosine/scalar", func(b *testing.B) { runU8(b, cosineDistanceUint8) }) + b.Run("cosine/avx2", func(b *testing.B) { runU8(b, cosineDistanceUint8AVX2) }) + if hasAVX512 { + b.Run("cosine/avx512", func(b *testing.B) { runU8(b, cosineDistanceUint8SIMD) }) + } + }) +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_avx2_amd64.go b/pkg/vectorindex/metric/distance_func_narrow_avx2_amd64.go new file mode 100644 index 0000000000000..5583891e18a8c --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_avx2_amd64.go @@ -0,0 +1,431 @@ +//go:build amd64 && go1.26 && goexperiment.simd + +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// AVX2 (256-bit, 8-lane) narrow distance kernels — the middle fallback tier for +// CPUs that have AVX2 but not AVX-512. Mirrors the AVX-512 (x16) kernels exactly +// with Float32x8 / Int32x8 / Uint32x8 ops. Each type's init() (in its x16 file) +// selects AVX-512 -> AVX2 -> scalar. +// +// The narrow types benefit from AVX2 (unlike f32, where AVX2 ~= scalar): their +// decode (bf16 shift / int8 sign-extend / f16 magic-multiply) is pure scalar +// overhead that AVX2 vectorizes. Measured ~7-9x over scalar at dim=1024; AVX-512 +// adds another ~1.3-1.5x for bf16/f16, and ~nothing for int8 (byte-unpack bound). + +package metric + +import ( + "math" + "os" + + "simd/archsimd" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +// hasAVX2 gates the middle tier. AVX512 implies AVX2, so init() checks hasAVX512 +// first; this only decides AVX2-vs-scalar on non-AVX512 CPUs. +// TESTING-ONLY override: set MO_METRIC_NO_AVX2=1 (typically with +// MO_METRIC_NO_AVX512=1) to force the pure-Go scalar fallback for coverage. +var hasAVX2 = archsimd.X86.AVX2() && os.Getenv("MO_METRIC_NO_AVX2") == "" + +func sumF32x8(v archsimd.Float32x8) float32 { + var a [8]float32 + v.Store(&a) + return ((a[0] + a[1]) + (a[2] + a[3])) + ((a[4] + a[5]) + (a[6] + a[7])) +} + +func sumI32x8(v archsimd.Int32x8) int64 { + var a [8]int32 + v.Store(&a) + var s int64 + for _, x := range a { + s += int64(x) + } + return s +} + +// ---- bf16 (AVX2) ---- + +func l2sqBF16AVX2(a, b []types.BF16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := bf16AsU32(a), bf16AsU32(b) + hi := archsimd.BroadcastUint32x8(0xFFFF0000) + acc0, acc1 := archsimd.Float32x8{}, archsimd.Float32x8{} + np, j := len(au), 0 + for ; j <= np-8; j += 8 { + ua := archsimd.LoadUint32x8Slice(au[j : j+8]) + ub := archsimd.LoadUint32x8Slice(bu[j : j+8]) + dE := ua.ShiftAllLeft(16).AsFloat32x8().Sub(ub.ShiftAllLeft(16).AsFloat32x8()) + dO := ua.And(hi).AsFloat32x8().Sub(ub.And(hi).AsFloat32x8()) + acc0 = dE.MulAdd(dE, acc0) + acc1 = dO.MulAdd(dO, acc1) + } + sum := sumF32x8(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + d := a[i].ToFloat32() - b[i].ToFloat32() + sum += d * d + } + return float64(sum), nil +} + +func innerProductBF16AVX2(a, b []types.BF16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := bf16AsU32(a), bf16AsU32(b) + hi := archsimd.BroadcastUint32x8(0xFFFF0000) + acc0, acc1 := archsimd.Float32x8{}, archsimd.Float32x8{} + np, j := len(au), 0 + for ; j <= np-8; j += 8 { + ua := archsimd.LoadUint32x8Slice(au[j : j+8]) + ub := archsimd.LoadUint32x8Slice(bu[j : j+8]) + acc0 = ua.ShiftAllLeft(16).AsFloat32x8().MulAdd(ub.ShiftAllLeft(16).AsFloat32x8(), acc0) + acc1 = ua.And(hi).AsFloat32x8().MulAdd(ub.And(hi).AsFloat32x8(), acc1) + } + sum := sumF32x8(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + sum += a[i].ToFloat32() * b[i].ToFloat32() + } + return float64(-sum), nil +} + +func l1DistanceBF16AVX2(a, b []types.BF16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := bf16AsU32(a), bf16AsU32(b) + hi := archsimd.BroadcastUint32x8(0xFFFF0000) + absMask := archsimd.BroadcastUint32x8(0x7FFFFFFF) + acc0, acc1 := archsimd.Float32x8{}, archsimd.Float32x8{} + np, j := len(au), 0 + for ; j <= np-8; j += 8 { + ua := archsimd.LoadUint32x8Slice(au[j : j+8]) + ub := archsimd.LoadUint32x8Slice(bu[j : j+8]) + dE := ua.ShiftAllLeft(16).AsFloat32x8().Sub(ub.ShiftAllLeft(16).AsFloat32x8()) + dO := ua.And(hi).AsFloat32x8().Sub(ub.And(hi).AsFloat32x8()) + acc0 = acc0.Add(dE.AsUint32x8().And(absMask).AsFloat32x8()) + acc1 = acc1.Add(dO.AsUint32x8().And(absMask).AsFloat32x8()) + } + sum := sumF32x8(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + d := a[i].ToFloat32() - b[i].ToFloat32() + if d < 0 { + d = -d + } + sum += d + } + return float64(sum), nil +} + +func cosineDistanceBF16AVX2(a, b []types.BF16) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := bf16AsU32(a), bf16AsU32(b) + hi := archsimd.BroadcastUint32x8(0xFFFF0000) + dot0, dot1 := archsimd.Float32x8{}, archsimd.Float32x8{} + na0, na1 := archsimd.Float32x8{}, archsimd.Float32x8{} + nb0, nb1 := archsimd.Float32x8{}, archsimd.Float32x8{} + np, j := len(au), 0 + for ; j <= np-8; j += 8 { + ua := archsimd.LoadUint32x8Slice(au[j : j+8]) + ub := archsimd.LoadUint32x8Slice(bu[j : j+8]) + aE := ua.ShiftAllLeft(16).AsFloat32x8() + aO := ua.And(hi).AsFloat32x8() + bE := ub.ShiftAllLeft(16).AsFloat32x8() + bO := ub.And(hi).AsFloat32x8() + dot0 = aE.MulAdd(bE, dot0) + dot1 = aO.MulAdd(bO, dot1) + na0 = aE.MulAdd(aE, na0) + na1 = aO.MulAdd(aO, na1) + nb0 = bE.MulAdd(bE, nb0) + nb1 = bO.MulAdd(bO, nb1) + } + dot := sumF32x8(dot0.Add(dot1)) + na2 := sumF32x8(na0.Add(na1)) + nb2 := sumF32x8(nb0.Add(nb1)) + for i := j * 2; i < n; i++ { + ai, bi := a[i].ToFloat32(), b[i].ToFloat32() + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} + +// ---- int8 (AVX2), integer-exact ---- + +func unpackI8x8(u archsimd.Int32x8) (v0, v1, v2, v3 archsimd.Int32x8) { + v0 = u.ShiftAllLeft(24).ShiftAllRight(24) + v1 = u.ShiftAllLeft(16).ShiftAllRight(24) + v2 = u.ShiftAllLeft(8).ShiftAllRight(24) + v3 = u.ShiftAllRight(24) + return +} + +func l2sqInt8AVX2(a, b []int8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + ai, bi := int8AsI32(a), int8AsI32(b) + acc := archsimd.Int32x8{} + nq, j := len(ai), 0 + for ; j <= nq-8; j += 8 { + a0, a1, a2, a3 := unpackI8x8(archsimd.LoadInt32x8Slice(ai[j : j+8])) + b0, b1, b2, b3 := unpackI8x8(archsimd.LoadInt32x8Slice(bi[j : j+8])) + d0, d1, d2, d3 := a0.Sub(b0), a1.Sub(b1), a2.Sub(b2), a3.Sub(b3) + acc = acc.Add(d0.Mul(d0).Add(d1.Mul(d1)).Add(d2.Mul(d2).Add(d3.Mul(d3)))) + } + sum := sumI32x8(acc) + for i := j * 4; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + sum += int64(d * d) + } + return float64(sum), nil +} + +func innerProductInt8AVX2(a, b []int8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + ai, bi := int8AsI32(a), int8AsI32(b) + acc := archsimd.Int32x8{} + nq, j := len(ai), 0 + for ; j <= nq-8; j += 8 { + a0, a1, a2, a3 := unpackI8x8(archsimd.LoadInt32x8Slice(ai[j : j+8])) + b0, b1, b2, b3 := unpackI8x8(archsimd.LoadInt32x8Slice(bi[j : j+8])) + acc = acc.Add(a0.Mul(b0).Add(a1.Mul(b1)).Add(a2.Mul(b2).Add(a3.Mul(b3)))) + } + sum := sumI32x8(acc) + for i := j * 4; i < n; i++ { + sum += int64(int32(a[i]) * int32(b[i])) + } + return float64(-sum), nil +} + +func l1DistanceInt8AVX2(a, b []int8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + ai, bi := int8AsI32(a), int8AsI32(b) + zero := archsimd.Int32x8{} + acc := archsimd.Int32x8{} + abs := func(d archsimd.Int32x8) archsimd.Int32x8 { return d.Max(zero.Sub(d)) } + nq, j := len(ai), 0 + for ; j <= nq-8; j += 8 { + a0, a1, a2, a3 := unpackI8x8(archsimd.LoadInt32x8Slice(ai[j : j+8])) + b0, b1, b2, b3 := unpackI8x8(archsimd.LoadInt32x8Slice(bi[j : j+8])) + acc = acc.Add(abs(a0.Sub(b0)).Add(abs(a1.Sub(b1))).Add(abs(a2.Sub(b2)).Add(abs(a3.Sub(b3))))) + } + sum := sumI32x8(acc) + for i := j * 4; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + if d < 0 { + d = -d + } + sum += int64(d) + } + return float64(sum), nil +} + +func cosineDistanceInt8AVX2(a, b []int8) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + ai, bi := int8AsI32(a), int8AsI32(b) + dotA, naA, nbA := archsimd.Int32x8{}, archsimd.Int32x8{}, archsimd.Int32x8{} + nq, j := len(ai), 0 + for ; j <= nq-8; j += 8 { + a0, a1, a2, a3 := unpackI8x8(archsimd.LoadInt32x8Slice(ai[j : j+8])) + b0, b1, b2, b3 := unpackI8x8(archsimd.LoadInt32x8Slice(bi[j : j+8])) + dotA = dotA.Add(a0.Mul(b0).Add(a1.Mul(b1)).Add(a2.Mul(b2).Add(a3.Mul(b3)))) + naA = naA.Add(a0.Mul(a0).Add(a1.Mul(a1)).Add(a2.Mul(a2).Add(a3.Mul(a3)))) + nbA = nbA.Add(b0.Mul(b0).Add(b1.Mul(b1)).Add(b2.Mul(b2).Add(b3.Mul(b3)))) + } + dot, na2, nb2 := sumI32x8(dotA), sumI32x8(naA), sumI32x8(nbA) + for i := j * 4; i < n; i++ { + ai8, bi8 := int64(a[i]), int64(b[i]) + dot += ai8 * bi8 + na2 += ai8 * ai8 + nb2 += bi8 * bi8 + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} + +// ---- f16 (AVX2) ---- + +func f16decX8(h, m7fff, m8000, mInf archsimd.Uint32x8, magic, infNan archsimd.Float32x8) archsimd.Float32x8 { + o := h.And(m7fff).ShiftAllLeft(13) + of := o.AsFloat32x8().Mul(magic) + ou := of.AsUint32x8() + ou = ou.Or(mInf).Merge(ou, of.GreaterEqual(infNan)) + return ou.Or(h.And(m8000).ShiftAllLeft(16)).AsFloat32x8() +} + +func f16DecodeConstsX8() (m7fff, m8000, mLo, mInf archsimd.Uint32x8, magic, infNan archsimd.Float32x8) { + m7fff = archsimd.BroadcastUint32x8(0x7fff) + m8000 = archsimd.BroadcastUint32x8(0x8000) + mLo = archsimd.BroadcastUint32x8(0xffff) + mInf = archsimd.BroadcastUint32x8(255 << 23) + magic = archsimd.BroadcastFloat32x8(f16Magic) + infNan = archsimd.BroadcastFloat32x8(f16WasInfNan) + return +} + +func l2sqF16AVX2(a, b []types.Float16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := f16AsU32(a), f16AsU32(b) + m7fff, m8000, mLo, mInf, magic, infNan := f16DecodeConstsX8() + acc0, acc1 := archsimd.Float32x8{}, archsimd.Float32x8{} + np, j := len(au), 0 + for ; j <= np-8; j += 8 { + ua := archsimd.LoadUint32x8Slice(au[j : j+8]) + ub := archsimd.LoadUint32x8Slice(bu[j : j+8]) + dE := f16decX8(ua.And(mLo), m7fff, m8000, mInf, magic, infNan).Sub(f16decX8(ub.And(mLo), m7fff, m8000, mInf, magic, infNan)) + dO := f16decX8(ua.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan).Sub(f16decX8(ub.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan)) + acc0 = dE.MulAdd(dE, acc0) + acc1 = dO.MulAdd(dO, acc1) + } + sum := sumF32x8(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + d := f16fast(a[i]) - f16fast(b[i]) + sum += d * d + } + return float64(sum), nil +} + +func innerProductF16AVX2(a, b []types.Float16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := f16AsU32(a), f16AsU32(b) + m7fff, m8000, mLo, mInf, magic, infNan := f16DecodeConstsX8() + acc0, acc1 := archsimd.Float32x8{}, archsimd.Float32x8{} + np, j := len(au), 0 + for ; j <= np-8; j += 8 { + ua := archsimd.LoadUint32x8Slice(au[j : j+8]) + ub := archsimd.LoadUint32x8Slice(bu[j : j+8]) + acc0 = f16decX8(ua.And(mLo), m7fff, m8000, mInf, magic, infNan).MulAdd(f16decX8(ub.And(mLo), m7fff, m8000, mInf, magic, infNan), acc0) + acc1 = f16decX8(ua.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan).MulAdd(f16decX8(ub.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan), acc1) + } + sum := sumF32x8(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + sum += f16fast(a[i]) * f16fast(b[i]) + } + return float64(-sum), nil +} + +func l1DistanceF16AVX2(a, b []types.Float16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := f16AsU32(a), f16AsU32(b) + m7fff, m8000, mLo, mInf, magic, infNan := f16DecodeConstsX8() + absMask := archsimd.BroadcastUint32x8(0x7fffffff) + acc0, acc1 := archsimd.Float32x8{}, archsimd.Float32x8{} + np, j := len(au), 0 + for ; j <= np-8; j += 8 { + ua := archsimd.LoadUint32x8Slice(au[j : j+8]) + ub := archsimd.LoadUint32x8Slice(bu[j : j+8]) + dE := f16decX8(ua.And(mLo), m7fff, m8000, mInf, magic, infNan).Sub(f16decX8(ub.And(mLo), m7fff, m8000, mInf, magic, infNan)) + dO := f16decX8(ua.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan).Sub(f16decX8(ub.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan)) + acc0 = acc0.Add(dE.AsUint32x8().And(absMask).AsFloat32x8()) + acc1 = acc1.Add(dO.AsUint32x8().And(absMask).AsFloat32x8()) + } + sum := sumF32x8(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + d := f16fast(a[i]) - f16fast(b[i]) + if d < 0 { + d = -d + } + sum += d + } + return float64(sum), nil +} + +func cosineDistanceF16AVX2(a, b []types.Float16) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := f16AsU32(a), f16AsU32(b) + m7fff, m8000, mLo, mInf, magic, infNan := f16DecodeConstsX8() + dot0, dot1 := archsimd.Float32x8{}, archsimd.Float32x8{} + na0, na1 := archsimd.Float32x8{}, archsimd.Float32x8{} + nb0, nb1 := archsimd.Float32x8{}, archsimd.Float32x8{} + np, j := len(au), 0 + for ; j <= np-8; j += 8 { + ua := archsimd.LoadUint32x8Slice(au[j : j+8]) + ub := archsimd.LoadUint32x8Slice(bu[j : j+8]) + aE := f16decX8(ua.And(mLo), m7fff, m8000, mInf, magic, infNan) + aO := f16decX8(ua.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan) + bE := f16decX8(ub.And(mLo), m7fff, m8000, mInf, magic, infNan) + bO := f16decX8(ub.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan) + dot0 = aE.MulAdd(bE, dot0) + dot1 = aO.MulAdd(bO, dot1) + na0 = aE.MulAdd(aE, na0) + na1 = aO.MulAdd(aO, na1) + nb0 = bE.MulAdd(bE, nb0) + nb1 = bO.MulAdd(bO, nb1) + } + dot := sumF32x8(dot0.Add(dot1)) + na2 := sumF32x8(na0.Add(na1)) + nb2 := sumF32x8(nb0.Add(nb1)) + for i := j * 2; i < n; i++ { + ai, bi := f16fast(a[i]), f16fast(b[i]) + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_avx2_amd64_test.go b/pkg/vectorindex/metric/distance_func_narrow_avx2_amd64_test.go new file mode 100644 index 0000000000000..3da29e34971af --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_avx2_amd64_test.go @@ -0,0 +1,207 @@ +//go:build amd64 && go1.26 && goexperiment.simd + +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests + benchmark for the AVX2 (256-bit) narrow fallback tier. The AVX2 +// kernels live in distance_func_narrow_avx2_amd64.go (production); here we prove +// they match the scalar oracle and benchmark scalar / AVX2 / AVX-512 side by side +// in one binary. Only built under `GOEXPERIMENT=simd GOAMD64=v3`. + +package metric + +import ( + "math" + "math/rand" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// TestAVX2NarrowMatchesScalar checks all four metrics of each AVX2 narrow kernel +// against the scalar oracle across dims covering the 8-lane loop + every tail. +func TestAVX2NarrowMatchesScalar(t *testing.T) { + r := rand.New(rand.NewSource(11)) + chk := func(name string, dim int, got, want float64, exact bool) { + t.Helper() + if exact { + require.Equal(t, want, got, "%s dim=%d", name, dim) + return + } + require.InDelta(t, want, got, 1e-4*(1+math.Abs(want)), "%s dim=%d", name, dim) + } + for _, dim := range narrowSIMDDims { + bfa, bfb := randBF16(dim, r), randBF16(dim, r) + for _, k := range []struct { + name string + avx2, scalar func(a, b []types.BF16) (float64, error) + }{ + {"bf16/l2sq", l2sqBF16AVX2, l2sqBF16}, + {"bf16/ip", innerProductBF16AVX2, innerProductBF16}, + {"bf16/l1", l1DistanceBF16AVX2, l1DistanceBF16}, + {"bf16/cosine", cosineDistanceBF16AVX2, cosineDistanceBF16}, + } { + g, _ := k.avx2(bfa, bfb) + w, _ := k.scalar(bfa, bfb) + chk(k.name, dim, g, w, false) + } + + fa, fb := randF16(dim, r), randF16(dim, r) + for _, k := range []struct { + name string + avx2, scalar func(a, b []types.Float16) (float64, error) + }{ + {"f16/l2sq", l2sqF16AVX2, l2sqF16}, + {"f16/ip", innerProductF16AVX2, innerProductF16}, + {"f16/l1", l1DistanceF16AVX2, l1DistanceF16}, + {"f16/cosine", cosineDistanceF16AVX2, cosineDistanceF16}, + } { + g, _ := k.avx2(fa, fb) + w, _ := k.scalar(fa, fb) + chk(k.name, dim, g, w, false) + } + + i8a, i8b := randI8(dim, r), randI8(dim, r) + for _, k := range []struct { + name string + avx2, scalar func(a, b []int8) (float64, error) + exact bool + }{ + {"int8/l2sq", l2sqInt8AVX2, l2sqInt8, true}, + {"int8/ip", innerProductInt8AVX2, innerProductInt8, true}, + {"int8/l1", l1DistanceInt8AVX2, l1DistanceInt8, true}, + {"int8/cosine", cosineDistanceInt8AVX2, cosineDistanceInt8, false}, + } { + g, _ := k.avx2(i8a, i8b) + w, _ := k.scalar(i8a, i8b) + chk(k.name, dim, g, w, k.exact) + } + + // uint8: the SIMD dispatcher picks AVX-512 over AVX2 on a capable CPU, so + // call the AVX2 uint8 kernels directly to cover them regardless of host. + u8a, u8b := randU8(dim, r), randU8(dim, r) + for _, k := range []struct { + name string + avx2, scalar func(a, b []uint8) (float64, error) + exact bool + }{ + {"uint8/l2sq", l2sqUint8AVX2, l2sqUint8, true}, + {"uint8/ip", innerProductUint8AVX2, innerProductUint8, true}, + {"uint8/l1", l1DistanceUint8AVX2, l1DistanceUint8, true}, + {"uint8/cosine", cosineDistanceUint8AVX2, cosineDistanceUint8, false}, + } { + g, _ := k.avx2(u8a, u8b) + w, _ := k.scalar(u8a, u8b) + chk(k.name, dim, g, w, k.exact) + } + } +} + +// TestAVX2NarrowEdgeCases covers the early-return branches the match-scalar test +// never hits: dimension mismatch, cosine empty-input, and cosine zero-norm. All +// three return before (or skip) the 8-lane loop, so a tiny/zero dim is enough and +// no AVX2 execution is required for the guard branches. +func TestAVX2NarrowEdgeCases(t *testing.T) { + // Dimension mismatch -> error, returned before any SIMD work. + t.Run("mismatch", func(t *testing.T) { + bfA, bfB := make([]types.BF16, 8), make([]types.BF16, 7) + fA, fB := make([]types.Float16, 8), make([]types.Float16, 7) + iA, iB := make([]int8, 8), make([]int8, 7) + uA, uB := make([]uint8, 8), make([]uint8, 7) + for name, fn := range map[string]func() (float64, error){ + "bf16/l2sq": func() (float64, error) { return l2sqBF16AVX2(bfA, bfB) }, + "bf16/ip": func() (float64, error) { return innerProductBF16AVX2(bfA, bfB) }, + "bf16/l1": func() (float64, error) { return l1DistanceBF16AVX2(bfA, bfB) }, + "bf16/cosine": func() (float64, error) { return cosineDistanceBF16AVX2(bfA, bfB) }, + "f16/l2sq": func() (float64, error) { return l2sqF16AVX2(fA, fB) }, + "f16/ip": func() (float64, error) { return innerProductF16AVX2(fA, fB) }, + "f16/l1": func() (float64, error) { return l1DistanceF16AVX2(fA, fB) }, + "f16/cosine": func() (float64, error) { return cosineDistanceF16AVX2(fA, fB) }, + "int8/l2sq": func() (float64, error) { return l2sqInt8AVX2(iA, iB) }, + "int8/ip": func() (float64, error) { return innerProductInt8AVX2(iA, iB) }, + "int8/l1": func() (float64, error) { return l1DistanceInt8AVX2(iA, iB) }, + "int8/cosine": func() (float64, error) { return cosineDistanceInt8AVX2(iA, iB) }, + "uint8/l2sq": func() (float64, error) { return l2sqUint8AVX2(uA, uB) }, + "uint8/ip": func() (float64, error) { return innerProductUint8AVX2(uA, uB) }, + "uint8/l1": func() (float64, error) { return l1DistanceUint8AVX2(uA, uB) }, + "uint8/cosine": func() (float64, error) { return cosineDistanceUint8AVX2(uA, uB) }, + } { + _, err := fn() + require.Error(t, err, name) + } + }) + + // cosine on empty input returns (0, nil) before the length check. + t.Run("empty", func(t *testing.T) { + for name, fn := range map[string]func() (float64, error){ + "bf16": func() (float64, error) { return cosineDistanceBF16AVX2(nil, nil) }, + "f16": func() (float64, error) { return cosineDistanceF16AVX2(nil, nil) }, + "int8": func() (float64, error) { return cosineDistanceInt8AVX2(nil, nil) }, + "uint8": func() (float64, error) { return cosineDistanceUint8AVX2(nil, nil) }, + } { + d, err := fn() + require.NoError(t, err, name) + require.Equal(t, 0.0, d, name) + } + }) + + // A zero-norm vector hits the denom==0 guard -> distance 1.0. dim=4 keeps the + // 8-lane loop from running, so the tail alone drives na2/nb2 to zero. + t.Run("zero_norm", func(t *testing.T) { + const dim = 4 + for name, fn := range map[string]func() (float64, error){ + "bf16": func() (float64, error) { + return cosineDistanceBF16AVX2(make([]types.BF16, dim), make([]types.BF16, dim)) + }, + "f16": func() (float64, error) { + return cosineDistanceF16AVX2(make([]types.Float16, dim), make([]types.Float16, dim)) + }, + "int8": func() (float64, error) { return cosineDistanceInt8AVX2(make([]int8, dim), make([]int8, dim)) }, + "uint8": func() (float64, error) { return cosineDistanceUint8AVX2(make([]uint8, dim), make([]uint8, dim)) }, + } { + d, err := fn() + require.NoError(t, err, name) + require.Equal(t, 1.0, d, name) + } + }) +} + +// Benchmark_Narrow_AVX2vsAVX512 compares scalar / AVX2 (x8) / AVX-512 (x16) for +// the narrow L2sq kernels in one binary. +// +// GOEXPERIMENT=simd GOAMD64=v3 go test ./pkg/vectorindex/metric/ \ +// -run x -bench Benchmark_Narrow_AVX2vsAVX512 +func Benchmark_Narrow_AVX2vsAVX512(b *testing.B) { + const dim = 1024 + r := rand.New(rand.NewSource(1)) + bfa, bfb := randBF16(dim, r), randBF16(dim, r) + fa, fb := randF16(dim, r), randF16(dim, r) + i8a, i8b := randI8(dim, r), randI8(dim, r) + + run := func(b *testing.B, fn func() (float64, error)) { + for i := 0; i < b.N; i++ { + _, _ = fn() + } + } + b.Run("bf16/scalar", func(b *testing.B) { run(b, func() (float64, error) { return l2sqBF16(bfa, bfb) }) }) + b.Run("bf16/avx2", func(b *testing.B) { run(b, func() (float64, error) { return l2sqBF16AVX2(bfa, bfb) }) }) + b.Run("bf16/avx512", func(b *testing.B) { run(b, func() (float64, error) { return l2sqBF16SIMD(bfa, bfb) }) }) + b.Run("f16/scalar", func(b *testing.B) { run(b, func() (float64, error) { return l2sqF16(fa, fb) }) }) + b.Run("f16/avx2", func(b *testing.B) { run(b, func() (float64, error) { return l2sqF16AVX2(fa, fb) }) }) + b.Run("f16/avx512", func(b *testing.B) { run(b, func() (float64, error) { return l2sqF16SIMD(fa, fb) }) }) + b.Run("int8/scalar", func(b *testing.B) { run(b, func() (float64, error) { return l2sqInt8(i8a, i8b) }) }) + b.Run("int8/avx2", func(b *testing.B) { run(b, func() (float64, error) { return l2sqInt8AVX2(i8a, i8b) }) }) + b.Run("int8/avx512", func(b *testing.B) { run(b, func() (float64, error) { return l2sqInt8SIMD(i8a, i8b) }) }) +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_bench_test.go b/pkg/vectorindex/metric/distance_func_narrow_bench_test.go new file mode 100644 index 0000000000000..de5e312b3803f --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_bench_test.go @@ -0,0 +1,139 @@ +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metric + +import ( + "math/rand" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +// Compares the narrow element types (bf16/f16/int8) against f32/f64 on the +// realistic workload: distance computed from the raw stored bytes (so the cost +// includes each type's decode — free reinterpret for f32/f64/int8, an +// upcast-to-float32 for bf16/f16). At dim=1024 the per-vector load is: +// f64 8KB · f32 4KB · bf16/f16 2KB · int8 1KB +// so int8 should win on bandwidth + integer ops; bf16/f16 trade half the +// bandwidth for the fp32 upcast. +// +// Run: go test ./pkg/vectorindex/metric/ -run x -bench Benchmark_NarrowVsFloat -benchmem + +const ( + narrowBenchDim = 1024 + narrowBenchPool = 256 +) + +func benchF32Pool(n, dim int) [][]float32 { + out := make([][]float32, n) + for i := range out { + v := make([]float32, dim) + for j := range v { + v[j] = float32(rand.Float64()*16 - 8) // [-8, 8) + } + out[i] = v + } + return out +} + +func benchInt8Pool(n, dim int) [][]int8 { + out := make([][]int8, n) + for i := range out { + v := make([]int8, dim) + for j := range v { + v[j] = int8(rand.Intn(255) - 127) // [-127, 127] + } + out[i] = v + } + return out +} + +func toBytesPool[T types.ArrayElement](vecs [][]T) [][]byte { + out := make([][]byte, len(vecs)) + for i, v := range vecs { + out[i] = append([]byte(nil), types.ArrayToBytes(v)...) + } + return out +} + +func Benchmark_NarrowVsFloat(b *testing.B) { + dim, n := narrowBenchDim, narrowBenchPool + + f32 := benchF32Pool(n, dim) + f64 := make([][]float64, n) + bf16 := make([][]types.BF16, n) + f16 := make([][]types.Float16, n) + for i, v := range f32 { + d := make([]float64, dim) + for j, x := range v { + d[j] = float64(x) + } + f64[i] = d + bf16[i] = types.Float32ToBF16Slice(v) + f16[i] = types.Float32ToFloat16Slice(v) + } + i8 := benchInt8Pool(n, dim) + + f64b := toBytesPool(f64) + f32b := toBytesPool(f32) + bf16b := toBytesPool(bf16) + f16b := toBytesPool(f16) + i8b := toBytesPool(i8) + + metrics := []struct { + name string + mt MetricType + }{ + {"L2sq", Metric_L2sqDistance}, + {"InnerProduct", Metric_InnerProduct}, + {"Cosine", Metric_CosineDistance}, + } + + for _, m := range metrics { + b.Run(m.name, func(b *testing.B) { + b.Run("f64", func(b *testing.B) { benchFloatFromBytes[float64](b, m.mt, f64b) }) + b.Run("f32", func(b *testing.B) { benchFloatFromBytes[float32](b, m.mt, f32b) }) + b.Run("bf16", func(b *testing.B) { benchNarrowFromBytes[types.BF16](b, m.mt, bf16b) }) + b.Run("f16", func(b *testing.B) { benchNarrowFromBytes[types.Float16](b, m.mt, f16b) }) + b.Run("int8", func(b *testing.B) { benchNarrowFromBytes[int8](b, m.mt, i8b) }) + }) + } +} + +func benchFloatFromBytes[T float32 | float64](b *testing.B, m MetricType, pool [][]byte) { + fn, err := ResolveDistanceFn[T, T](m) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + a := types.BytesToArray[T](pool[i%len(pool)]) + c := types.BytesToArray[T](pool[(i+1)%len(pool)]) + _, _ = fn(a, c) + } +} + +func benchNarrowFromBytes[T types.ArrayElement](b *testing.B, m MetricType, pool [][]byte) { + fn, err := ResolveDistanceFn[T, float32](m) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + a := types.BytesToArray[T](pool[i%len(pool)]) + c := types.BytesToArray[T](pool[(i+1)%len(pool)]) + _, _ = fn(a, c) + } +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_f16_amd64.go b/pkg/vectorindex/metric/distance_func_narrow_f16_amd64.go new file mode 100644 index 0000000000000..9b436c40b2745 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_f16_amd64.go @@ -0,0 +1,209 @@ +//go:build amd64 && go1.26 && goexperiment.simd + +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// AVX-512 SIMD distance kernels for vecf16 (types.Float16). +// +// IEEE half->float32 is NOT a plain shift (exponent rebias + subnormals), and Go +// archsimd exposes no f16 type (this CPU also lacks avx512_fp16, and F16C is not +// surfaced). So we vectorize the same magic-multiply f16fast() the scalar path +// uses (Fabian Giesen / rygorous): rescale the exponent via a float multiply and +// fix up Inf/NaN with a masked Merge. Inputs load as Uint32x16 (32 f16/load), +// even/odd 16-bit halves split out, decoded, then fed to the existing AVX-512 +// float32 reduction (sumF32x16). Matches f16fast bit-for-bit so it agrees with +// the scalar oracle. +// +// PERF: the six decode constants are passed to f16dec as individual vector args, +// NOT bundled in a struct. A by-value struct of vectors gets spilled to the stack +// and reloaded on every field access (77 MOVUPS in the inner loop -> ~7x slower); +// individual args stay in zmm registers. With that, f16 SIMD is ~8x over scalar +// (was ~1.3x with the struct) — the difference between "not worth it" and "worth +// it", so the swap below is ON. + +package metric + +import ( + "math" + "unsafe" + + "simd/archsimd" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +func init() { + switch { + case hasAVX512: + f16L2sqFn = l2sqF16SIMD + f16IPFn = innerProductF16SIMD + f16CosineFn = cosineDistanceF16SIMD + f16L1Fn = l1DistanceF16SIMD + case hasAVX2: + f16L2sqFn = l2sqF16AVX2 + f16IPFn = innerProductF16AVX2 + f16CosineFn = cosineDistanceF16AVX2 + f16L1Fn = l1DistanceF16AVX2 + } +} + +func f16AsU32(s []types.Float16) []uint32 { + if len(s) < 2 { + return nil + } + return unsafe.Slice((*uint32)(unsafe.Pointer(unsafe.SliceData(s))), len(s)/2) +} + +// f16dec decodes 16 half-floats (each in the low 16 bits of a uint32 lane) to +// float32 — the SIMD form of f16fast(), Inf/NaN fixup included. Constants are +// individual args (see file header: a struct spills; args stay in registers). +func f16dec(h, m7fff, m8000, mInf archsimd.Uint32x16, magic, infNan archsimd.Float32x16) archsimd.Float32x16 { + o := h.And(m7fff).ShiftAllLeft(13) + of := o.AsFloat32x16().Mul(magic) + ou := of.AsUint32x16() + ou = ou.Or(mInf).Merge(ou, of.GreaterEqual(infNan)) // ou|=inf where >=infNan + return ou.Or(h.And(m8000).ShiftAllLeft(16)).AsFloat32x16() +} + +// f16Decode constants, built once per kernel as locals. +func f16DecodeConsts() (m7fff, m8000, mLo, mInf archsimd.Uint32x16, magic, infNan archsimd.Float32x16) { + m7fff = archsimd.BroadcastUint32x16(0x7fff) + m8000 = archsimd.BroadcastUint32x16(0x8000) + mLo = archsimd.BroadcastUint32x16(0xffff) + mInf = archsimd.BroadcastUint32x16(255 << 23) + magic = archsimd.BroadcastFloat32x16(f16Magic) + infNan = archsimd.BroadcastFloat32x16(f16WasInfNan) + return +} + +func l2sqF16SIMD(a, b []types.Float16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := f16AsU32(a), f16AsU32(b) + m7fff, m8000, mLo, mInf, magic, infNan := f16DecodeConsts() + acc0, acc1 := archsimd.Float32x16{}, archsimd.Float32x16{} + np, j := len(au), 0 + for ; j <= np-16; j += 16 { + ua := archsimd.LoadUint32x16Slice(au[j : j+16]) + ub := archsimd.LoadUint32x16Slice(bu[j : j+16]) + dE := f16dec(ua.And(mLo), m7fff, m8000, mInf, magic, infNan).Sub(f16dec(ub.And(mLo), m7fff, m8000, mInf, magic, infNan)) + dO := f16dec(ua.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan).Sub(f16dec(ub.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan)) + acc0 = dE.MulAdd(dE, acc0) + acc1 = dO.MulAdd(dO, acc1) + } + sum := sumF32x16(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + d := f16fast(a[i]) - f16fast(b[i]) + sum += d * d + } + return float64(sum), nil +} + +func innerProductF16SIMD(a, b []types.Float16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := f16AsU32(a), f16AsU32(b) + m7fff, m8000, mLo, mInf, magic, infNan := f16DecodeConsts() + acc0, acc1 := archsimd.Float32x16{}, archsimd.Float32x16{} + np, j := len(au), 0 + for ; j <= np-16; j += 16 { + ua := archsimd.LoadUint32x16Slice(au[j : j+16]) + ub := archsimd.LoadUint32x16Slice(bu[j : j+16]) + acc0 = f16dec(ua.And(mLo), m7fff, m8000, mInf, magic, infNan).MulAdd(f16dec(ub.And(mLo), m7fff, m8000, mInf, magic, infNan), acc0) + acc1 = f16dec(ua.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan).MulAdd(f16dec(ub.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan), acc1) + } + sum := sumF32x16(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + sum += f16fast(a[i]) * f16fast(b[i]) + } + return float64(-sum), nil +} + +func l1DistanceF16SIMD(a, b []types.Float16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := f16AsU32(a), f16AsU32(b) + m7fff, m8000, mLo, mInf, magic, infNan := f16DecodeConsts() + absMask := archsimd.BroadcastUint32x16(0x7fffffff) + acc0, acc1 := archsimd.Float32x16{}, archsimd.Float32x16{} + np, j := len(au), 0 + for ; j <= np-16; j += 16 { + ua := archsimd.LoadUint32x16Slice(au[j : j+16]) + ub := archsimd.LoadUint32x16Slice(bu[j : j+16]) + dE := f16dec(ua.And(mLo), m7fff, m8000, mInf, magic, infNan).Sub(f16dec(ub.And(mLo), m7fff, m8000, mInf, magic, infNan)) + dO := f16dec(ua.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan).Sub(f16dec(ub.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan)) + acc0 = acc0.Add(dE.AsUint32x16().And(absMask).AsFloat32x16()) + acc1 = acc1.Add(dO.AsUint32x16().And(absMask).AsFloat32x16()) + } + sum := sumF32x16(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + d := f16fast(a[i]) - f16fast(b[i]) + if d < 0 { + d = -d + } + sum += d + } + return float64(sum), nil +} + +func cosineDistanceF16SIMD(a, b []types.Float16) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := f16AsU32(a), f16AsU32(b) + m7fff, m8000, mLo, mInf, magic, infNan := f16DecodeConsts() + dot0, dot1 := archsimd.Float32x16{}, archsimd.Float32x16{} + na0, na1 := archsimd.Float32x16{}, archsimd.Float32x16{} + nb0, nb1 := archsimd.Float32x16{}, archsimd.Float32x16{} + np, j := len(au), 0 + for ; j <= np-16; j += 16 { + ua := archsimd.LoadUint32x16Slice(au[j : j+16]) + ub := archsimd.LoadUint32x16Slice(bu[j : j+16]) + aE := f16dec(ua.And(mLo), m7fff, m8000, mInf, magic, infNan) + aO := f16dec(ua.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan) + bE := f16dec(ub.And(mLo), m7fff, m8000, mInf, magic, infNan) + bO := f16dec(ub.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan) + dot0 = aE.MulAdd(bE, dot0) + dot1 = aO.MulAdd(bO, dot1) + na0 = aE.MulAdd(aE, na0) + na1 = aO.MulAdd(aO, na1) + nb0 = bE.MulAdd(bE, nb0) + nb1 = bO.MulAdd(bO, nb1) + } + dot := sumF32x16(dot0.Add(dot1)) + na2 := sumF32x16(na0.Add(na1)) + nb2 := sumF32x16(nb0.Add(nb1)) + for i := j * 2; i < n; i++ { + ai, bi := f16fast(a[i]), f16fast(b[i]) + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_int8_amd64.go b/pkg/vectorindex/metric/distance_func_narrow_int8_amd64.go new file mode 100644 index 0000000000000..1e8dc1d6c9f6f --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_int8_amd64.go @@ -0,0 +1,182 @@ +//go:build amd64 && go1.26 && goexperiment.simd + +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// AVX-512 SIMD distance kernels for vecint8 ([]int8), INTEGER-EXACT (bit-for-bit +// identical to the int64-accumulating pure-Go oracle). +// +// archsimd has no int8->int32 widening op, so we load the raw bytes as Int32x16 +// (64 int8 per load) and sign-extend the four byte lanes with shifts: +// byteJ = (u << (24-8J)) >>arith 24 (Int32x16.ShiftAllRight is arithmetic). All +// arithmetic then stays in int32 lanes — exact, since for the max dimension +// (65535) a lane accumulates < 1024 terms each <= 255^2, far under 2^31 — and the +// final horizontal reduction is in int64. No float, so results equal the oracle +// exactly (the int8 equivalence test asserts ==, not approx). + +package metric + +import ( + "math" + "unsafe" + + "simd/archsimd" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" +) + +func init() { + switch { + case hasAVX512: + int8L2sqFn = l2sqInt8SIMD + int8IPFn = innerProductInt8SIMD + int8CosineFn = cosineDistanceInt8SIMD + int8L1Fn = l1DistanceInt8SIMD + case hasAVX2: + int8L2sqFn = l2sqInt8AVX2 + int8IPFn = innerProductInt8AVX2 + int8CosineFn = cosineDistanceInt8AVX2 + int8L1Fn = l1DistanceInt8AVX2 + } +} + +// int8AsI32 reinterprets a []int8 as []int32 viewing its first len/4 dwords. +func int8AsI32(s []int8) []int32 { + if len(s) < 4 { + return nil + } + return unsafe.Slice((*int32)(unsafe.Pointer(unsafe.SliceData(s))), len(s)/4) +} + +// sumI32x16 horizontally adds the 16 int32 lanes into an int64 (lane values are +// bounded well under 2^31, but the 16-lane total can exceed it). +func sumI32x16(v archsimd.Int32x16) int64 { + var a [16]int32 + v.Store(&a) + var s int64 + for _, x := range a { + s += int64(x) + } + return s +} + +// unpackI8 sign-extends the four byte lanes of an Int32x16 (64 packed int8) into +// four Int32x16 vectors. Lane k of vJ holds int8[4k+J]. +func unpackI8(u archsimd.Int32x16) (v0, v1, v2, v3 archsimd.Int32x16) { + v0 = u.ShiftAllLeft(24).ShiftAllRight(24) + v1 = u.ShiftAllLeft(16).ShiftAllRight(24) + v2 = u.ShiftAllLeft(8).ShiftAllRight(24) + v3 = u.ShiftAllRight(24) + return +} + +func l2sqInt8SIMD(a, b []int8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + ai, bi := int8AsI32(a), int8AsI32(b) + acc := archsimd.Int32x16{} + nq, j := len(ai), 0 + for ; j <= nq-16; j += 16 { + a0, a1, a2, a3 := unpackI8(archsimd.LoadInt32x16Slice(ai[j : j+16])) + b0, b1, b2, b3 := unpackI8(archsimd.LoadInt32x16Slice(bi[j : j+16])) + d0, d1, d2, d3 := a0.Sub(b0), a1.Sub(b1), a2.Sub(b2), a3.Sub(b3) + acc = acc.Add(d0.Mul(d0).Add(d1.Mul(d1)).Add(d2.Mul(d2).Add(d3.Mul(d3)))) + } + sum := sumI32x16(acc) + for i := j * 4; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + sum += int64(d * d) + } + return float64(sum), nil +} + +func innerProductInt8SIMD(a, b []int8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + ai, bi := int8AsI32(a), int8AsI32(b) + acc := archsimd.Int32x16{} + nq, j := len(ai), 0 + for ; j <= nq-16; j += 16 { + a0, a1, a2, a3 := unpackI8(archsimd.LoadInt32x16Slice(ai[j : j+16])) + b0, b1, b2, b3 := unpackI8(archsimd.LoadInt32x16Slice(bi[j : j+16])) + acc = acc.Add(a0.Mul(b0).Add(a1.Mul(b1)).Add(a2.Mul(b2).Add(a3.Mul(b3)))) + } + sum := sumI32x16(acc) + for i := j * 4; i < n; i++ { + sum += int64(int32(a[i]) * int32(b[i])) + } + return float64(-sum), nil +} + +func l1DistanceInt8SIMD(a, b []int8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + ai, bi := int8AsI32(a), int8AsI32(b) + zero := archsimd.Int32x16{} + acc := archsimd.Int32x16{} + abs := func(d archsimd.Int32x16) archsimd.Int32x16 { return d.Max(zero.Sub(d)) } + nq, j := len(ai), 0 + for ; j <= nq-16; j += 16 { + a0, a1, a2, a3 := unpackI8(archsimd.LoadInt32x16Slice(ai[j : j+16])) + b0, b1, b2, b3 := unpackI8(archsimd.LoadInt32x16Slice(bi[j : j+16])) + acc = acc.Add(abs(a0.Sub(b0)).Add(abs(a1.Sub(b1))).Add(abs(a2.Sub(b2)).Add(abs(a3.Sub(b3))))) + } + sum := sumI32x16(acc) + for i := j * 4; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + if d < 0 { + d = -d + } + sum += int64(d) + } + return float64(sum), nil +} + +func cosineDistanceInt8SIMD(a, b []int8) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + ai, bi := int8AsI32(a), int8AsI32(b) + dotA, naA, nbA := archsimd.Int32x16{}, archsimd.Int32x16{}, archsimd.Int32x16{} + nq, j := len(ai), 0 + for ; j <= nq-16; j += 16 { + a0, a1, a2, a3 := unpackI8(archsimd.LoadInt32x16Slice(ai[j : j+16])) + b0, b1, b2, b3 := unpackI8(archsimd.LoadInt32x16Slice(bi[j : j+16])) + dotA = dotA.Add(a0.Mul(b0).Add(a1.Mul(b1)).Add(a2.Mul(b2).Add(a3.Mul(b3)))) + naA = naA.Add(a0.Mul(a0).Add(a1.Mul(a1)).Add(a2.Mul(a2).Add(a3.Mul(a3)))) + nbA = nbA.Add(b0.Mul(b0).Add(b1.Mul(b1)).Add(b2.Mul(b2).Add(b3.Mul(b3)))) + } + dot, na2, nb2 := sumI32x16(dotA), sumI32x16(naA), sumI32x16(nbA) + for i := j * 4; i < n; i++ { + ai8, bi8 := int64(a[i]), int64(b[i]) + dot += ai8 * bi8 + na2 += ai8 * ai8 + nb2 += bi8 * bi8 + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_test.go b/pkg/vectorindex/metric/distance_func_narrow_test.go new file mode 100644 index 0000000000000..cf88b39d15089 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_test.go @@ -0,0 +1,403 @@ +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metric + +import ( + "math" + "math/rand" + "testing" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// resolveNarrowBytes mirrors the (removed) byte-keyed narrow resolver for these +// tests: an oid-keyed, raw-bytes-in, float64-out distance fn built on the merged +// ResolveDistanceFn. R=float64 keeps int8's int64 sum exact (the exact-match +// oracle), and errors for non-narrow oids / invalid metrics exactly as before. +func resolveNarrowBytes(oid types.T, m MetricType) (func(a, b []byte) (float64, error), error) { + switch oid { + case types.T_array_bf16: + fn, err := ResolveDistanceFn[types.BF16, float64](m) + if err != nil { + return nil, err + } + return func(a, b []byte) (float64, error) { + return fn(types.BytesToArray[types.BF16](a), types.BytesToArray[types.BF16](b)) + }, nil + case types.T_array_float16: + fn, err := ResolveDistanceFn[types.Float16, float64](m) + if err != nil { + return nil, err + } + return func(a, b []byte) (float64, error) { + return fn(types.BytesToArray[types.Float16](a), types.BytesToArray[types.Float16](b)) + }, nil + case types.T_array_int8: + fn, err := ResolveDistanceFn[int8, float64](m) + if err != nil { + return nil, err + } + return func(a, b []byte) (float64, error) { + return fn(types.BytesToArray[int8](a), types.BytesToArray[int8](b)) + }, nil + case types.T_array_uint8: + fn, err := ResolveDistanceFn[uint8, float64](m) + if err != nil { + return nil, err + } + return func(a, b []byte) (float64, error) { + return fn(types.BytesToArray[uint8](a), types.BytesToArray[uint8](b)) + }, nil + default: + return nil, moerr.NewInternalErrorNoCtx("resolveNarrowBytes: not a narrow vector type") + } +} + +// reference distance over float64, mirroring ResolveDistanceFn semantics. +func refDist(metric MetricType, a, b []float64) float64 { + switch metric { + case Metric_L2Distance, Metric_L2sqDistance: + var s float64 + for i := range a { + d := a[i] - b[i] + s += d * d + } + return s + case Metric_InnerProduct: + var s float64 + for i := range a { + s += a[i] * b[i] + } + return -s + case Metric_L1Distance: + var s float64 + for i := range a { + s += math.Abs(a[i] - b[i]) + } + return s + case Metric_CosineDistance: + var dot, na2, nb2 float64 + for i := range a { + dot += a[i] * b[i] + na2 += a[i] * a[i] + nb2 += b[i] * b[i] + } + den := math.Sqrt(na2) * math.Sqrt(nb2) + if den == 0 { + return 1.0 + } + sim := dot / den + if sim > 1 { + sim = 1 + } else if sim < -1 { + sim = -1 + } + return 1.0 - sim + } + return 0 +} + +var narrowMetrics = []MetricType{Metric_L2Distance, Metric_L2sqDistance, Metric_InnerProduct, Metric_CosineDistance, Metric_L1Distance} + +func TestNarrowInt8KernelsExact(t *testing.T) { + // int8 values -> exact integer arithmetic, must match float64 reference exactly. + a := []int8{1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11} + b := []int8{-1, 2, -3, 4, 0, 6, -7, 8, -9, 1, 2} + af := make([]float64, len(a)) + bf := make([]float64, len(b)) + for i := range a { + af[i] = float64(a[i]) + bf[i] = float64(b[i]) + } + ab := types.ArrayToBytes(a) + bb := types.ArrayToBytes(b) + for _, m := range narrowMetrics { + fn, err := resolveNarrowBytes(types.T_array_int8, m) + if err != nil { + t.Fatalf("resolve int8 m=%d: %v", m, err) + } + got, err := fn(ab, bb) + if err != nil { + t.Fatalf("int8 dist m=%d: %v", m, err) + } + want := refDist(m, af, bf) + if math.Abs(got-want) > 1e-9 { + t.Errorf("int8 m=%d: got %v want %v", m, got, want) + } + } +} + +func TestNarrowUint8KernelsExact(t *testing.T) { + // uint8 values (0..255) -> exact integer arithmetic, must match float64 reference. + a := []uint8{1, 2, 3, 4, 5, 250, 7, 8, 9, 10, 255} + b := []uint8{255, 2, 0, 4, 100, 6, 7, 200, 9, 1, 2} + af := make([]float64, len(a)) + bf := make([]float64, len(b)) + for i := range a { + af[i] = float64(a[i]) + bf[i] = float64(b[i]) + } + ab := types.ArrayToBytes(a) + bb := types.ArrayToBytes(b) + for _, m := range narrowMetrics { + fn, err := resolveNarrowBytes(types.T_array_uint8, m) + if err != nil { + t.Fatalf("resolve uint8 m=%d: %v", m, err) + } + got, err := fn(ab, bb) + if err != nil { + t.Fatalf("uint8 dist m=%d: %v", m, err) + } + want := refDist(m, af, bf) + if math.Abs(got-want) > 1e-9 { + t.Errorf("uint8 m=%d: got %v want %v", m, got, want) + } + } +} + +func TestNarrowBF16F16Kernels(t *testing.T) { + src1 := []float32{1, 2, 3, 0.5, -4, 6, 7.5, -8, 9, 10, 11} + src2 := []float32{-1, 2, 0.25, 4, 5, 6, -7, 8, -9, 1, 2} + // bf16 + bf1 := types.Float32ToBF16Slice(src1) + bf2 := types.Float32ToBF16Slice(src2) + af := types.BF16ToFloat32Slice(bf1) + bf := types.BF16ToFloat32Slice(bf2) + af64 := f32to64(af) + bf64 := f32to64(bf) + for _, m := range narrowMetrics { + fn, _ := resolveNarrowBytes(types.T_array_bf16, m) + got, err := fn(types.ArrayToBytes(bf1), types.ArrayToBytes(bf2)) + if err != nil { + t.Fatalf("bf16 m=%d: %v", m, err) + } + want := refDist(m, af64, bf64) + if math.Abs(got-want) > 1e-4 { + t.Errorf("bf16 m=%d: got %v want %v", m, got, want) + } + } + // f16 + h1 := types.Float32ToFloat16Slice(src1) + h2 := types.Float32ToFloat16Slice(src2) + haf := f32to64(types.Float16ToFloat32Slice(h1)) + hbf := f32to64(types.Float16ToFloat32Slice(h2)) + for _, m := range narrowMetrics { + fn, _ := resolveNarrowBytes(types.T_array_float16, m) + got, err := fn(types.ArrayToBytes(h1), types.ArrayToBytes(h2)) + if err != nil { + t.Fatalf("f16 m=%d: %v", m, err) + } + want := refDist(m, haf, hbf) + if math.Abs(got-want) > 1e-4 { + t.Errorf("f16 m=%d: got %v want %v", m, got, want) + } + } +} + +func TestNarrowResolveErrors(t *testing.T) { + if _, err := resolveNarrowBytes(types.T_array_float32, Metric_L2Distance); err == nil { + t.Errorf("expected error for non-narrow oid") + } + if _, err := resolveNarrowBytes(types.T_array_int8, MetricType(999)); err == nil { + t.Errorf("expected error for invalid metric") + } +} + +func f32to64(s []float32) []float64 { + out := make([]float64, len(s)) + for i, v := range s { + out[i] = float64(v) + } + return out +} + +func TestF16FastExhaustive(t *testing.T) { + for u := 0; u < 65536; u++ { + h := types.Float16(uint16(u)) + want := h.ToFloat32() + got := f16fast(h) + if math.IsNaN(float64(want)) { + if !math.IsNaN(float64(got)) { + t.Fatalf("h=0x%04x: want NaN, got %v", u, got) + } + continue + } + if math.Float32bits(got) != math.Float32bits(want) { + t.Fatalf("h=0x%04x: f16fast=%v (0x%08x) ToFloat32=%v (0x%08x)", + u, got, math.Float32bits(got), want, math.Float32bits(want)) + } + } +} + +func TestNarrowKernelEdgeCases(t *testing.T) { + narrowOids := []types.T{types.T_array_bf16, types.T_array_float16, types.T_array_int8} + + // dimension mismatch -> error on every metric/type. + for _, oid := range narrowOids { + for _, m := range narrowMetrics { + fn, err := resolveNarrowBytes(oid, m) + require.NoError(t, err) + var a, b []byte + switch oid { + case types.T_array_int8: + a = types.ArrayToBytes([]int8{1, 2, 3}) + b = types.ArrayToBytes([]int8{1, 2}) + case types.T_array_bf16: + a = types.ArrayToBytes(types.Float32ToBF16Slice([]float32{1, 2, 3})) + b = types.ArrayToBytes(types.Float32ToBF16Slice([]float32{1, 2})) + default: + a = types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{1, 2, 3})) + b = types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{1, 2})) + } + _, err = fn(a, b) + require.Errorf(t, err, "oid=%d metric=%d dim mismatch", oid, m) + } + } + + // empty vectors: distance 0 for all metrics/types (cosine has an explicit + // empty guard; the rest sum nothing). + for _, oid := range narrowOids { + for _, m := range narrowMetrics { + fn, _ := resolveNarrowBytes(oid, m) + got, err := fn(nil, nil) + require.NoError(t, err) + require.InDeltaf(t, 0.0, got, 1e-9, "oid=%d metric=%d empty", oid, m) + } + } + + // cosine of a zero vector -> 1.0 (denominator 0). + for _, oid := range narrowOids { + fn, _ := resolveNarrowBytes(oid, Metric_CosineDistance) + var z []byte + switch oid { + case types.T_array_int8: + z = types.ArrayToBytes([]int8{0, 0, 0, 0}) + case types.T_array_bf16: + z = types.ArrayToBytes(types.Float32ToBF16Slice([]float32{0, 0, 0, 0})) + default: + z = types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{0, 0, 0, 0})) + } + got, err := fn(z, z) + require.NoError(t, err) + require.InDeltaf(t, 1.0, got, 1e-9, "oid=%d zero cosine", oid) + } + + // int8 extremes over a large dimension: integer accumulation must not overflow. + // L2: dim * (127-(-128))^2 = 1000 * 255^2 = 65025000, exact in int64. + dim := 1000 + amax := make([]int8, dim) + amin := make([]int8, dim) + for i := range amax { + amax[i] = 127 + amin[i] = -128 + } + fn, _ := resolveNarrowBytes(types.T_array_int8, Metric_L2sqDistance) + got, err := fn(types.ArrayToBytes(amax), types.ArrayToBytes(amin)) + require.NoError(t, err) + require.InDelta(t, float64(dim)*255.0*255.0, got, 1e-6) + + // single-element vectors work (loop-remainder path). + for _, oid := range narrowOids { + fn, _ := resolveNarrowBytes(oid, Metric_L2sqDistance) + var a, b []byte + switch oid { + case types.T_array_int8: + a, b = types.ArrayToBytes([]int8{3}), types.ArrayToBytes([]int8{1}) + case types.T_array_bf16: + a, b = types.ArrayToBytes(types.Float32ToBF16Slice([]float32{3})), types.ArrayToBytes(types.Float32ToBF16Slice([]float32{1})) + default: + a, b = types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{3})), types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{1})) + } + got, err := fn(a, b) + require.NoError(t, err) + require.InDeltaf(t, 4.0, got, 1e-3, "oid=%d single elem", oid) // (3-1)^2 + } +} + +// TestCosineDistanceClampNonNegative guards the [-1,1] similarity clamp. For two +// identical vectors the true cosine similarity is exactly 1, but sqrt(n)*sqrt(n) +// rounds a hair below n for many inputs, so dot/denom lands slightly above 1 and +// an unclamped kernel returns a tiny NEGATIVE distance (outside cosine's [0,2] +// domain). This exercises whichever kernel is active in the build (scalar in the +// default build, AVX512/AVX2 SIMD under GOEXPERIMENT=simd) — every one must clamp +// to >= 0. Random non-parallel vectors never reach sim>1, so the existing +// equivalence tests miss this; here we sample many identical vectors (fixed seed, +// several dims) so the unclamped path reliably produces negatives and fails. +func TestCosineDistanceClampNonNegative(t *testing.T) { + r := rand.New(rand.NewSource(42)) + const samplesPerDim = 400 + dims := []int{4, 7, 16, 17, 31, 64} // cover SIMD-block (>=16) and tail paths + + genF32 := func(dim int) []float32 { + v := make([]float32, dim) + for i := range v { + v[i] = float32(r.Float64()*16 - 8) + } + return v + } + + bf16, err := ResolveDistanceFn[types.BF16, float32](Metric_CosineDistance) + require.NoError(t, err) + f16, err := ResolveDistanceFn[types.Float16, float32](Metric_CosineDistance) + require.NoError(t, err) + i8, err := ResolveDistanceFn[int8, float32](Metric_CosineDistance) + require.NoError(t, err) + u8, err := ResolveDistanceFn[uint8, float32](Metric_CosineDistance) + require.NoError(t, err) + + for _, dim := range dims { + for s := 0; s < samplesPerDim; s++ { + f := genF32(dim) + + // f32 / f64 native + d32, err := CosineDistance(f, f) + require.NoError(t, err) + require.GreaterOrEqualf(t, d32, float32(0), "f32 cosine of identical vector must be >= 0 (dim=%d)", dim) + + f64v := make([]float64, dim) + i8v := make([]int8, dim) + u8v := make([]uint8, dim) + for i, x := range f { + f64v[i] = float64(x) + i8v[i] = int8(x * 8) // [-64,64) + u8v[i] = uint8(x*8 + 128) // [0,256) + } + d64, err := CosineDistance(f64v, f64v) + require.NoError(t, err) + require.GreaterOrEqualf(t, d64, float64(0), "f64 cosine of identical vector must be >= 0 (dim=%d)", dim) + + // narrow types + bv := types.Float32ToBF16Slice(f) + db, err := bf16(bv, bv) + require.NoError(t, err) + require.GreaterOrEqualf(t, db, float32(0), "bf16 cosine of identical vector must be >= 0 (dim=%d)", dim) + + hv := types.Float32ToFloat16Slice(f) + dh, err := f16(hv, hv) + require.NoError(t, err) + require.GreaterOrEqualf(t, dh, float32(0), "f16 cosine of identical vector must be >= 0 (dim=%d)", dim) + + di, err := i8(i8v, i8v) + require.NoError(t, err) + require.GreaterOrEqualf(t, di, float32(0), "int8 cosine of identical vector must be >= 0 (dim=%d)", dim) + + du, err := u8(u8v, u8v) + require.NoError(t, err) + require.GreaterOrEqualf(t, du, float32(0), "uint8 cosine of identical vector must be >= 0 (dim=%d)", dim) + } + } +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_uint8.go b/pkg/vectorindex/metric/distance_func_narrow_uint8.go new file mode 100644 index 0000000000000..6d9e24864c7f3 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_uint8.go @@ -0,0 +1,163 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metric + +import ( + "math" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" +) + +// Pure-Go INTEGER (int64-accumulated) distance kernels for vecuint8 ([]uint8), +// mirroring the vecint8 kernels. uint8 values are promoted to int32/int64 before +// arithmetic so the math is identical to int8 (per-element |d|<=255 -> d*d<=65025, +// a*b in [0,65025]); int64 accumulation never overflows at MaxArrayDimension. +// +// The kernel function pointers are swappable so a future +// distance_func_narrow_uint8_amd64.go can drop in SIMD impls via init(), exactly +// as int8 does — there is no SIMD variant yet, so they point at the Go kernels. +var ( + uint8L2sqFn = l2sqUint8 + uint8IPFn = innerProductUint8 + uint8CosineFn = cosineDistanceUint8 + uint8L1Fn = l1DistanceUint8 +) + +func resolveUint8Kernel(metric MetricType) (func(a, b []uint8) (float64, error), error) { + switch metric { + case Metric_L2Distance, Metric_L2sqDistance: + return uint8L2sqFn, nil + case Metric_InnerProduct: + return uint8IPFn, nil + case Metric_CosineDistance: + return uint8CosineFn, nil + case Metric_L1Distance: + return uint8L1Fn, nil + default: + return nil, moerr.NewInternalErrorNoCtx("invalid distance type") + } +} + +func l2sqUint8(a, b []uint8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum int64 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + d0 := int32(aa[0]) - int32(bb[0]) + d1 := int32(aa[1]) - int32(bb[1]) + d2 := int32(aa[2]) - int32(bb[2]) + d3 := int32(aa[3]) - int32(bb[3]) + d4 := int32(aa[4]) - int32(bb[4]) + d5 := int32(aa[5]) - int32(bb[5]) + d6 := int32(aa[6]) - int32(bb[6]) + d7 := int32(aa[7]) - int32(bb[7]) + sum += int64(d0*d0+d1*d1) + int64(d2*d2+d3*d3) + int64(d4*d4+d5*d5) + int64(d6*d6+d7*d7) + } + for ; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + sum += int64(d * d) + } + return float64(sum), nil +} + +func innerProductUint8(a, b []uint8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum int64 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + sum += int64(int32(aa[0])*int32(bb[0])+int32(aa[1])*int32(bb[1])) + + int64(int32(aa[2])*int32(bb[2])+int32(aa[3])*int32(bb[3])) + + int64(int32(aa[4])*int32(bb[4])+int32(aa[5])*int32(bb[5])) + + int64(int32(aa[6])*int32(bb[6])+int32(aa[7])*int32(bb[7])) + } + for ; i < n; i++ { + sum += int64(int32(a[i]) * int32(b[i])) + } + // matches metric.InnerProduct: returns -dot + return float64(-sum), nil +} + +func l1DistanceUint8(a, b []uint8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum int64 + n := len(a) + i := 0 + abs := func(x int32) int32 { + if x < 0 { + return -x + } + return x + } + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + sum += int64(abs(int32(aa[0])-int32(bb[0]))+abs(int32(aa[1])-int32(bb[1]))) + + int64(abs(int32(aa[2])-int32(bb[2]))+abs(int32(aa[3])-int32(bb[3]))) + + int64(abs(int32(aa[4])-int32(bb[4]))+abs(int32(aa[5])-int32(bb[5]))) + + int64(abs(int32(aa[6])-int32(bb[6]))+abs(int32(aa[7])-int32(bb[7]))) + } + for ; i < n; i++ { + sum += int64(abs(int32(a[i]) - int32(b[i]))) + } + return float64(sum), nil +} + +func cosineDistanceUint8(a, b []uint8) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var dot, na2, nb2 int64 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + for k := 0; k < 8; k++ { + ai := int64(aa[k]) + bi := int64(bb[k]) + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + } + for ; i < n; i++ { + ai := int64(a[i]) + bi := int64(b[i]) + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + // matches metric.CosineDistance: denominator 0 -> distance 1.0 + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_uint8_amd64.go b/pkg/vectorindex/metric/distance_func_narrow_uint8_amd64.go new file mode 100644 index 0000000000000..4f19d44ec00c8 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_uint8_amd64.go @@ -0,0 +1,301 @@ +//go:build amd64 && go1.26 && goexperiment.simd + +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// AVX-512 / AVX2 SIMD distance kernels for vecuint8 ([]uint8), INTEGER-EXACT +// (bit-for-bit identical to the int64-accumulating pure-Go oracle). +// +// Like the int8 kernels we load the raw bytes as a 32-bit-lane vector (4 uint8 +// per lane) and split the four byte lanes. The ONLY difference from int8 is the +// unpack: uint8 is ZERO-extended (mask each byte with 0xFF / logical shift), +// whereas int8 sign-extends with arithmetic shifts. The values land in [0,255] +// so reinterpreting the masked Uint32 lanes as Int32 (AsInt32x16) is exact, and +// all subsequent arithmetic (Sub/Mul/Add/Max) stays in signed int32 lanes — +// d=a-b is in [-255,255], d*d <= 65025, a*b in [0,65025]. For the max dimension +// (65535) a lane accumulates < 1100 terms each <= 65025, i.e. < 2^28, far under +// 2^31; the final horizontal reduction is in int64. No float, so results equal +// the oracle exactly (the equivalence test asserts == for L2sq/IP/L1). +// +// The pure-Go kernels in distance_func_narrow_uint8.go stay the fallback +// (non-AVX2 CPUs) and the equivalence oracle; init() only swaps the selection +// vars when AVX-512 / AVX2 is present. hasAVX512 / hasAVX2 / sumI32x16 / +// sumI32x8 are shared with the int8/bf16/f16 kernels (same package + build tag). + +package metric + +import ( + "math" + "unsafe" + + "simd/archsimd" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" +) + +func init() { + switch { + case hasAVX512: + uint8L2sqFn = l2sqUint8SIMD + uint8IPFn = innerProductUint8SIMD + uint8CosineFn = cosineDistanceUint8SIMD + uint8L1Fn = l1DistanceUint8SIMD + case hasAVX2: + uint8L2sqFn = l2sqUint8AVX2 + uint8IPFn = innerProductUint8AVX2 + uint8CosineFn = cosineDistanceUint8AVX2 + uint8L1Fn = l1DistanceUint8AVX2 + } +} + +// uint8AsU32 reinterprets a []uint8 as []uint32 viewing its first len/4 dwords +// (4 packed uint8 per lane). +func uint8AsU32(s []uint8) []uint32 { + if len(s) < 4 { + return nil + } + return unsafe.Slice((*uint32)(unsafe.Pointer(unsafe.SliceData(s))), len(s)/4) +} + +// ---- uint8 (AVX-512), integer-exact ---- + +// unpackU8 zero-extends the four byte lanes of a Uint32x16 (64 packed uint8) +// into four Int32x16 vectors. Lane k of vJ holds uint8[4k+J] as a value in +// [0,255]. mask is BroadcastUint32x16(0xFF); the top byte (>>24) needs no mask. +func unpackU8(u, mask archsimd.Uint32x16) (v0, v1, v2, v3 archsimd.Int32x16) { + v0 = u.And(mask).AsInt32x16() + v1 = u.ShiftAllRight(8).And(mask).AsInt32x16() + v2 = u.ShiftAllRight(16).And(mask).AsInt32x16() + v3 = u.ShiftAllRight(24).AsInt32x16() + return +} + +func l2sqUint8SIMD(a, b []uint8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := uint8AsU32(a), uint8AsU32(b) + mask := archsimd.BroadcastUint32x16(0xFF) + acc := archsimd.Int32x16{} + nq, j := len(au), 0 + for ; j <= nq-16; j += 16 { + a0, a1, a2, a3 := unpackU8(archsimd.LoadUint32x16Slice(au[j:j+16]), mask) + b0, b1, b2, b3 := unpackU8(archsimd.LoadUint32x16Slice(bu[j:j+16]), mask) + d0, d1, d2, d3 := a0.Sub(b0), a1.Sub(b1), a2.Sub(b2), a3.Sub(b3) + acc = acc.Add(d0.Mul(d0).Add(d1.Mul(d1)).Add(d2.Mul(d2).Add(d3.Mul(d3)))) + } + sum := sumI32x16(acc) + for i := j * 4; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + sum += int64(d * d) + } + return float64(sum), nil +} + +func innerProductUint8SIMD(a, b []uint8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := uint8AsU32(a), uint8AsU32(b) + mask := archsimd.BroadcastUint32x16(0xFF) + acc := archsimd.Int32x16{} + nq, j := len(au), 0 + for ; j <= nq-16; j += 16 { + a0, a1, a2, a3 := unpackU8(archsimd.LoadUint32x16Slice(au[j:j+16]), mask) + b0, b1, b2, b3 := unpackU8(archsimd.LoadUint32x16Slice(bu[j:j+16]), mask) + acc = acc.Add(a0.Mul(b0).Add(a1.Mul(b1)).Add(a2.Mul(b2).Add(a3.Mul(b3)))) + } + sum := sumI32x16(acc) + for i := j * 4; i < n; i++ { + sum += int64(int32(a[i]) * int32(b[i])) + } + return float64(-sum), nil +} + +func l1DistanceUint8SIMD(a, b []uint8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := uint8AsU32(a), uint8AsU32(b) + mask := archsimd.BroadcastUint32x16(0xFF) + zero := archsimd.Int32x16{} + acc := archsimd.Int32x16{} + abs := func(d archsimd.Int32x16) archsimd.Int32x16 { return d.Max(zero.Sub(d)) } + nq, j := len(au), 0 + for ; j <= nq-16; j += 16 { + a0, a1, a2, a3 := unpackU8(archsimd.LoadUint32x16Slice(au[j:j+16]), mask) + b0, b1, b2, b3 := unpackU8(archsimd.LoadUint32x16Slice(bu[j:j+16]), mask) + acc = acc.Add(abs(a0.Sub(b0)).Add(abs(a1.Sub(b1))).Add(abs(a2.Sub(b2)).Add(abs(a3.Sub(b3))))) + } + sum := sumI32x16(acc) + for i := j * 4; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + if d < 0 { + d = -d + } + sum += int64(d) + } + return float64(sum), nil +} + +func cosineDistanceUint8SIMD(a, b []uint8) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := uint8AsU32(a), uint8AsU32(b) + mask := archsimd.BroadcastUint32x16(0xFF) + dotA, naA, nbA := archsimd.Int32x16{}, archsimd.Int32x16{}, archsimd.Int32x16{} + nq, j := len(au), 0 + for ; j <= nq-16; j += 16 { + a0, a1, a2, a3 := unpackU8(archsimd.LoadUint32x16Slice(au[j:j+16]), mask) + b0, b1, b2, b3 := unpackU8(archsimd.LoadUint32x16Slice(bu[j:j+16]), mask) + dotA = dotA.Add(a0.Mul(b0).Add(a1.Mul(b1)).Add(a2.Mul(b2).Add(a3.Mul(b3)))) + naA = naA.Add(a0.Mul(a0).Add(a1.Mul(a1)).Add(a2.Mul(a2).Add(a3.Mul(a3)))) + nbA = nbA.Add(b0.Mul(b0).Add(b1.Mul(b1)).Add(b2.Mul(b2).Add(b3.Mul(b3)))) + } + dot, na2, nb2 := sumI32x16(dotA), sumI32x16(naA), sumI32x16(nbA) + for i := j * 4; i < n; i++ { + ai, bi := int64(a[i]), int64(b[i]) + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} + +// ---- uint8 (AVX2), integer-exact ---- + +// unpackU8x8 is the 256-bit (8-lane) twin of unpackU8. +func unpackU8x8(u, mask archsimd.Uint32x8) (v0, v1, v2, v3 archsimd.Int32x8) { + v0 = u.And(mask).AsInt32x8() + v1 = u.ShiftAllRight(8).And(mask).AsInt32x8() + v2 = u.ShiftAllRight(16).And(mask).AsInt32x8() + v3 = u.ShiftAllRight(24).AsInt32x8() + return +} + +func l2sqUint8AVX2(a, b []uint8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := uint8AsU32(a), uint8AsU32(b) + mask := archsimd.BroadcastUint32x8(0xFF) + acc := archsimd.Int32x8{} + nq, j := len(au), 0 + for ; j <= nq-8; j += 8 { + a0, a1, a2, a3 := unpackU8x8(archsimd.LoadUint32x8Slice(au[j:j+8]), mask) + b0, b1, b2, b3 := unpackU8x8(archsimd.LoadUint32x8Slice(bu[j:j+8]), mask) + d0, d1, d2, d3 := a0.Sub(b0), a1.Sub(b1), a2.Sub(b2), a3.Sub(b3) + acc = acc.Add(d0.Mul(d0).Add(d1.Mul(d1)).Add(d2.Mul(d2).Add(d3.Mul(d3)))) + } + sum := sumI32x8(acc) + for i := j * 4; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + sum += int64(d * d) + } + return float64(sum), nil +} + +func innerProductUint8AVX2(a, b []uint8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := uint8AsU32(a), uint8AsU32(b) + mask := archsimd.BroadcastUint32x8(0xFF) + acc := archsimd.Int32x8{} + nq, j := len(au), 0 + for ; j <= nq-8; j += 8 { + a0, a1, a2, a3 := unpackU8x8(archsimd.LoadUint32x8Slice(au[j:j+8]), mask) + b0, b1, b2, b3 := unpackU8x8(archsimd.LoadUint32x8Slice(bu[j:j+8]), mask) + acc = acc.Add(a0.Mul(b0).Add(a1.Mul(b1)).Add(a2.Mul(b2).Add(a3.Mul(b3)))) + } + sum := sumI32x8(acc) + for i := j * 4; i < n; i++ { + sum += int64(int32(a[i]) * int32(b[i])) + } + return float64(-sum), nil +} + +func l1DistanceUint8AVX2(a, b []uint8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := uint8AsU32(a), uint8AsU32(b) + mask := archsimd.BroadcastUint32x8(0xFF) + zero := archsimd.Int32x8{} + acc := archsimd.Int32x8{} + abs := func(d archsimd.Int32x8) archsimd.Int32x8 { return d.Max(zero.Sub(d)) } + nq, j := len(au), 0 + for ; j <= nq-8; j += 8 { + a0, a1, a2, a3 := unpackU8x8(archsimd.LoadUint32x8Slice(au[j:j+8]), mask) + b0, b1, b2, b3 := unpackU8x8(archsimd.LoadUint32x8Slice(bu[j:j+8]), mask) + acc = acc.Add(abs(a0.Sub(b0)).Add(abs(a1.Sub(b1))).Add(abs(a2.Sub(b2)).Add(abs(a3.Sub(b3))))) + } + sum := sumI32x8(acc) + for i := j * 4; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + if d < 0 { + d = -d + } + sum += int64(d) + } + return float64(sum), nil +} + +func cosineDistanceUint8AVX2(a, b []uint8) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := uint8AsU32(a), uint8AsU32(b) + mask := archsimd.BroadcastUint32x8(0xFF) + dotA, naA, nbA := archsimd.Int32x8{}, archsimd.Int32x8{}, archsimd.Int32x8{} + nq, j := len(au), 0 + for ; j <= nq-8; j += 8 { + a0, a1, a2, a3 := unpackU8x8(archsimd.LoadUint32x8Slice(au[j:j+8]), mask) + b0, b1, b2, b3 := unpackU8x8(archsimd.LoadUint32x8Slice(bu[j:j+8]), mask) + dotA = dotA.Add(a0.Mul(b0).Add(a1.Mul(b1)).Add(a2.Mul(b2).Add(a3.Mul(b3)))) + naA = naA.Add(a0.Mul(a0).Add(a1.Mul(a1)).Add(a2.Mul(a2).Add(a3.Mul(a3)))) + nbA = nbA.Add(b0.Mul(b0).Add(b1.Mul(b1)).Add(b2.Mul(b2).Add(b3.Mul(b3)))) + } + dot, na2, nb2 := sumI32x8(dotA), sumI32x8(naA), sumI32x8(nbA) + for i := j * 4; i < n; i++ { + ai, bi := int64(a[i]), int64(b[i]) + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} diff --git a/pkg/vectorindex/metric/distance_func_test.go b/pkg/vectorindex/metric/distance_func_test.go index 057e47a2c4e30..4dcaa99f100aa 100644 --- a/pkg/vectorindex/metric/distance_func_test.go +++ b/pkg/vectorindex/metric/distance_func_test.go @@ -15,7 +15,6 @@ package metric import ( - "fmt" "math" "testing" @@ -47,10 +46,8 @@ func Test_Blas32(t *testing.T) { distfn, _, err := ResolveKmeansDistanceFn[float32](Metric_L2Distance, false) require.Nil(t, err) - v, err := distfn(v1.Data, v2.Data) + _, err = distfn(v1.Data, v2.Data) require.Nil(t, err) - - fmt.Printf("blas32 v = %v\n", v) } func Test_ResolveFun(t *testing.T) { @@ -213,6 +210,22 @@ func Test_L2Distance(t *testing.T) { }, want: 3.1622776601683795, }, + { + name: "Test 5", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 5.196152422706632, + }, + { + name: "Test 6", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2}, + }, + want: 4.58257569495584, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -281,6 +294,22 @@ func Test_L1Distance(t *testing.T) { }, want: 10, }, + { + name: "Test 5", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 27, + }, + { + name: "Test 6", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2}, + }, + want: 21, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -349,6 +378,22 @@ func Test_CosineDistance(t *testing.T) { }, want: 0.0021238962030426523, }, + { + name: "Test 5", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 0.0025062434610066964, + }, + { + name: "Test 6", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2}, + }, + want: 0.002478147161370292, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -359,6 +404,90 @@ func Test_CosineDistance(t *testing.T) { } } +func Test_CosineSimilarity(t *testing.T) { + type args struct { + v1 []float64 + v2 []float64 + } + tests := []struct { + name string + args args + want float64 + }{ + { + name: "Test 1", + args: args{ + v1: []float64{1, 2, 3, 4}, + v2: []float64{1, 2, 4, 5}, + }, + want: 0.9960065188076063, + }, + { + name: "Test 2", + args: args{ + v1: []float64{10, 20, 30, 40}, + v2: []float64{10.5, 21.5, 31.5, 43.5}, + }, + want: 0.9998746426104126, + }, + { + name: "Test 3.a", + args: args{ + v1: []float64{1, 1}, + v2: []float64{4, 1}, + }, + want: 0.8574929257125441, + }, + { + name: "Test 3.b", + args: args{ + v1: []float64{4, 1}, + v2: []float64{1, 4}, + }, + want: 0.47058823529411764, + }, + { + name: "Test 3.c", + args: args{ + v1: []float64{1, 4}, + v2: []float64{1, 1}, + }, + want: 0.8574929257125441, + }, + { + name: "Test 4", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + }, + want: 0.9978761037969573, + }, + { + name: "Test 5", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 0.9974937565389933, + }, + { + name: "Test 6", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2}, + }, + want: 0.9975218528386297, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, err := CosineSimilarity[float64](tt.args.v1, tt.args.v2); err != nil || got != tt.want { + t.Errorf("CosineSimilarity() = %v, want %v", got, tt.want) + } + }) + } +} + func Test_InnerProduct(t *testing.T) { type args struct { v1 []float64 @@ -417,6 +546,22 @@ func Test_InnerProduct(t *testing.T) { }, want: -440, }, + { + name: "Test 5", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: -1048, + }, + { + name: "Test 6", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2}, + }, + want: -882, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -485,6 +630,22 @@ func Test_L2DistanceSq(t *testing.T) { }, want: 10, }, + { + name: "Test 5", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 27, + }, + { + name: "Test 6", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2}, + }, + want: 21, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -555,6 +716,22 @@ func Test_AngularDistance(t *testing.T) { }, want: 0, }, + { + name: "Test 5", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 0, + }, + { + name: "Test 6", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2}, + }, + want: 0, + }, // Test 4: Triangle Inequality check on **normalized** vector // A(1,0),B(2,2), C(0,1) => AB + AC >= BC => 0.25 + 0.25 >= 0.5 diff --git a/pkg/vectorindex/metric/gpu.go b/pkg/vectorindex/metric/gpu.go index 97d15a4babe57..5c474bd0ab76e 100644 --- a/pkg/vectorindex/metric/gpu.go +++ b/pkg/vectorindex/metric/gpu.go @@ -53,7 +53,18 @@ var ( } ) -func PairWiseDistance[T types.RealNumbers]( +// gpuPairwiseSupported reports whether T is a cuVS-supported pairwise element +// type (float32 or types.Float16). bf16/int8/uint8/float64 run on CPU. +func gpuPairwiseSupported[T types.ArrayElement]() bool { + switch any(*new(T)).(type) { + case float32, types.Float16: + return true + default: + return false + } +} + +func PairWiseDistance[T types.ArrayElement]( x [][]T, y [][]T, metric MetricType, @@ -78,8 +89,7 @@ func PairWiseDistance[T types.RealNumbers]( return GoPairWiseDistance(x, y, metric) } - var zero T - if _, isF32 := any(zero).(float32); isF32 { + if gpuPairwiseSupported[T]() { res := make([]float32, nX*nY) handle, err := PairwiseDistanceLaunch(x, y, metric, res, GPUThresholdSync, gpuMode) if err != nil { @@ -145,7 +155,7 @@ func (m *gpuJobManager) pop(jobID uint64) *gpuJob { // It flattens the input vectors on the CPU and then launches a CUDA kernel. // This allows for overlapping the CPU-bound flattening work with GPU execution // when pipelined at the reader level. -func PairwiseDistanceLaunch[T types.RealNumbers]( +func PairwiseDistanceLaunch[T types.ArrayElement]( x [][]T, y [][]T, metric MetricType, @@ -168,63 +178,87 @@ func PairwiseDistanceLaunch[T types.RealNumbers]( dim := len(x[0]) cuvsMetric, ok := MetricTypeToCuvsMetric[metric] - var zero T - _, isF32 := any(zero).(float32) - - if ok && isF32 && uint64(nX)*uint64(nY)*uint64(dim) >= minWorkSize { - allocator := malloc.NewCAllocator() - - // 1. Flatten Y - yf32Slice, yDeallocator, err := allocator.Allocate(uint64(nY*dim*4), malloc.NoClear) - if err != nil { - return 0, err - } - yf32 := util.UnsafeSliceCast[float32](yf32Slice) - y32 := any(y).([][]float32) - for i, v := range y32 { - copy(yf32[i*dim:(i+1)*dim], v) + if ok && uint64(nX)*uint64(nY)*uint64(dim) >= minWorkSize { + // cuVS pairwise supports float32 and Float16 only. + switch xs := any(x).(type) { + case [][]float32: + return gpuPairwiseLaunch[float32](xs, any(y).([][]float32), dim, cuvsMetric, dist, 4) + case [][]types.Float16: + ys := any(y).([][]types.Float16) + // types.Float16 and cuvs.Float16 are both IEEE binary16 uint16 — a + // per-row reinterpret (no element copy) hands them to the cuVS kernel. + xc := make([][]cuvs.Float16, len(xs)) + for i, v := range xs { + xc[i] = util.UnsafeSliceCast[cuvs.Float16](v) + } + yc := make([][]cuvs.Float16, len(ys)) + for i, v := range ys { + yc[i] = util.UnsafeSliceCast[cuvs.Float16](v) + } + return gpuPairwiseLaunch[cuvs.Float16](xc, yc, dim, cuvsMetric, dist, 2) } + } - // 2. Flatten X - xf32Slice, xDeallocator, err := allocator.Allocate(uint64(nX*dim*4), malloc.NoClear) - if err != nil { - yDeallocator.Deallocate() - return 0, err - } - xf32 := util.UnsafeSliceCast[float32](xf32Slice) - x32 := any(x).([][]float32) - for i, v := range x32 { - copy(xf32[i*dim:(i+1)*dim], v) - } + return PairwiseDistanceLaunchCPU(x, y, metric, dist) +} - // Register job before launch so the slot exists if Wait is called - // concurrently. On launch failure, pop removes it before returning; - // no caller can see the job because cuvsJobID is only set by update() - // below, which is never reached on this error path. - gpuID := globalGpuJobManager.add(dist) - - cuvsID, err := cuvs.PairwiseDistanceLaunch( - xf32, - uint64(nX), - yf32, - uint64(nY), - uint32(dim), - cuvsMetric, - dist, - ) - if err != nil { - xDeallocator.Deallocate() - yDeallocator.Deallocate() - globalGpuJobManager.pop(gpuID) - return 0, err - } +// gpuPairwiseLaunch flattens [][]C into a C-allocator buffer (elemSize bytes per +// element) and launches the async cuVS pairwise distance. C is float32 (4B) or +// cuvs.Float16 (2B). Mirrors the old f32-only path, generalized over the element. +func gpuPairwiseLaunch[C cuvs.VectorType]( + x, y [][]C, + dim int, + cuvsMetric cuvs.DistanceType, + dist []float32, + elemSize int, +) (PairwiseJobHandle, error) { + nX, nY := len(x), len(y) + allocator := malloc.NewCAllocator() + + // 1. Flatten Y + yBuf, yDeallocator, err := allocator.Allocate(uint64(nY*dim*elemSize), malloc.NoClear) + if err != nil { + return 0, err + } + yf := util.UnsafeSliceCast[C](yBuf) + for i, v := range y { + copy(yf[i*dim:(i+1)*dim], v) + } - globalGpuJobManager.update(gpuID, cuvsID, xDeallocator, yDeallocator) + // 2. Flatten X + xBuf, xDeallocator, err := allocator.Allocate(uint64(nX*dim*elemSize), malloc.NoClear) + if err != nil { + yDeallocator.Deallocate() + return 0, err + } + xf := util.UnsafeSliceCast[C](xBuf) + for i, v := range x { + copy(xf[i*dim:(i+1)*dim], v) + } - return PairwiseJobHandle(gpuID), nil + // Register job before launch so the slot exists if Wait is called + // concurrently. On launch failure, pop removes it before returning. + gpuID := globalGpuJobManager.add(dist) + + cuvsID, err := cuvs.PairwiseDistanceLaunch( + xf, + uint64(nX), + yf, + uint64(nY), + uint32(dim), + cuvsMetric, + dist, + ) + if err != nil { + xDeallocator.Deallocate() + yDeallocator.Deallocate() + globalGpuJobManager.pop(gpuID) + return 0, err } - return PairwiseDistanceLaunchCPU(x, y, metric, dist) + globalGpuJobManager.update(gpuID, cuvsID, xDeallocator, yDeallocator) + + return PairwiseJobHandle(gpuID), nil } // PairwiseDistanceWait waits for the completion of the asynchronous GPU distance diff --git a/pkg/vectorindex/metric/pairwise.go b/pkg/vectorindex/metric/pairwise.go index 08ad7135ca600..a012c5f5ade2c 100644 --- a/pkg/vectorindex/metric/pairwise.go +++ b/pkg/vectorindex/metric/pairwise.go @@ -54,13 +54,14 @@ var ( // While this is currently synchronous for CPU (it performs the calculation in Launch), // it follows the asynchronous interface to support the pipelined execution model // used in the block reader. -func PairwiseDistanceLaunchCPU[T types.RealNumbers]( +func PairwiseDistanceLaunchCPU[T types.ArrayElement]( x [][]T, y [][]T, metric MetricType, dist []float32, ) (PairwiseJobHandle, error) { - distFn, err := ResolveDistanceFn[T](metric) + // R=float32: the output is []float32, matching the prior float32(d) truncation. + distFn, err := ResolveDistanceFn[T, float32](metric) if err != nil { return 0, err } @@ -75,38 +76,18 @@ func PairwiseDistanceLaunchCPU[T types.RealNumbers]( dist: dist, } - // Do the calculation in Launch - switch xTyped := any(x).(type) { - case [][]float32: - yTyped := any(y).([][]float32) - dFn := any(distFn).(DistanceFunction[float32]) - for r := 0; r < nX; r++ { - xr := xTyped[r] - for c := 0; c < nY; c++ { - d, err := dFn(xr, yTyped[c]) - if err != nil { - job.err = err - goto DONE - } - dist[r*nY+c] = float32(d) + // One unified loop over any ArrayElement type — the resolver handles f32/f64 + // and the narrow kernels (bf16/f16/int8/uint8) uniformly. + for r := 0; r < nX; r++ { + xr := x[r] + for c := 0; c < nY; c++ { + d, err := distFn(xr, y[c]) + if err != nil { + job.err = err + goto DONE } + dist[r*nY+c] = d } - case [][]float64: - yTyped := any(y).([][]float64) - dFn := any(distFn).(DistanceFunction[float64]) - for r := 0; r < nX; r++ { - xr := xTyped[r] - for c := 0; c < nY; c++ { - d, err := dFn(xr, yTyped[c]) - if err != nil { - job.err = err - goto DONE - } - dist[r*nY+c] = float32(d) - } - } - default: - return 0, moerr.NewInternalErrorNoCtx("unsupported type in PairwiseDistanceLaunchCPU") } if metric == Metric_L2Distance { diff --git a/pkg/vectorindex/metric/pairwise_test.go b/pkg/vectorindex/metric/pairwise_test.go index d12956ee50a06..8aab26132f5b1 100644 --- a/pkg/vectorindex/metric/pairwise_test.go +++ b/pkg/vectorindex/metric/pairwise_test.go @@ -50,7 +50,7 @@ func TestPairWiseDistance(t *testing.T) { require.Equal(t, nX*nY, len(dist)) // Verify against direct calls - distFn, err := ResolveDistanceFn[float32](m) + distFn, err := ResolveDistanceFn[float32, float32](m) require.NoError(t, err) for i := 0; i < nX; i++ { diff --git a/pkg/vectorindex/metric/resolve.go b/pkg/vectorindex/metric/resolve.go new file mode 100644 index 0000000000000..349f80baaf19f --- /dev/null +++ b/pkg/vectorindex/metric/resolve.go @@ -0,0 +1,233 @@ +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// NOTE: This file is intentionally UNTAGGED (no //go:build constraint). +// The distance KERNELS live in build-tag alternatives — distance_func.go +// (scalar, !(amd64 && goexperiment.simd)) and distance_func_amd64.go (SIMD, +// amd64 && go1.26 && goexperiment.simd) — so only one compiles per build. The +// resolver / orchestration helpers below are build-tag-independent (they just +// pick and call a kernel), so they must NOT live in a tagged file, or they +// would vanish on the SIMD build and break every caller (kmeans / brute_force / +// ivf). They belong here, where they compile in every build. + +package metric + +import ( + "math" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +// IMPORTANT: Elkans Kmeans always use L2Distance for dense vector or images. After getting the centroids, we can use other distance function +// specified by user to assign vector to corresponding centroids (CENTROIDX JOIN / ProductL2). + +func ResolveKmeansDistanceFn[T types.RealNumbers](metric MetricType, spherical bool) (DistanceFunction[T], bool, error) { + if spherical { + return ResolveKmeansDistanceFnForSparse[T](metric) + } + return ResolveKmeansDistanceFnForDense[T](metric) +} + +func ResolveKmeansDistanceFnForDense[T types.RealNumbers](metric MetricType) (DistanceFunction[T], bool, error) { + var distanceFunction DistanceFunction[T] + normalize := false + switch metric { + case Metric_L2Distance: + distanceFunction = L2Distance[T] + normalize = false + case Metric_L2sqDistance: + // Elkans Kmeans always uses true L2Distance regardless of user metric. + distanceFunction = L2Distance[T] + normalize = false + case Metric_InnerProduct: + distanceFunction = L2Distance[T] + normalize = false + case Metric_CosineDistance: + distanceFunction = L2Distance[T] + normalize = false + case Metric_L1Distance: + distanceFunction = L2Distance[T] + normalize = false + default: + return nil, normalize, moerr.NewInternalErrorNoCtx("invalid distance type") + } + return distanceFunction, normalize, nil +} + +// IMPORTANT: Spherical Kmeans always use Spherical Distance / Cosine Similarity for Sparse vector or text embedding (TD-IDF). +// After getting the centroids, we can use other distance function +// specified by user to assign vector to corresponding centroids (CENTROIDX JOIN / ProductL2). +func ResolveKmeansDistanceFnForSparse[T types.RealNumbers](metric MetricType) (DistanceFunction[T], bool, error) { + var distanceFunction DistanceFunction[T] + normalize := false + switch metric { + case Metric_L2Distance: + distanceFunction = L2Distance[T] + normalize = false + case Metric_L2sqDistance: + distanceFunction = L2Distance[T] + normalize = false + case Metric_InnerProduct: + distanceFunction = SphericalDistance[T] + normalize = true + case Metric_CosineDistance: + distanceFunction = SphericalDistance[T] + normalize = true + case Metric_L1Distance: + distanceFunction = L2Distance[T] + normalize = false + default: + return nil, normalize, moerr.NewInternalErrorNoCtx("invalid distance type") + } + return distanceFunction, normalize, nil +} + +// resolveRealKernel picks the float32/float64 metric kernel (returning the value +// in its own type T). It is the f32/f64 half of ResolveDistanceFn. +func resolveRealKernel[T types.RealNumbers](metric MetricType) (DistanceFunction[T], error) { + switch metric { + case Metric_L2Distance: + return L2DistanceSq[T], nil // caller must sqrt; squared distance + case Metric_L2sqDistance: + return L2DistanceSq[T], nil + case Metric_InnerProduct: + return InnerProduct[T], nil + case Metric_CosineDistance: + return CosineDistance[T], nil + case Metric_L1Distance: + return L1Distance[T], nil + default: + return nil, moerr.NewInternalErrorNoCtx("invalid distance type") + } +} + +// ResolveDistanceFn is the single distance resolver for search / assign-to- +// centroid (CENTROIDX JOIN / ProductL2), brute force, pairwise and topn. It +// works for any storage element type T (types.ArrayElement) and returns the +// distance in a caller-chosen result type R (types.RealNumbers): pass +// R=float32 for the common path and R=float64 only where f64 precision is +// needed (f64 input, topn ordering values). f32/f64 use the metric kernels; +// bf16/f16/int8/uint8 use the native narrow kernels (which compute in +// float32/int64 and are cast to R — casting their float64 down to float32 is +// bit-identical to a native-float32 kernel, since the intermediate is exact). +// +// IMPORTANT: Don't use it for Elkans Kmeans (use ResolveKmeansDistanceFn). +// NOTE: Metric_L2Distance returns squared L2; callers needing true L2 sqrt the +// result (as GoPairWiseDistance does). +func ResolveDistanceFn[T types.ArrayElement, R types.RealNumbers](metric MetricType) (func(a, b []T) (R, error), error) { + // Each case resolves the CONCRETE element kernel, then rebinds it to + // func([]T,...)(R,error) ONCE here (not per call). When R already equals the + // kernel's native result type (e.g. f32 input with R=float32, or a narrow + // kernel's float64 with R=float64), the kernel IS that type — return it + // directly, so the hot loop is a single direct call exactly like before. Only + // when R differs (a cast is genuinely needed, e.g. narrow float64 -> float32) + // do we add a thin casting wrapper. + switch any(*new(T)).(type) { + case float32: + fn, err := resolveRealKernel[float32](metric) + if err != nil { + return nil, err + } + if f, ok := any(fn).(func(a, b []T) (R, error)); ok { + return f, nil + } + w := func(a, b []float32) (R, error) { d, e := fn(a, b); return R(d), e } + return any(w).(func(a, b []T) (R, error)), nil + case float64: + fn, err := resolveRealKernel[float64](metric) + if err != nil { + return nil, err + } + if f, ok := any(fn).(func(a, b []T) (R, error)); ok { + return f, nil + } + w := func(a, b []float64) (R, error) { d, e := fn(a, b); return R(d), e } + return any(w).(func(a, b []T) (R, error)), nil + case types.BF16: + k, err := resolveBF16Kernel(metric) + if err != nil { + return nil, err + } + if f, ok := any(k).(func(a, b []T) (R, error)); ok { + return f, nil + } + w := func(a, b []types.BF16) (R, error) { d, e := k(a, b); return R(d), e } + return any(w).(func(a, b []T) (R, error)), nil + case types.Float16: + k, err := resolveF16Kernel(metric) + if err != nil { + return nil, err + } + if f, ok := any(k).(func(a, b []T) (R, error)); ok { + return f, nil + } + w := func(a, b []types.Float16) (R, error) { d, e := k(a, b); return R(d), e } + return any(w).(func(a, b []T) (R, error)), nil + case int8: + k, err := resolveInt8Kernel(metric) + if err != nil { + return nil, err + } + if f, ok := any(k).(func(a, b []T) (R, error)); ok { + return f, nil + } + w := func(a, b []int8) (R, error) { d, e := k(a, b); return R(d), e } + return any(w).(func(a, b []T) (R, error)), nil + case uint8: + k, err := resolveUint8Kernel(metric) + if err != nil { + return nil, err + } + if f, ok := any(k).(func(a, b []T) (R, error)); ok { + return f, nil + } + w := func(a, b []uint8) (R, error) { d, e := k(a, b); return R(d), e } + return any(w).(func(a, b []T) (R, error)), nil + default: + return nil, moerr.NewInternalErrorNoCtx("ResolveDistanceFn: unsupported element type") + } +} + +func GoPairWiseDistance[T types.ArrayElement]( + x [][]T, + y [][]T, + metric MetricType, +) ([]float32, error) { + distFn, err := ResolveDistanceFn[T, float32](metric) + if err != nil { + return nil, err + } + + nX := len(x) + nY := len(y) + res := make([]float32, nX*nY) + for i := 0; i < nX; i++ { + for j := 0; j < nY; j++ { + d, err := distFn(x[i], y[j]) + if err != nil { + return nil, err + } + res[i*nY+j] = d + } + } + + if metric == Metric_L2Distance { + for i := range res { + res[i] = float32(math.Sqrt(float64(res[i]))) + } + } + + return res, nil +} diff --git a/pkg/vectorindex/metric/types.go b/pkg/vectorindex/metric/types.go index 41d4db6715de9..b2dc729fd1b64 100644 --- a/pkg/vectorindex/metric/types.go +++ b/pkg/vectorindex/metric/types.go @@ -65,6 +65,7 @@ const ( const ( Quantization_F32_Str = "float32" Quantization_F16_Str = "float16" + Quantization_BF16_Str = "bf16" Quantization_INT8_Str = "int8" Quantization_UINT8_Str = "uint8" Quantization_F64_Str = "float64" diff --git a/pkg/vectorindex/quantizer/quantizer.go b/pkg/vectorindex/quantizer/quantizer.go new file mode 100644 index 0000000000000..2c5a008df802d --- /dev/null +++ b/pkg/vectorindex/quantizer/quantizer.go @@ -0,0 +1,246 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package quantizer is the single source of truth for ivfflat QUANTIZATION: the +// mapping from a CREATE INDEX QUANTIZATION='...' name to the narrow entry type, +// the cuVS-style asymmetric int8 scalar quantizer (training the [min,max] bounds, +// deriving the q(x)=round(x*mul+add) transform, applying it to a query vector, +// and emitting the equivalent SQL entry projection), and the SQL type names. +// +// The same q(x)=x*mul+add must be applied identically on three sides — the +// synchronous build (compile), the CDC delta writer (iscp), and search — so the +// formula lives here once and each side calls in rather than re-deriving it. +package quantizer + +import ( + "fmt" + "math" + "slices" + "strings" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/vectorindex/metric" +) + +const ( + // Int8Lo/Int8Hi are the signed int8 range the quantizer maps [min,max] onto. + Int8Lo = -128.0 + Int8Hi = 127.0 + // int8Span is the number of quantization steps (Int8Hi-Int8Lo = 255). + int8Span = Int8Hi - Int8Lo + + // Uint8Lo/Uint8Hi are the unsigned uint8 range the quantizer maps [min,max] + // onto (same step count as int8, just shifted to start at 0 — no -128 offset). + Uint8Lo = 0.0 + Uint8Hi = 255.0 + // uint8Span is the number of quantization steps (Uint8Hi-Uint8Lo = 255). + uint8Span = Uint8Hi - Uint8Lo +) + +// ToVectorType maps a CREATE INDEX QUANTIZATION='...' value to the vector element +// type the ivfflat ENTRIES are down-cast to (the base column and centroids are +// unaffected). The accepted names are the canonical metric.Quantization_*_Str +// constants (case-insensitive): float32 -> vecf32, float16 -> vecf16, +// bf16 -> vecbf16, int8 -> vecint8, uint8 -> vecuint8. float32/float16/bf16 are +// float formats (plain cast); int8/uint8 use the trained scalar quantizer. float32 +// is accepted because it is a real down-cast for an f64 base; float64 (an up-cast) +// and "" return ok=false (no quantization; entries keep the base type). +func ToVectorType(q string) (types.T, bool) { + switch strings.ToLower(strings.TrimSpace(q)) { + case metric.Quantization_F32_Str: + return types.T_array_float32, true + case metric.Quantization_F16_Str: + return types.T_array_float16, true + case metric.Quantization_BF16_Str: + return types.T_array_bf16, true + case metric.Quantization_INT8_Str: + return types.T_array_int8, true + case metric.Quantization_UINT8_Str: + return types.T_array_uint8, true + } + return 0, false +} + +// SQLTypeName returns the SQL type name for a vector element type, for use in +// CAST(... AS (dim)). Thin alias over types.T.ArraySQLName (the canonical +// source of the lowercase SQL spellings). +func SQLTypeName(t types.T) string { + return t.ArraySQLName() +} + +// Int8Params returns (mul, add) for the cuVS-style asymmetric int8 scalar +// quantizer that maps [min,max] onto the full int8 range [Int8Lo,Int8Hi]: +// +// q(x) = round(x*mul + add), clamped to [-128,127] +// +// add folds the -min offset and the -128 int8 shift into one constant, so both +// the build (cast(base*mul+add as vecint8)) and search apply it with a single +// multiply-add. A degenerate range (max<=min) falls back to identity. +func Int8Params(min, max float64) (mul, add float64) { + rng := max - min + // !(rng > 0) also rejects NaN (every NaN comparison is false), and the IsInf + // guard rejects a non-finite range — either would otherwise yield NaN/Inf mul + // that poisons the build SQL and the query transform. + if !(rng > 0) || math.IsInf(rng, 0) { + return 1.0, 0.0 + } + mul = int8Span / rng + add = -min*mul + Int8Lo + return mul, add +} + +// TrainInt8 returns (P0.1, P99.9) of the sample values — the bounds for the +// asymmetric int8 scalar quantizer. Percentiles (not raw min/max) clip outliers +// so the quantization grid isn't wasted on a few extreme values. Returns (-1,1) +// for empty data and widens a degenerate range. Subsamples to bound cost. +func TrainInt8[T types.RealNumbers](data [][]T) (min, max float64) { + const maxVals = 2_000_000 + total := 0 + for _, v := range data { + total += len(v) + } + if total == 0 { + return -1, 1 + } + stride := 1 + if total > maxVals { + stride = total/maxVals + 1 + } + vals := make([]float64, 0, total/stride+1) + k := 0 + for _, v := range data { + for _, x := range v { + if k%stride == 0 { + f := float64(x) + // Skip NaN/Inf: they make slices.Sort's order undefined (so a + // percentile pick could land on NaN) and would poison the trained + // bounds and the SQL literal. + if !math.IsNaN(f) && !math.IsInf(f, 0) { + vals = append(vals, f) + } + } + k++ + } + } + if len(vals) == 0 { + return -1, 1 + } + slices.Sort(vals) + lo := vals[int(float64(len(vals)-1)*0.001)] + hi := vals[int(float64(len(vals)-1)*0.999)] + if hi <= lo { + hi = lo + 1 + } + return lo, hi +} + +// ApplyInt8 applies q(x)=x*mul+add to a float32 query vector and narrows to int8 +// (round+clamp), matching the entry build. (mul,add)=(1,0) is identity (no +// quantizer trained), so the raw narrowing cast is used. The multiply-add is done +// in float64 (then narrowed) to match the build side, whose entry SQL +// `cast(base*mul+add as vecint8)` evaluates the f64 literals in the base column's +// arithmetic — doing it in float32 here could bucket a boundary component +// differently. qf32 is never mutated. +func ApplyInt8(qf32 []float32, mul, add float64) []int8 { + if mul == 1.0 && add == 0.0 { + return types.Float32ToInt8Slice(qf32) + } + sq := make([]float32, len(qf32)) + for i, x := range qf32 { + sq[i] = float32(float64(x)*mul + add) + } + return types.Float32ToInt8Slice(sq) +} + +// CastSQL builds `cast( as (dim))` for narrowing an entry to a +// quantization type without scaling — float formats (float16/bf16/float32), or +// int8 when no [min,max] bounds were trained (the implicit cast does identity +// round+clamp). colExpr is the already-quoted column reference or sub-expression. +func CastSQL(colExpr string, t types.T, dim int32) string { + return fmt.Sprintf("cast(%s as %s(%d))", colExpr, SQLTypeName(t), dim) +} + +// Int8EntrySQL builds the int8 entry projection `cast( * mul + add as +// vecint8(dim))` from precomputed literal bounds (the synchronous build path, +// where compile has already read the trained [min,max] from metadata). colExpr is +// the already-quoted column reference. +func Int8EntrySQL(colExpr string, mul, add float64, dim int32) string { + return fmt.Sprintf("cast(%s * %.9g + (%.9g) as vecint8(%d))", colExpr, mul, add, dim) +} + +// Int8EntrySQLFromBounds builds the int8 entry projection where the bounds are SQL +// expressions (e.g. metadata-table subqueries) rather than precomputed literals — +// the CDC delta path, which cannot read metadata into Go before building the +// REPLACE. It inlines q(x)=x*mul+add with mul=int8Span/(max-min) and +// add=-min*mul-128, and wraps mul/add in COALESCE so an absent bound (a pure-async +// index that never trained) falls back to identity (1,0) — matching what search +// does when the bounds are missing. colExpr/minExpr/maxExpr are SQL sub-expressions. +func Int8EntrySQLFromBounds(colExpr, minExpr, maxExpr string, dim int32) string { + // 255.0 == int8Span, 128.0 == -Int8Lo, kept as literals so the division/offset + // evaluate in DOUBLE in SQL (and the generated text is stable). + rng := fmt.Sprintf("(%s - %s)", maxExpr, minExpr) + mul := fmt.Sprintf("COALESCE(255.0 / %s, 1.0)", rng) + add := fmt.Sprintf("COALESCE(0.0 - %s * 255.0 / %s - 128.0, 0.0)", minExpr, rng) + return fmt.Sprintf("cast(%s * %s + %s as vecint8(%d))", colExpr, mul, add, dim) +} + +// Uint8Params returns (mul, add) for the cuVS-style asymmetric uint8 scalar +// quantizer that maps [min,max] onto the full uint8 range [Uint8Lo,Uint8Hi]: +// +// q(x) = round(x*mul + add), clamped to [0,255] +// +// It is the int8 quantizer shifted to start at 0 (add folds only the -min offset; +// there is no -128 shift). A degenerate range (max<=min) falls back to identity. +// The trained bounds come from TrainInt8 (the percentile training is the same). +func Uint8Params(min, max float64) (mul, add float64) { + rng := max - min + if !(rng > 0) || math.IsInf(rng, 0) { + return 1.0, 0.0 + } + mul = uint8Span / rng + add = -min*mul + Uint8Lo + return mul, add +} + +// ApplyUint8 is the uint8 analog of ApplyInt8: applies q(x)=x*mul+add to a float32 +// query vector and narrows to uint8 (round+clamp to [0,255]). (mul,add)=(1,0) is +// identity. The multiply-add is done in float64 to match the build side. qf32 is +// never mutated. +func ApplyUint8(qf32 []float32, mul, add float64) []uint8 { + if mul == 1.0 && add == 0.0 { + return types.Float32ToUint8Slice(qf32) + } + sq := make([]float32, len(qf32)) + for i, x := range qf32 { + sq[i] = float32(float64(x)*mul + add) + } + return types.Float32ToUint8Slice(sq) +} + +// Uint8EntrySQL is the uint8 analog of Int8EntrySQL: the build-side entry +// projection `cast( * mul + add as vecuint8(dim))` from literal bounds. +func Uint8EntrySQL(colExpr string, mul, add float64, dim int32) string { + return fmt.Sprintf("cast(%s * %.9g + (%.9g) as vecuint8(%d))", colExpr, mul, add, dim) +} + +// Uint8EntrySQLFromBounds is the uint8 analog of Int8EntrySQLFromBounds (CDC delta +// path). q(x)=x*mul+add with mul=255/(max-min) and add=-min*255/(max-min) — no -128 +// shift — wrapped in COALESCE for identity fallback when a bound is absent. +func Uint8EntrySQLFromBounds(colExpr, minExpr, maxExpr string, dim int32) string { + // 255.0 == uint8Span; no offset term since Uint8Lo == 0. + rng := fmt.Sprintf("(%s - %s)", maxExpr, minExpr) + mul := fmt.Sprintf("COALESCE(255.0 / %s, 1.0)", rng) + add := fmt.Sprintf("COALESCE(0.0 - %s * 255.0 / %s, 0.0)", minExpr, rng) + return fmt.Sprintf("cast(%s * %s + %s as vecuint8(%d))", colExpr, mul, add, dim) +} diff --git a/pkg/vectorindex/quantizer/quantizer_test.go b/pkg/vectorindex/quantizer/quantizer_test.go new file mode 100644 index 0000000000000..3732863ed9f1f --- /dev/null +++ b/pkg/vectorindex/quantizer/quantizer_test.go @@ -0,0 +1,287 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quantizer + +import ( + "math" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +func TestToVectorType(t *testing.T) { + cases := []struct { + in string + want types.T + ok bool + }{ + {"float32", types.T_array_float32, true}, + {"float16", types.T_array_float16, true}, + {"bf16", types.T_array_bf16, true}, + {"int8", types.T_array_int8, true}, + {"uint8", types.T_array_uint8, true}, + // case-insensitive + surrounding space + {"FLOAT16", types.T_array_float16, true}, + {"BF16", types.T_array_bf16, true}, + {" Int8 ", types.T_array_int8, true}, + {"UINT8", types.T_array_uint8, true}, + // not quantization targets + {"float64", 0, false}, + {"f16", 0, false}, // only canonical names + {"bfloat16", 0, false}, + {"", 0, false}, + {"garbage", 0, false}, + } + for _, c := range cases { + got, ok := ToVectorType(c.in) + require.Equalf(t, c.ok, ok, "ok for %q", c.in) + if c.ok { + require.Equalf(t, c.want, got, "type for %q", c.in) + } + } +} + +func TestSQLTypeName(t *testing.T) { + require.Equal(t, "vecf32", SQLTypeName(types.T_array_float32)) + require.Equal(t, "vecf64", SQLTypeName(types.T_array_float64)) + require.Equal(t, "vecbf16", SQLTypeName(types.T_array_bf16)) + require.Equal(t, "vecf16", SQLTypeName(types.T_array_float16)) + require.Equal(t, "vecint8", SQLTypeName(types.T_array_int8)) + require.Equal(t, "", SQLTypeName(types.T_int32)) +} + +func TestInt8Params(t *testing.T) { + // q(x) = round(x*mul + add) must map min -> -128 and max -> +127. + min, max := -2.0, 6.0 + mul, add := Int8Params(min, max) + qmin := min*mul + add + qmax := max*mul + add + require.InDelta(t, -128.0, qmin, 1e-6) + require.InDelta(t, 127.0, qmax, 1e-6) + // midpoint maps near 0 (the [-128,127] center is -0.5) + mid := (min+max)/2*mul + add + require.InDelta(t, -0.5, mid, 1e-6) + + // asymmetric (all-positive) range still spans the full grid. + mul, add = Int8Params(0.07, 0.83) + require.InDelta(t, -128.0, 0.07*mul+add, 1e-6) + require.InDelta(t, 127.0, 0.83*mul+add, 1e-6) + + // degenerate range -> identity (no panic / no inf). + mul, add = Int8Params(1.0, 1.0) + require.Equal(t, 1.0, mul) + require.Equal(t, 0.0, add) + mul, add = Int8Params(5.0, 1.0) + require.Equal(t, 1.0, mul) + require.Equal(t, 0.0, add) + require.False(t, math.IsInf(mul, 0)) +} + +func TestInt8ParamsEdgeCases(t *testing.T) { + // Across a variety of ranges, q(min) must hit -128 and q(max) must hit +127. + ranges := [][2]float64{ + {-10, -2}, // all-negative + {-5, 5}, // symmetric about 0 + {0.999, 1.001}, // tiny range near 1 + {-1e6, 1e6}, // huge range + {0, 255}, // exactly the int8-span width + } + for _, r := range ranges { + mul, add := Int8Params(r[0], r[1]) + require.InDeltaf(t, -128.0, r[0]*mul+add, 1e-6, "min %v", r) + require.InDeltaf(t, 127.0, r[1]*mul+add, 1e-6, "max %v", r) + // a value inside the range stays inside [-128,127]. + mid := (r[0] + r[1]) / 2 + q := mid*mul + add + require.GreaterOrEqualf(t, q, -128.0-1e-6, "mid in range %v", r) + require.LessOrEqualf(t, q, 127.0+1e-6, "mid in range %v", r) + } + + // dequant round-trip: x ~= (q - add) / mul within one quantization step. + min, max := -3.0, 7.0 + mul, add := Int8Params(min, max) + step := (max - min) / 255.0 + for _, x := range []float64{-3, -1.5, 0, 2.2, 6.99} { + q := math.Round(x*mul + add) + deq := (q - add) / mul + require.InDeltaf(t, x, deq, step, "round-trip x=%v", x) + } +} + +func TestTrainInt8(t *testing.T) { + // empty -> (-1, 1) + lo, hi := TrainInt8([][]float32{}) + require.Equal(t, -1.0, lo) + require.Equal(t, 1.0, hi) + + // uniform 0..999: P0.1 near 0, P99.9 near 999 + d := make([][]float32, 1) + d[0] = make([]float32, 1000) + for i := range d[0] { + d[0][i] = float32(i) + } + lo, hi = TrainInt8(d) + require.InDelta(t, 0.0, lo, 2) + require.InDelta(t, 999.0, hi, 2) + + // degenerate (all equal) -> (v, v+1) so the range is never zero. + lo, hi = TrainInt8([][]float32{{5, 5, 5, 5}}) + require.Equal(t, 5.0, lo) + require.Equal(t, 6.0, hi) + + // a single extreme outlier is clipped by the P99.9 percentile. + o := make([][]float32, 1) + o[0] = make([]float32, 1000) + for i := 0; i < 999; i++ { + o[0][i] = 1.0 + } + o[0][999] = 1e6 + _, hi = TrainInt8(o) + require.Less(t, hi, 1e6) + + // works on float64 too (f64-base quantization): bounds are sane and ordered + // (exact percentiles of a 4-element array are not the raw min/max). + lo64, hi64 := TrainInt8([][]float64{{-3, -1, 1, 3}}) + require.GreaterOrEqual(t, lo64, -3.0) + require.LessOrEqual(t, hi64, 3.0) + require.Less(t, lo64, hi64) +} + +func TestTrainInt8Edge(t *testing.T) { + // single value -> degenerate (v, v+1) + lo, hi := TrainInt8([][]float32{{5}}) + require.Equal(t, 5.0, lo) + require.Equal(t, 6.0, hi) + + // all-negative data: bounds stay inside the data range and ordered. + lo, hi = TrainInt8([][]float32{{-10, -8, -5, -3, -2}}) + require.GreaterOrEqual(t, lo, -10.0) + require.LessOrEqual(t, hi, -2.0) + require.Less(t, lo, hi) + + // subsampling path: > 2M values (stride > 1) must not panic and stays in range. + big := make([][]float32, 2500) + for i := range big { + v := make([]float32, 1000) // 2.5M values total + for j := range v { + v[j] = float32((i*1000 + j) % 1000) // cycles 0..999 + } + big[i] = v + } + lo, hi = TrainInt8(big) + require.GreaterOrEqual(t, lo, 0.0) + require.LessOrEqual(t, hi, 999.0) + require.Less(t, lo, hi) + + // NaN/Inf are skipped: a poisoned sample still trains finite, ordered bounds. + lo, hi = TrainInt8([][]float64{{math.NaN(), math.Inf(1), math.Inf(-1), 1, 2, 3, 4}}) + require.False(t, math.IsNaN(lo) || math.IsInf(lo, 0)) + require.False(t, math.IsNaN(hi) || math.IsInf(hi, 0)) + require.Less(t, lo, hi) +} + +func TestApplyInt8(t *testing.T) { + // identity (mul,add)=(1,0): plain round+clamp narrowing, input unchanged. + in := []float32{-130, -1.4, 0.6, 5, 200} + got := ApplyInt8(in, 1.0, 0.0) + require.Equal(t, []int8{-128, -1, 1, 5, 127}, got) + require.Equal(t, []float32{-130, -1.4, 0.6, 5, 200}, in, "input must not be mutated") + + // trained transform matches q(x)=round(x*mul+add): map [0.1,0.99] -> full range. + mul, add := Int8Params(0.10, 0.99) + q := ApplyInt8([]float32{0.10, 0.99, 0.50}, mul, add) + require.Equal(t, int8(-128), q[0]) // min -> -128 + require.Equal(t, int8(127), q[1]) // max -> +127 + // 0.50 matches the float64 multiply-add, rounded. + want := int8(math.Round(0.50*mul + add)) + require.Equal(t, want, q[2]) + + // empty input -> empty output, no panic. + require.Empty(t, ApplyInt8([]float32{}, mul, add)) +} + +func TestEntrySQLBuilders(t *testing.T) { + // literal-bounds (build) projection. + require.Equal(t, + "cast(`v` * 286.516854 + (-156.651685) as vecint8(4))", + Int8EntrySQL("`v`", 286.516854, -156.651685, 4)) + + // metadata-subquery (CDC) projection with COALESCE identity fallback. + min := "(SELECT m FROM meta WHERE k='quantize_min')" + max := "(SELECT m FROM meta WHERE k='quantize_max')" + got := Int8EntrySQLFromBounds("src1", min, max, 4) + require.Equal(t, + "cast(src1 * COALESCE(255.0 / ("+max+" - "+min+"), 1.0) + "+ + "COALESCE(0.0 - "+min+" * 255.0 / ("+max+" - "+min+") - 128.0, 0.0) as vecint8(4))", + got) + + // plain narrowing cast (float formats / untrained int8). + require.Equal(t, "cast(`v` as vecf16(8))", CastSQL("`v`", types.T_array_float16, 8)) + require.Equal(t, "cast(`v` as vecint8(8))", CastSQL("`v`", types.T_array_int8, 8)) + require.Equal(t, "cast(`v` as vecuint8(8))", CastSQL("`v`", types.T_array_uint8, 8)) +} + +func TestUint8Params(t *testing.T) { + // q(x)=round(x*mul+add) must map min -> 0 and max -> 255 (unsigned range). + min, max := -2.0, 6.0 + mul, add := Uint8Params(min, max) + require.InDelta(t, 0.0, min*mul+add, 1e-6) + require.InDelta(t, 255.0, max*mul+add, 1e-6) + // midpoint maps near the center 127.5. + require.InDelta(t, 127.5, (min+max)/2*mul+add, 1e-6) + + // all-positive range still spans the full grid. + mul, add = Uint8Params(0.07, 0.83) + require.InDelta(t, 0.0, 0.07*mul+add, 1e-6) + require.InDelta(t, 255.0, 0.83*mul+add, 1e-6) + + // degenerate range -> identity. + mul, add = Uint8Params(1.0, 1.0) + require.Equal(t, 1.0, mul) + require.Equal(t, 0.0, add) +} + +func TestApplyUint8(t *testing.T) { + // identity: round+clamp to [0,255], input unchanged. + in := []float32{-5, 0.6, 5, 254.5, 300} + got := ApplyUint8(in, 1.0, 0.0) + require.Equal(t, []uint8{0, 1, 5, 255, 255}, got) + require.Equal(t, []float32{-5, 0.6, 5, 254.5, 300}, in, "input must not be mutated") + + // trained transform maps [0.1,0.99] -> [0,255]. + mul, add := Uint8Params(0.10, 0.99) + q := ApplyUint8([]float32{0.10, 0.99, 0.50}, mul, add) + require.Equal(t, uint8(0), q[0]) + require.Equal(t, uint8(255), q[1]) + require.Equal(t, uint8(math.Round(0.50*mul+add)), q[2]) + + require.Empty(t, ApplyUint8([]float32{}, mul, add)) +} + +func TestUint8EntrySQLBuilders(t *testing.T) { + // literal-bounds (build) projection -> vecuint8. + require.Equal(t, + "cast(`v` * 286.516854 + (28.6516854) as vecuint8(4))", + Uint8EntrySQL("`v`", 286.516854, 28.6516854, 4)) + + // metadata-subquery (CDC) projection: no -128 offset, identity COALESCE fallback. + min := "(SELECT m FROM meta WHERE k='quantize_min')" + max := "(SELECT m FROM meta WHERE k='quantize_max')" + require.Equal(t, + "cast(src1 * COALESCE(255.0 / ("+max+" - "+min+"), 1.0) + "+ + "COALESCE(0.0 - "+min+" * 255.0 / ("+max+" - "+min+"), 0.0) as vecuint8(4))", + Uint8EntrySQLFromBounds("src1", min, max, 4)) +} diff --git a/pkg/vectorindex/types.go b/pkg/vectorindex/types.go index 33f875a8463ec..026df490539e1 100644 --- a/pkg/vectorindex/types.go +++ b/pkg/vectorindex/types.go @@ -23,6 +23,9 @@ import ( usearch "github.com/unum-cloud/usearch/golang" ) +// QUANTIZATION lives in pkg/vectorindex/quantizer: ToVectorType, Int8Params, +// SQLTypeName, TrainInt8, ApplyInt8, and the SQL entry-projection builders. + /* HNSW vector index using usearch @@ -179,13 +182,19 @@ type CagraParam struct { } type IvfflatIndexConfig struct { - Lists uint - Metric uint16 - InitType uint16 - Dimensions uint - Spherical bool - Version int64 - VectorType int32 + Lists uint + Metric uint16 + InitType uint16 + Dimensions uint + Spherical bool + Version int64 + VectorType int32 + // CentroidType is the element type the centroid hidden table is stored in. + // Entries always keep VectorType (the input/quantization type); centroids may + // be f32 (decoupled — best recall, fast f32 SIMD search, negligible RAM for + // few centroids) or follow VectorType (least RAM, narrow-native search). 0 == + // unset is treated as T_array_float32. (cuVS allows the same choice.) + CentroidType int32 KmeansTrainPercent float64 KmeansMaxIteration int64 } @@ -252,6 +261,14 @@ type RuntimeConfig struct { // Go never parses this payload; it's produced by the SQL layer and // consumed by the C++ eval_filter_bitmap_cpu. FilterJSON string + + // Emit, when non-nil, requests a STREAMING search: instead of returning all + // results at once, the index yields them in bounded batches by calling Emit + // once per batch (Search then returns empty keys/distances). Only the bm25 + // index honors it, and only for the no-LIMIT case (return every matching + // doc, ranked by an upstream ORDER BY) — so it walks and streams without a + // top-K heap. Other algorithms ignore this field. + Emit func(keys []any, distances []float64) error } type VectorIndexCdc[T types.RealNumbers] struct { diff --git a/pkg/vm/engine/disttae/logtailreplay/change_handle.go b/pkg/vm/engine/disttae/logtailreplay/change_handle.go index 3a7c3382ebbed..c50194ad2da5a 100755 --- a/pkg/vm/engine/disttae/logtailreplay/change_handle.go +++ b/pkg/vm/engine/disttae/logtailreplay/change_handle.go @@ -2516,7 +2516,8 @@ func appendFromEntry(src, vec *vector.Vector, offset int, mp *mpool.MPool) { case types.T_Blockid: val = vector.GetFixedAtNoTypeCheck[types.Blockid](src, offset) case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_json, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink: + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink: val = src.GetBytesAt(offset) default: //return vector.ErrVecTypeNotSupport diff --git a/pkg/vm/engine/disttae/util.go b/pkg/vm/engine/disttae/util.go index fe23578fda4b2..f678d64647da4 100644 --- a/pkg/vm/engine/disttae/util.go +++ b/pkg/vm/engine/disttae/util.go @@ -191,6 +191,26 @@ func LinearSearchOffsetByValFactory(pk *vector.Vector) func(*vector.Vector) []in v := types.ArrayToString[float64](vector.GetArrayAt[float64](pk, i)) mp[v] = true } + case types.T_array_bf16: + for i := 0; i < pk.Length(); i++ { + v := types.ArrayToString[types.BF16](vector.GetArrayAt[types.BF16](pk, i)) + mp[v] = true + } + case types.T_array_float16: + for i := 0; i < pk.Length(); i++ { + v := types.ArrayToString[types.Float16](vector.GetArrayAt[types.Float16](pk, i)) + mp[v] = true + } + case types.T_array_int8: + for i := 0; i < pk.Length(); i++ { + v := types.ArrayToString[int8](vector.GetArrayAt[int8](pk, i)) + mp[v] = true + } + case types.T_array_uint8: + for i := 0; i < pk.Length(); i++ { + v := types.ArrayToString[uint8](vector.GetArrayAt[uint8](pk, i)) + mp[v] = true + } default: panic(moerr.NewInternalErrorNoCtxf("%s not supported", pk.GetType().String())) } @@ -399,6 +419,34 @@ func LinearSearchOffsetByValFactory(pk *vector.Vector) func(*vector.Vector) []in sels = append(sels, int64(i)) } } + case types.T_array_bf16: + for i := 0; i < vec.Length(); i++ { + v := types.ArrayToString[types.BF16](vector.GetArrayAt[types.BF16](vec, i)) + if mp[v] { + sels = append(sels, int64(i)) + } + } + case types.T_array_float16: + for i := 0; i < vec.Length(); i++ { + v := types.ArrayToString[types.Float16](vector.GetArrayAt[types.Float16](vec, i)) + if mp[v] { + sels = append(sels, int64(i)) + } + } + case types.T_array_int8: + for i := 0; i < vec.Length(); i++ { + v := types.ArrayToString[int8](vector.GetArrayAt[int8](vec, i)) + if mp[v] { + sels = append(sels, int64(i)) + } + } + case types.T_array_uint8: + for i := 0; i < vec.Length(); i++ { + v := types.ArrayToString[uint8](vector.GetArrayAt[uint8](vec, i)) + if mp[v] { + sels = append(sels, int64(i)) + } + } default: panic(moerr.NewInternalErrorNoCtxf("%s not supported", vec.GetType().String())) } diff --git a/pkg/vm/engine/disttae/util_test.go b/pkg/vm/engine/disttae/util_test.go index a86bdff26af91..043c6b325379a 100644 --- a/pkg/vm/engine/disttae/util_test.go +++ b/pkg/vm/engine/disttae/util_test.go @@ -73,6 +73,50 @@ func TestLinearSearchOffsetByValFactory_Varchar(t *testing.T) { target2.Free(mp) } +// narrowArrayLinearSearch exercises LinearSearchOffsetByValFactory for a narrow +// vector array key type (vecbf16/vecf16/vecint8/vecuint8). Both the key-side map +// build and the target-side search switch on the element type, so a narrow array +// must be handled in both or this panics "not supported". +func narrowArrayLinearSearch[T types.ArrayElement](t *testing.T, mp *mpool.MPool, oid types.T, a, b, c []T) { + typ := types.New(oid, int32(len(a)), 0) + + keys := vector.NewVec(typ) + require.NoError(t, vector.AppendArray[T](keys, a, false, mp)) + require.NoError(t, vector.AppendArray[T](keys, b, false, mp)) + searchFn := LinearSearchOffsetByValFactory(keys) + + // target with no matching key + target := vector.NewVec(typ) + require.NoError(t, vector.AppendArray[T](target, c, false, mp)) + require.Empty(t, searchFn(target)) + + // target containing key b at index 1 + target2 := vector.NewVec(typ) + require.NoError(t, vector.AppendArray[T](target2, c, false, mp)) + require.NoError(t, vector.AppendArray[T](target2, b, false, mp)) + require.Equal(t, []int64{1}, searchFn(target2)) + + keys.Free(mp) + target.Free(mp) + target2.Free(mp) +} + +func TestLinearSearchOffsetByValFactory_NarrowArray(t *testing.T) { + mp := mpool.MustNewZero() + narrowArrayLinearSearch[types.Float16](t, mp, types.T_array_float16, + types.Float32ToFloat16Slice([]float32{1, 1}), + types.Float32ToFloat16Slice([]float32{2, 2}), + types.Float32ToFloat16Slice([]float32{3, 3})) + narrowArrayLinearSearch[types.BF16](t, mp, types.T_array_bf16, + types.Float32ToBF16Slice([]float32{1, 1}), + types.Float32ToBF16Slice([]float32{2, 2}), + types.Float32ToBF16Slice([]float32{3, 3})) + narrowArrayLinearSearch[int8](t, mp, types.T_array_int8, + []int8{1, 1}, []int8{2, 2}, []int8{3, 3}) + narrowArrayLinearSearch[uint8](t, mp, types.T_array_uint8, + []uint8{1, 1}, []uint8{2, 2}, []uint8{3, 3}) +} + func TestLinearSearchOffsetByValFactory_Int64(t *testing.T) { mp := mpool.MustNewZero() diff --git a/pkg/vm/engine/tae/blockio/read.go b/pkg/vm/engine/tae/blockio/read.go index 34db82a3dfe62..b435c6c5a97cc 100644 --- a/pkg/vm/engine/tae/blockio/read.go +++ b/pkg/vm/engine/tae/blockio/read.go @@ -387,6 +387,20 @@ func BlockDataReadBackup( return } +// topnDistOf builds a per-row distance closure for the topn order-by-limit scan: +// it decodes the query and each row's raw column bytes as element type T and +// returns the float64 distance via the merged ResolveDistanceFn (R=float64). +func topnDistOf[T types.ArrayElement](numVec []byte, m metric.MetricType) (func([]byte) (float64, error), error) { + distFunc, err := metric.ResolveDistanceFn[T, float64](m) + if err != nil { + return nil, err + } + rhs := types.BytesToArray[T](numVec) + return func(b []byte) (float64, error) { + return distFunc(types.BytesToArray[T](b), rhs) + }, nil +} + func HandleOrderByLimitOnIVFFlatIndex( ctx context.Context, selectRows []int64, @@ -411,110 +425,71 @@ func HandleOrderByLimitOnIVFFlatIndex( return nil, nil, err } + // Per-type distance closure: returns the float64 distance between a row's raw + // column bytes and the query vector. The single merged ResolveDistanceFn[T, + // float64] handles f32/f64 and the narrow quantizations uniformly; the bounds + // + top-k heap loop below is shared. + var distOf func(colBytes []byte) (float64, error) switch orderByLimit.Typ { case types.T_array_float32: - distFunc, err := metric.ResolveDistanceFn[float32](orderByLimit.MetricType) + distOf, err = topnDistOf[float32](orderByLimit.NumVec, orderByLimit.MetricType) + case types.T_array_float64: + distOf, err = topnDistOf[float64](orderByLimit.NumVec, orderByLimit.MetricType) + case types.T_array_bf16: + distOf, err = topnDistOf[types.BF16](orderByLimit.NumVec, orderByLimit.MetricType) + case types.T_array_float16: + distOf, err = topnDistOf[types.Float16](orderByLimit.NumVec, orderByLimit.MetricType) + case types.T_array_int8: + distOf, err = topnDistOf[int8](orderByLimit.NumVec, orderByLimit.MetricType) + case types.T_array_uint8: + distOf, err = topnDistOf[uint8](orderByLimit.NumVec, orderByLimit.MetricType) + default: + return nil, nil, moerr.NewInternalError(ctx, fmt.Sprintf("only support float32/float64/bf16/float16/int8/uint8 type for topn: %s", orderByLimit.Typ)) + } + if err != nil { + return nil, nil, err + } + + for _, row := range selectRows { + dist64, err := distOf(vecCol.GetBytesAt(int(row))) if err != nil { return nil, nil, err } - rhs := types.BytesToArray[float32](orderByLimit.NumVec) - - for _, row := range selectRows { - dist, err := distFunc(types.BytesToArray[float32](vecCol.GetBytesAt(int(row))), rhs) - if err != nil { - return nil, nil, err - } - dist64 := float64(dist) - - if orderByLimit.LowerBoundType == plan.BoundType_INCLUSIVE { - if dist64 < orderByLimit.LowerBound { - continue - } - } else if orderByLimit.LowerBoundType == plan.BoundType_EXCLUSIVE { - if dist64 <= orderByLimit.LowerBound { - continue - } + if orderByLimit.LowerBoundType == plan.BoundType_INCLUSIVE { + if dist64 < orderByLimit.LowerBound { + continue } - if orderByLimit.UpperBoundType == plan.BoundType_INCLUSIVE { - if dist64 > orderByLimit.UpperBound { - continue - } - } else if orderByLimit.UpperBoundType == plan.BoundType_EXCLUSIVE { - if dist64 >= orderByLimit.UpperBound { - continue - } - } - - if len(orderByLimit.DistHeap) >= topLimit { - if dist64 < orderByLimit.DistHeap[0] { - orderByLimit.DistHeap[0] = dist64 - heap.Fix(&orderByLimit.DistHeap, 0) - } else { - continue - } - } else { - heap.Push(&orderByLimit.DistHeap, dist64) + } else if orderByLimit.LowerBoundType == plan.BoundType_EXCLUSIVE { + if dist64 <= orderByLimit.LowerBound { + continue } - - searchResults = append(searchResults, vectorindex.SearchResult{ - Id: row, - Distance: dist64, - }) } - - case types.T_array_float64: - distFunc, err := metric.ResolveDistanceFn[float64](orderByLimit.MetricType) - if err != nil { - return nil, nil, err - } - - rhs := types.BytesToArray[float64](orderByLimit.NumVec) - - for _, row := range selectRows { - dist64, err := distFunc(types.BytesToArray[float64](vecCol.GetBytesAt(int(row))), rhs) - if err != nil { - return nil, nil, err - } - - if orderByLimit.LowerBoundType == plan.BoundType_INCLUSIVE { - if dist64 < orderByLimit.LowerBound { - continue - } - } else if orderByLimit.LowerBoundType == plan.BoundType_EXCLUSIVE { - if dist64 <= orderByLimit.LowerBound { - continue - } + if orderByLimit.UpperBoundType == plan.BoundType_INCLUSIVE { + if dist64 > orderByLimit.UpperBound { + continue } - if orderByLimit.UpperBoundType == plan.BoundType_INCLUSIVE { - if dist64 > orderByLimit.UpperBound { - continue - } - } else if orderByLimit.UpperBoundType == plan.BoundType_EXCLUSIVE { - if dist64 >= orderByLimit.UpperBound { - continue - } + } else if orderByLimit.UpperBoundType == plan.BoundType_EXCLUSIVE { + if dist64 >= orderByLimit.UpperBound { + continue } + } - if len(orderByLimit.DistHeap) >= topLimit { - if dist64 < orderByLimit.DistHeap[0] { - orderByLimit.DistHeap[0] = dist64 - heap.Fix(&orderByLimit.DistHeap, 0) - } else { - continue - } + if len(orderByLimit.DistHeap) >= topLimit { + if dist64 < orderByLimit.DistHeap[0] { + orderByLimit.DistHeap[0] = dist64 + heap.Fix(&orderByLimit.DistHeap, 0) } else { - heap.Push(&orderByLimit.DistHeap, dist64) + continue } - - searchResults = append(searchResults, vectorindex.SearchResult{ - Id: row, - Distance: dist64, - }) + } else { + heap.Push(&orderByLimit.DistHeap, dist64) } - default: - return nil, nil, moerr.NewInternalError(ctx, fmt.Sprintf("only support float32/float64 type for topn: %s", orderByLimit.Typ)) + searchResults = append(searchResults, vectorindex.SearchResult{ + Id: row, + Distance: dist64, + }) } searchResults = slices.DeleteFunc(searchResults, func(res vectorindex.SearchResult) bool { diff --git a/pkg/vm/engine/tae/blockio/read_test.go b/pkg/vm/engine/tae/blockio/read_test.go index 26b286c2cc179..60102049d9c39 100644 --- a/pkg/vm/engine/tae/blockio/read_test.go +++ b/pkg/vm/engine/tae/blockio/read_test.go @@ -527,3 +527,65 @@ func TestHandleOrderByLimitOnLiveRowsForOrderedLimit(t *testing.T) { require.Nil(t, dists) require.Equal(t, []int64{1, 2, 3, 5}, rows) } + +// TestHandleOrderByLimitOnSelectRows_Narrow exercises the narrow (bf16/f16/int8) +// branch of the optimized vector top-k scan (the merged distOf path). Same data +// in each type: rows [10,10],[1,2],[5,5] vs query [0,0] -> dists 200,5,50, so the +// top-2 are row 1 then row 2. +func TestHandleOrderByLimitOnSelectRows_Narrow(t *testing.T) { + mp := mpool.MustNewZero() + defer mpool.DeleteMPool(mp) + ctx := context.Background() + + cases := []struct { + name string + oid types.T + rows [][]byte + num []byte + }{ + {"int8", types.T_array_int8, [][]byte{ + types.ArrayToBytes([]int8{10, 10}), + types.ArrayToBytes([]int8{1, 2}), + types.ArrayToBytes([]int8{5, 5}), + }, types.ArrayToBytes([]int8{0, 0})}, + {"bf16", types.T_array_bf16, [][]byte{ + types.ArrayToBytes(types.Float32ToBF16Slice([]float32{10, 10})), + types.ArrayToBytes(types.Float32ToBF16Slice([]float32{1, 2})), + types.ArrayToBytes(types.Float32ToBF16Slice([]float32{5, 5})), + }, types.ArrayToBytes(types.Float32ToBF16Slice([]float32{0, 0}))}, + {"f16", types.T_array_float16, [][]byte{ + types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{10, 10})), + types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{1, 2})), + types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{5, 5})), + }, types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{0, 0}))}, + } + + for _, c := range cases { + vec0 := vector.NewVec(types.T_int32.ToType()) + vec1 := vector.NewVec(c.oid.ToType()) + for i := 0; i < 3; i++ { + vector.AppendFixed(vec0, int32(i), false, mp) + } + for _, b := range c.rows { + vector.AppendBytes(vec1, b, false, mp) + } + cacheVectors := make(containers.Vectors, 2) + cacheVectors[0] = *vec0 + cacheVectors[1] = *vec1 + + orderByLimit := &objectio.IndexReaderTopOp{ + ColPos: 1, + Limit: 2, + Typ: c.oid, + NumVec: c.num, + MetricType: metric.Metric_L2Distance, + DistHeap: make(objectio.Float64Heap, 0, 2), + } + resSels, resDists, err := handleOrderByLimitOnSelectRows(ctx, []int64{0, 1, 2}, orderByLimit, nil, -1, cacheVectors) + require.NoErrorf(t, err, c.name) + require.Lenf(t, resSels, 2, c.name) + require.Lenf(t, resDists, 2, c.name) + require.Equalf(t, int64(1), resSels[0], "%s closest", c.name) + require.Equalf(t, int64(2), resSels[1], "%s next", c.name) + } +} diff --git a/pkg/vm/engine/tae/compute/compute.go b/pkg/vm/engine/tae/compute/compute.go index b23b3d0ab2bcd..f867281d49769 100644 --- a/pkg/vm/engine/tae/compute/compute.go +++ b/pkg/vm/engine/tae/compute/compute.go @@ -245,7 +245,8 @@ func GetOffsetByVal(data containers.Vector, v any, skipmask *nulls.Bitmap) (offs skipmask) case types.T_char, types.T_varchar, types.T_blob, types.T_binary, types.T_varbinary, types.T_json, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink: + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink: // data is retrieved from DN vector, hence T_array can be handled here. val := v.([]byte) start, end := 0, data.Length()-1 diff --git a/pkg/vm/engine/tae/compute/compute_test.go b/pkg/vm/engine/tae/compute/compute_test.go index 1dac3df5f7132..40f1492fe1da1 100644 --- a/pkg/vm/engine/tae/compute/compute_test.go +++ b/pkg/vm/engine/tae/compute/compute_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/RoaringBitmap/roaring/v2" + "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/nulls" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers" @@ -26,6 +27,28 @@ import ( "github.com/stretchr/testify/require" ) +// TestGetOffsetByValNarrowArray covers GetOffsetByVal binary search over a +// narrow vector array column (vecint8 here). The array case compares raw bytes; +// it previously listed only vecf32/vecf64, so a narrow array fell through. +func TestGetOffsetByValNarrowArray(t *testing.T) { + defer testutils.AfterTest(t)() + mp := mpool.MustNewZero() + typ := types.New(types.T_array_int8, 2, 0) + vec := containers.MakeVector(typ, mp) + defer vec.Close() + // byte-sorted rows so binary search is well-defined + vec.Append(types.ArrayToBytes([]int8{1, 1}), false) + vec.Append(types.ArrayToBytes([]int8{2, 2}), false) + vec.Append(types.ArrayToBytes([]int8{3, 3}), false) + + off, exist := GetOffsetByVal(vec, types.ArrayToBytes([]int8{2, 2}), nil) + require.True(t, exist) + require.Equal(t, 1, off) + + _, exist = GetOffsetByVal(vec, types.ArrayToBytes([]int8{9, 9}), nil) + require.False(t, exist) +} + func TestSortAndDedup(t *testing.T) { defer testutils.AfterTest(t)() vals := []int{2, 1, 3, 4, 5, 1, 2, 3, 4, 5} diff --git a/pkg/vm/engine/tae/containers/utils.go b/pkg/vm/engine/tae/containers/utils.go index c5f0eee22b7d4..85b7170eef685 100644 --- a/pkg/vm/engine/tae/containers/utils.go +++ b/pkg/vm/engine/tae/containers/utils.go @@ -485,7 +485,8 @@ func getNonNullValue(col *vector.Vector, row uint32) any { case types.T_Blockid: return vector.GetFixedAtNoTypeCheck[types.Blockid](col, int(row)) case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_json, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink: + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink: return col.GetBytesAt(int(row)) default: //return vector.ErrVecTypeNotSupport @@ -577,7 +578,8 @@ func UpdateValue(col *vector.Vector, row uint32, val any, isNull bool, mp *mpool GenericUpdateFixedValue[types.Blockid](col, row, val, isNull, mp) case types.T_varchar, types.T_char, types.T_json, types.T_binary, types.T_varbinary, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink: + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink: GenericUpdateBytes(col, row, val, isNull, mp) default: panic(moerr.NewInternalErrorNoCtxf("%v not supported", col.GetType())) diff --git a/pkg/vm/engine/tae/containers/utils_test.go b/pkg/vm/engine/tae/containers/utils_test.go index 85f160e18a837..6df8db50f851b 100644 --- a/pkg/vm/engine/tae/containers/utils_test.go +++ b/pkg/vm/engine/tae/containers/utils_test.go @@ -106,6 +106,46 @@ func TestGeneralBatchBuffer1(t *testing.T) { require.Equal(t, int64(0), mp.CurrNB()) } +// TestNarrowArrayValue covers getNonNullValue + UpdateValue for the narrow +// vector array types (vecbf16/vecf16/vecint8/vecuint8). Both switch on the +// element type and previously listed only vecf32/vecf64, so a narrow array +// panicked ("No Support" / "not supported"). All array types are varlen byte +// storage, so the round-trip is read/update as raw bytes. +func TestNarrowArrayValue(t *testing.T) { + mp := mpool.MustNewZero() + + cases := []struct { + oid types.T + initial []byte + updated []byte + }{ + {types.T_array_float16, + types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{1, 2, 3})), + types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{4, 5, 6}))}, + {types.T_array_bf16, + types.ArrayToBytes(types.Float32ToBF16Slice([]float32{1, 2, 3})), + types.ArrayToBytes(types.Float32ToBF16Slice([]float32{4, 5, 6}))}, + {types.T_array_int8, + types.ArrayToBytes([]int8{1, 2, 3}), types.ArrayToBytes([]int8{4, 5, 6})}, + {types.T_array_uint8, + types.ArrayToBytes([]uint8{1, 2, 3}), types.ArrayToBytes([]uint8{4, 5, 6})}, + } + + for _, c := range cases { + vec := vector.NewVec(types.New(c.oid, 3, 0)) + require.NoError(t, vector.AppendBytes(vec, c.initial, false, mp)) + + // read back (was panic "No Support") + require.Equal(t, c.initial, getNonNullValue(vec, 0).([]byte), c.oid.String()) + + // update in place (was panic "not supported"), then read back the new value + UpdateValue(vec, 0, c.updated, false, mp) + require.Equal(t, c.updated, getNonNullValue(vec, 0).([]byte), c.oid.String()) + + vec.Free(mp) + } +} + func TestVectorsCopyToBatch(t *testing.T) { var vecs Vectors require.NoError(t, VectorsCopyToBatch(vecs, nil, nil)) diff --git a/pkg/vm/engine/tae/txn/txnimpl/index.go b/pkg/vm/engine/tae/txn/txnimpl/index.go index 5b7f8e5233ac4..e524975c2e4c0 100644 --- a/pkg/vm/engine/tae/txn/txnimpl/index.go +++ b/pkg/vm/engine/tae/txn/txnimpl/index.go @@ -108,7 +108,8 @@ func (idx *simpleTableIndex) KeyToVector(kType types.Type) containers.Vector { for k := range idx.tree { vec.Append([]byte(k.(string)), false) } - case types.T_array_float32, types.T_array_float64: + case types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: // No usage for this func. for k := range idx.tree { vec.Append(k.([]byte), false) diff --git a/proto/plan.proto b/proto/plan.proto index 18963a59cb7e6..b2f956432f61b 100644 --- a/proto/plan.proto +++ b/proto/plan.proto @@ -1414,6 +1414,7 @@ message AlterTableAlterReIndex { string index_name = 3; int64 index_algo_param_list = 4; bool force_sync = 5; + bool merge = 6; } message AlterTableAlterAutoUpdate { diff --git a/test/distributed/cases/array/array_vecnarrow.result b/test/distributed/cases/array/array_vecnarrow.result new file mode 100644 index 0000000000000..5e94664b8269a --- /dev/null +++ b/test/distributed/cases/array/array_vecnarrow.result @@ -0,0 +1,164 @@ +drop database if exists vecnarrowdb; +create database vecnarrowdb; +use vecnarrowdb; +drop table if exists nvec; +create table nvec(a int, bf vecbf16(3), f16 vecf16(3), i8 vecint8(3)); +desc nvec; +Field Type Null Key Default Extra Comment +a INT(32) YES null +bf VECBF16(3) YES null +f16 VECF16(3) YES null +i8 VECINT8(3) YES null +show create table nvec; +Table Create Table +nvec CREATE TABLE `nvec` (\n `a` int DEFAULT NULL,\n `bf` vecbf16(3) DEFAULT NULL,\n `f16` vecf16(3) DEFAULT NULL,\n `i8` vecint8(3) DEFAULT NULL\n) +insert into nvec values(1, "[1,2,3]", "[1,2,3]", "[1,2,3]"); +insert into nvec values(2, "[4,5,6]", "[4,5,6]", "[4,5,6]"); +select * from nvec; +a bf f16 i8 +1 [1, 2, 3] [1, 2, 3] [1, 2, 3] +2 [4, 5, 6] [4, 5, 6] [4, 5, 6] +drop table if exists i8t; +create table i8t(a int, v vecint8(4)); +insert into i8t values(1, "[127,-128,0,5]"); +select * from i8t; +a v +1 [127, -128, 0, 5] +insert into i8t values(2, "[200,-200,0,0]"); +internal error: error while casting 200 to VECINT8 +insert into i8t values(3, "[1.4,2.6,0,0]"); +internal error: error while casting 1.4 to VECINT8 +select cast(cast("[1.6,200,-3.5,-200]" as vecf32(4)) as vecint8(4)); +cast(cast([1.6,200,-3.5,-200] as vecf32(4)) as vecint8(4)) +[2, 127, -4, -128] +select cast("[1,2,3]" as vecbf16(3)); +cast([1,2,3] as vecbf16(3)) +[1, 2, 3] +select cast("[1,2,3]" as vecf16(3)); +cast([1,2,3] as vecf16(3)) +[1, 2, 3] +select cast("[1,2,3]" as vecint8(3)); +cast([1,2,3] as vecint8(3)) +[1, 2, 3] +select cast("[1.4,2.6,-3.5]" as vecint8(3)); +internal error: error while casting 1.4 to VECINT8 +select cast(bf as vecf32(3)), cast(f16 as vecf32(3)), cast(i8 as vecf32(3)) from nvec order by a; +cast(bf as vecf32(3)) cast(f16 as vecf32(3)) cast(i8 as vecf32(3)) +[1, 2, 3] [1, 2, 3] [1, 2, 3] +[4, 5, 6] [4, 5, 6] [4, 5, 6] +select cast(bf as vecf64(3)) from nvec order by a; +cast(bf as vecf64(3)) +[1, 2, 3] +[4, 5, 6] +select cast(cast("[1,2,3]" as vecf32(3)) as vecbf16(3)); +cast(cast([1,2,3] as vecf32(3)) as vecbf16(3)) +[1, 2, 3] +select cast(cast("[1,2,3]" as vecf32(3)) as vecf16(3)); +cast(cast([1,2,3] as vecf32(3)) as vecf16(3)) +[1, 2, 3] +select cast(cast("[1.6,2.4,-3.5]" as vecf32(3)) as vecint8(3)); +cast(cast([1.6,2.4,-3.5] as vecf32(3)) as vecint8(3)) +[2, 2, -4] +select cast(bf as vecf16(3)), cast(f16 as vecint8(3)), cast(i8 as vecbf16(3)) from nvec order by a; +cast(bf as vecf16(3)) cast(f16 as vecint8(3)) cast(i8 as vecbf16(3)) +[1, 2, 3] [1, 2, 3] [1, 2, 3] +[4, 5, 6] [4, 5, 6] [4, 5, 6] +select l2_distance(bf, "[1,2,3]") from nvec order by a; +l2_distance(bf, [1,2,3]) +0.0 +5.196152210235596 +select l2_distance_sq(bf, "[1,2,3]") from nvec order by a; +l2_distance_sq(bf, [1,2,3]) +0.0 +27.0 +select inner_product(bf, "[1,2,3]") from nvec order by a; +inner_product(bf, [1,2,3]) +-14.0 +-32.0 +select cosine_distance(bf, "[1,2,3]") from nvec order by a; +cosine_distance(bf, [1,2,3]) +0.0 +0.025368154048919678 +select cosine_similarity(bf, "[1,2,3]") from nvec order by a; +cosine_similarity(bf, [1,2,3]) +1.0 +0.9746318459510803 +select normalize_l2(bf) from nvec order by a; +normalize_l2(bf) +[0.26757812, 0.53515625, 0.80078125] +[0.45507812, 0.5703125, 0.68359375] +select l2_distance(f16, "[1,2,3]") from nvec order by a; +l2_distance(f16, [1,2,3]) +0.0 +5.196152210235596 +select inner_product(f16, "[1,2,3]") from nvec order by a; +inner_product(f16, [1,2,3]) +-14.0 +-32.0 +select cosine_distance(f16, "[1,2,3]") from nvec order by a; +cosine_distance(f16, [1,2,3]) +0.0 +0.025368154048919678 +select normalize_l2(f16) from nvec order by a; +normalize_l2(f16) +[0.26733398, 0.53466797, 0.8017578] +[0.45581055, 0.5698242, 0.68359375] +select l2_distance(i8, "[1,2,3]") from nvec order by a; +l2_distance(i8, [1,2,3]) +0.0 +5.196152210235596 +select inner_product(i8, "[1,2,3]") from nvec order by a; +inner_product(i8, [1,2,3]) +-14.0 +-32.0 +select cosine_distance(i8, "[1,2,3]") from nvec order by a; +cosine_distance(i8, [1,2,3]) +0.0 +0.025368154048919678 +select l2_distance(bf, cast("[4,5,6]" as vecbf16(3))) from nvec order by a; +l2_distance(bf, cast([4,5,6] as vecbf16(3))) +5.196152210235596 +0.0 +select l2_distance(i8, cast("[4,5,6]" as vecint8(3))) from nvec order by a; +l2_distance(i8, cast([4,5,6] as vecint8(3))) +5.196152210235596 +0.0 +select a FROM nvec ORDER BY l2_distance(bf, '[1,2,3]') LIMIT 5; +a +1 +2 +select a FROM nvec ORDER BY cosine_distance(f16, '[1,2,3]') LIMIT 5; +a +1 +2 +select a FROM nvec ORDER BY inner_product(i8, '[1,2,3]') LIMIT 5; +a +2 +1 +select * from nvec where i8 = "[1,2,3]"; +a bf f16 i8 +1 [1, 2, 3] [1, 2, 3] [1, 2, 3] +select * from nvec order by bf desc; +a bf f16 i8 +2 [4, 5, 6] [4, 5, 6] [4, 5, 6] +1 [1, 2, 3] [1, 2, 3] [1, 2, 3] +select distinct v from i8t order by v; +v +[127, -128, 0, 5] +select bf + bf from nvec; +invalid argument operator +, bad value [VECBF16 VECBF16] +select bf - bf from nvec; +invalid argument operator -, bad value [VECBF16 VECBF16] +select bf * bf from nvec; +invalid argument operator *, bad value [VECBF16 VECBF16] +select sqrt(bf) from nvec; +invalid argument function sqrt, bad value [VECBF16] +select abs(i8) from nvec; +invalid argument function abs, bad value [VECINT8] +select summation(f16) from nvec; +invalid argument function summation, bad value [VECF16] +select cast(bf as vecf32(3)) + cast(bf as vecf32(3)) from nvec order by a; +cast(bf as vecf32(3)) + cast(bf as vecf32(3)) +[2, 4, 6] +[8, 10, 12] +drop database if exists vecnarrowdb; diff --git a/test/distributed/cases/array/array_vecnarrow.sql b/test/distributed/cases/array/array_vecnarrow.sql new file mode 100644 index 0000000000000..e9178b55ad1b2 --- /dev/null +++ b/test/distributed/cases/array/array_vecnarrow.sql @@ -0,0 +1,94 @@ +-- vecbf16 / vecf16 / vecint8 narrow vector column types +-- Scope (per design): distance functions + casts + storage only. +-- Elementwise arithmetic (+ - * / sqrt abs summation subvector) is NOT +-- supported on the narrow types and must require an explicit CAST to vecf32. + +-- pre +drop database if exists vecnarrowdb; +create database vecnarrowdb; +use vecnarrowdb; +drop table if exists nvec; + +-- standard: one column of each new type +create table nvec(a int, bf vecbf16(3), f16 vecf16(3), i8 vecint8(3)); +desc nvec; +show create table nvec; +insert into nvec values(1, "[1,2,3]", "[1,2,3]", "[1,2,3]"); +insert into nvec values(2, "[4,5,6]", "[4,5,6]", "[4,5,6]"); +select * from nvec; + +-- int8: a string literal must be an integer in [-128,127]; boundary values OK. +drop table if exists i8t; +create table i8t(a int, v vecint8(4)); +insert into i8t values(1, "[127,-128,0,5]"); +select * from i8t; +-- out-of-range and non-integer string literals error (no silent round/clamp) +insert into i8t values(2, "[200,-200,0,0]"); +insert into i8t values(3, "[1.4,2.6,0,0]"); +-- rounding/clamping IS available, but only via the vecf32 -> vecint8 CAST path +select cast(cast("[1.6,200,-3.5,-200]" as vecf32(4)) as vecint8(4)); + +-- string -> narrow casts +select cast("[1,2,3]" as vecbf16(3)); +select cast("[1,2,3]" as vecf16(3)); +select cast("[1,2,3]" as vecint8(3)); +-- non-integer string literal -> vecint8 errors (strict) +select cast("[1.4,2.6,-3.5]" as vecint8(3)); + +-- narrow -> vecf32 / vecf64 casts (the explicit widening path) +select cast(bf as vecf32(3)), cast(f16 as vecf32(3)), cast(i8 as vecf32(3)) from nvec order by a; +select cast(bf as vecf64(3)) from nvec order by a; + +-- vecf32 -> narrow casts +select cast(cast("[1,2,3]" as vecf32(3)) as vecbf16(3)); +select cast(cast("[1,2,3]" as vecf32(3)) as vecf16(3)); +select cast(cast("[1.6,2.4,-3.5]" as vecf32(3)) as vecint8(3)); + +-- narrow -> narrow casts +select cast(bf as vecf16(3)), cast(f16 as vecint8(3)), cast(i8 as vecbf16(3)) from nvec order by a; + +-- distance functions on bf16 +select l2_distance(bf, "[1,2,3]") from nvec order by a; +select l2_distance_sq(bf, "[1,2,3]") from nvec order by a; +select inner_product(bf, "[1,2,3]") from nvec order by a; +select cosine_distance(bf, "[1,2,3]") from nvec order by a; +select cosine_similarity(bf, "[1,2,3]") from nvec order by a; +select normalize_l2(bf) from nvec order by a; + +-- distance functions on f16 +select l2_distance(f16, "[1,2,3]") from nvec order by a; +select inner_product(f16, "[1,2,3]") from nvec order by a; +select cosine_distance(f16, "[1,2,3]") from nvec order by a; +select normalize_l2(f16) from nvec order by a; + +-- distance functions on int8 +select l2_distance(i8, "[1,2,3]") from nvec order by a; +select inner_product(i8, "[1,2,3]") from nvec order by a; +select cosine_distance(i8, "[1,2,3]") from nvec order by a; + +-- distance between two narrow columns of the same type +select l2_distance(bf, cast("[4,5,6]" as vecbf16(3))) from nvec order by a; +select l2_distance(i8, cast("[4,5,6]" as vecint8(3))) from nvec order by a; + +-- top-K (ORDER BY distance + LIMIT) +select a FROM nvec ORDER BY l2_distance(bf, '[1,2,3]') LIMIT 5; +select a FROM nvec ORDER BY cosine_distance(f16, '[1,2,3]') LIMIT 5; +select a FROM nvec ORDER BY inner_product(i8, '[1,2,3]') LIMIT 5; + +-- filtering / equality / ordering +select * from nvec where i8 = "[1,2,3]"; +select * from nvec order by bf desc; +select distinct v from i8t order by v; + +-- negative: arithmetic is not allowed on narrow types (must CAST to vecf32 first) +select bf + bf from nvec; +select bf - bf from nvec; +select bf * bf from nvec; +select sqrt(bf) from nvec; +select abs(i8) from nvec; +select summation(f16) from nvec; +-- arithmetic IS allowed after an explicit cast to vecf32 +select cast(bf as vecf32(3)) + cast(bf as vecf32(3)) from nvec order by a; + +-- post +drop database if exists vecnarrowdb; diff --git a/test/distributed/cases/array/array_vecnarrow_dims.result b/test/distributed/cases/array/array_vecnarrow_dims.result new file mode 100644 index 0000000000000..2267a5fab3b64 --- /dev/null +++ b/test/distributed/cases/array/array_vecnarrow_dims.result @@ -0,0 +1,17 @@ +drop database if exists nvdims; +create database nvdims; +use nvdims; +create table t(a int, bf vecbf16(3), hf vecf16(5), i8 vecint8(4), u8 vecuint8(2)); +insert into t values(1, '[1,2,3]', '[1,2,3,4,5]', '[1,2,3,4]', '[1,2]'); +insert into t values(2, '[4,5,6]', '[6,7,8,9,10]', '[5,6,7,8]', '[9,8]'); +select a, vector_dims(bf) as bf, vector_dims(hf) as hf, vector_dims(i8) as i8, vector_dims(u8) as u8 from t order by a; +a bf hf i8 u8 +1 3 5 4 2 +2 3 5 4 2 +insert into t values(3, null, null, null, null); +select a, vector_dims(bf) as bf, vector_dims(u8) as u8 from t order by a; +a bf u8 +1 3 2 +2 3 2 +3 null null +drop database nvdims; diff --git a/test/distributed/cases/array/array_vecnarrow_dims.sql b/test/distributed/cases/array/array_vecnarrow_dims.sql new file mode 100644 index 0000000000000..e4937f8370a1d --- /dev/null +++ b/test/distributed/cases/array/array_vecnarrow_dims.sql @@ -0,0 +1,15 @@ +-- vector_dims on the narrow vector types (vecbf16/vecf16/vecint8/vecuint8). +-- Regression: vector_dims previously only had float32/float64 overloads and +-- errored on narrow types ("invalid argument function vector_dims, bad value +-- [VECINT8]"). It now returns the element count (= content bytes / sizeof(elem)), +-- like vecf32; a NULL vector yields NULL dims, matching vecf32 behavior. +drop database if exists nvdims; +create database nvdims; +use nvdims; +create table t(a int, bf vecbf16(3), hf vecf16(5), i8 vecint8(4), u8 vecuint8(2)); +insert into t values(1, '[1,2,3]', '[1,2,3,4,5]', '[1,2,3,4]', '[1,2]'); +insert into t values(2, '[4,5,6]', '[6,7,8,9,10]', '[5,6,7,8]', '[9,8]'); +select a, vector_dims(bf) as bf, vector_dims(hf) as hf, vector_dims(i8) as i8, vector_dims(u8) as u8 from t order by a; +insert into t values(3, null, null, null, null); +select a, vector_dims(bf) as bf, vector_dims(u8) as u8 from t order by a; +drop database nvdims; diff --git a/test/distributed/cases/array/array_vecnarrow_ops.result b/test/distributed/cases/array/array_vecnarrow_ops.result new file mode 100644 index 0000000000000..395ac6bb9ca27 --- /dev/null +++ b/test/distributed/cases/array/array_vecnarrow_ops.result @@ -0,0 +1,98 @@ +drop database if exists nvops; +create database nvops; +use nvops; +create table b(a int, v vecbf16(3)); +insert into b values (1,'[1,2,3]'),(2,'[4,5,6]'),(3,'[1,2,3]'),(4,'[7,8,9]'); +select a, v = cast('[1,2,3]' as vecbf16(3)) as eq, v != cast('[1,2,3]' as vecbf16(3)) as ne, v < cast('[4,5,6]' as vecbf16(3)) as lt, v > cast('[1,2,3]' as vecbf16(3)) as gt, v <= cast('[1,2,3]' as vecbf16(3)) as le, v >= cast('[4,5,6]' as vecbf16(3)) as ge from b order by a; +a eq ne lt gt le ge +1 1 0 1 0 1 0 +2 0 1 0 1 0 1 +3 1 0 1 0 1 0 +4 0 1 0 1 0 1 +select v from b order by v, a; +v +[1, 2, 3] +[1, 2, 3] +[4, 5, 6] +[7, 8, 9] +select distinct v from b order by v; +v +[1, 2, 3] +[4, 5, 6] +[7, 8, 9] +select v, count(*) as c from b group by v order by v; +v c +[1, 2, 3] 2 +[4, 5, 6] 1 +[7, 8, 9] 1 +select a from b where v = cast('[1,2,3]' as vecbf16(3)) order by a; +a +1 +3 +select a from b where v < cast('[4,5,6]' as vecbf16(3)) order by a; +a +1 +3 +select a, round(l2_distance(v, cast('[1,2,3]' as vecbf16(3))),4) as l2, round(l2_distance_sq(v, cast('[1,2,3]' as vecbf16(3))),4) as l2sq, round(inner_product(v, cast('[1,2,3]' as vecbf16(3))),4) as ip, round(cosine_distance(v, cast('[1,2,3]' as vecbf16(3))),4) as cd, round(cosine_similarity(v, cast('[1,2,3]' as vecbf16(3))),4) as cs from b order by a; +a l2 l2sq ip cd cs +1 0.0 0.0 -14.0 0.0 1.0 +2 5.1962 27.0 -32.0 0.0254 0.9746 +3 0.0 0.0 -14.0 0.0 1.0 +4 10.3923 108.0 -50.0 0.0406 0.9594 +select v + v from b; +invalid argument operator +, bad value [VECBF16 VECBF16] +select v - v from b; +invalid argument operator -, bad value [VECBF16 VECBF16] +select v * 2 from b; +invalid argument operator *, bad value [VECBF16 BIGINT] +select a, cast(v as vecf32(3)) + cast('[1,1,1]' as vecf32(3)) as r from b order by a; +a r +1 [2, 3, 4] +2 [5, 6, 7] +3 [2, 3, 4] +4 [8, 9, 10] +create table f(a int, v vecf16(3)); +insert into f values (1,'[1,2,3]'),(2,'[4,5,6]'),(3,'[1,2,3]'); +select a, v = cast('[1,2,3]' as vecf16(3)) as eq, v < cast('[4,5,6]' as vecf16(3)) as lt from f order by a; +a eq lt +1 1 1 +2 0 0 +3 1 1 +select distinct v from f order by v; +v +[1, 2, 3] +[4, 5, 6] +select v, count(*) as c from f group by v order by v; +v c +[1, 2, 3] 2 +[4, 5, 6] 1 +select a, round(l2_distance_sq(v, cast('[1,2,3]' as vecf16(3))),4) as l2sq, round(inner_product(v, cast('[1,2,3]' as vecf16(3))),4) as ip from f order by a; +a l2sq ip +1 0.0 -14.0 +2 27.0 -32.0 +3 0.0 -14.0 +select v * 2 from f; +invalid argument operator *, bad value [VECF16 BIGINT] +create table i(a int, v vecint8(3)); +insert into i values (1,'[1,2,3]'),(2,'[4,5,6]'),(3,'[1,2,3]'); +select a, v = cast('[1,2,3]' as vecint8(3)) as eq, v < cast('[4,5,6]' as vecint8(3)) as lt from i order by a; +a eq lt +1 1 1 +2 0 0 +3 1 1 +select distinct v from i order by v; +v +[1, 2, 3] +[4, 5, 6] +select v, count(*) as c from i group by v order by v; +v c +[1, 2, 3] 2 +[4, 5, 6] 1 +select a, round(l2_distance_sq(v, cast('[1,2,3]' as vecint8(3))),4) as l2sq, round(inner_product(v, cast('[1,2,3]' as vecint8(3))),4) as ip from i order by a; +a l2sq ip +1 0.0 -14.0 +2 27.0 -32.0 +3 0.0 -14.0 +select v + v from i; +invalid argument operator +, bad value [VECINT8 VECINT8] +drop database nvops; diff --git a/test/distributed/cases/array/array_vecnarrow_ops.sql b/test/distributed/cases/array/array_vecnarrow_ops.sql new file mode 100644 index 0000000000000..fb726482e536e --- /dev/null +++ b/test/distributed/cases/array/array_vecnarrow_ops.sql @@ -0,0 +1,48 @@ +-- narrow vector types (vecbf16/vecf16/vecint8): comparison operators, ordering/ +-- grouping/aggregates, distance functions, and the arithmetic rule (elementwise +-- arithmetic errors -- must CAST to vecf32 first). Small integers are exact in all +-- three narrow formats, so results are deterministic. +drop database if exists nvops; +create database nvops; +use nvops; + +-- ===== vecbf16 ===== +create table b(a int, v vecbf16(3)); +insert into b values (1,'[1,2,3]'),(2,'[4,5,6]'),(3,'[1,2,3]'),(4,'[7,8,9]'); +-- comparison operators +select a, v = cast('[1,2,3]' as vecbf16(3)) as eq, v != cast('[1,2,3]' as vecbf16(3)) as ne, v < cast('[4,5,6]' as vecbf16(3)) as lt, v > cast('[1,2,3]' as vecbf16(3)) as gt, v <= cast('[1,2,3]' as vecbf16(3)) as le, v >= cast('[4,5,6]' as vecbf16(3)) as ge from b order by a; +-- ordering / distinct / group by / aggregates (all use comparison) +select v from b order by v, a; +select distinct v from b order by v; +select v, count(*) as c from b group by v order by v; +-- where filter by comparison +select a from b where v = cast('[1,2,3]' as vecbf16(3)) order by a; +select a from b where v < cast('[4,5,6]' as vecbf16(3)) order by a; +-- distance functions +select a, round(l2_distance(v, cast('[1,2,3]' as vecbf16(3))),4) as l2, round(l2_distance_sq(v, cast('[1,2,3]' as vecbf16(3))),4) as l2sq, round(inner_product(v, cast('[1,2,3]' as vecbf16(3))),4) as ip, round(cosine_distance(v, cast('[1,2,3]' as vecbf16(3))),4) as cd, round(cosine_similarity(v, cast('[1,2,3]' as vecbf16(3))),4) as cs from b order by a; +-- arithmetic is NOT supported on narrow types directly +select v + v from b; +select v - v from b; +select v * 2 from b; +-- ... but works after an explicit CAST to vecf32 +select a, cast(v as vecf32(3)) + cast('[1,1,1]' as vecf32(3)) as r from b order by a; + +-- ===== vecf16 ===== +create table f(a int, v vecf16(3)); +insert into f values (1,'[1,2,3]'),(2,'[4,5,6]'),(3,'[1,2,3]'); +select a, v = cast('[1,2,3]' as vecf16(3)) as eq, v < cast('[4,5,6]' as vecf16(3)) as lt from f order by a; +select distinct v from f order by v; +select v, count(*) as c from f group by v order by v; +select a, round(l2_distance_sq(v, cast('[1,2,3]' as vecf16(3))),4) as l2sq, round(inner_product(v, cast('[1,2,3]' as vecf16(3))),4) as ip from f order by a; +select v * 2 from f; + +-- ===== vecint8 ===== +create table i(a int, v vecint8(3)); +insert into i values (1,'[1,2,3]'),(2,'[4,5,6]'),(3,'[1,2,3]'); +select a, v = cast('[1,2,3]' as vecint8(3)) as eq, v < cast('[4,5,6]' as vecint8(3)) as lt from i order by a; +select distinct v from i order by v; +select v, count(*) as c from i group by v order by v; +select a, round(l2_distance_sq(v, cast('[1,2,3]' as vecint8(3))),4) as l2sq, round(inner_product(v, cast('[1,2,3]' as vecint8(3))),4) as ip from i order by a; +select v + v from i; + +drop database nvops; diff --git a/test/distributed/cases/array/array_vecuint8.result b/test/distributed/cases/array/array_vecuint8.result new file mode 100644 index 0000000000000..bc207bbbe3aba --- /dev/null +++ b/test/distributed/cases/array/array_vecuint8.result @@ -0,0 +1,125 @@ +drop database if exists vecu8db; +create database vecu8db; +use vecu8db; +create table u8t(a int, v vecuint8(4)); +desc u8t; +Field Type Null Key Default Extra Comment +a INT(32) YES null +v VECUINT8(4) YES null +show create table u8t; +Table Create Table +u8t CREATE TABLE `u8t` (\n `a` int DEFAULT NULL,\n `v` vecuint8(4) DEFAULT NULL\n) +insert into u8t values(1, "[0,1,2,3]"); +insert into u8t values(2, "[255,254,0,128]"); +insert into u8t values(3, "[10,20,30,40]"); +select * from u8t order by a; +a v +1 [0, 1, 2, 3] +2 [255, 254, 0, 128] +3 [10, 20, 30, 40] +insert into u8t values(4, "[300,0,0,0]"); +internal error: error while casting 300 to VECUINT8 +insert into u8t values(5, "[-1,0,0,0]"); +internal error: error while casting -1 to VECUINT8 +insert into u8t values(6, "[1.4,0,0,0]"); +internal error: error while casting 1.4 to VECUINT8 +select cast(cast("[1.6,300,-5,200]" as vecf32(4)) as vecuint8(4)); +cast(cast([1.6,300,-5,200] as vecf32(4)) as vecuint8(4)) +[2, 255, 0, 200] +select cast("[1,2,3]" as vecuint8(3)); +cast([1,2,3] as vecuint8(3)) +[1, 2, 3] +select cast("[1.4,2.6,-3.5]" as vecuint8(3)); +internal error: error while casting 1.4 to VECUINT8 +select cast(v as vecf32(4)), cast(v as vecf64(4)) from u8t order by a; +cast(v as vecf32(4)) cast(v as vecf64(4)) +[0, 1, 2, 3] [0, 1, 2, 3] +[255, 254, 0, 128] [255, 254, 0, 128] +[10, 20, 30, 40] [10, 20, 30, 40] +select cast(cast("[1,2,3]" as vecf32(3)) as vecuint8(3)); +cast(cast([1,2,3] as vecf32(3)) as vecuint8(3)) +[1, 2, 3] +select cast(cast("[1,2,3]" as vecuint8(3)) as vecint8(3)); +cast(cast([1,2,3] as vecuint8(3)) as vecint8(3)) +[1, 2, 3] +select cast(cast("[1,2,3]" as vecuint8(3)) as vecbf16(3)); +cast(cast([1,2,3] as vecuint8(3)) as vecbf16(3)) +[1, 2, 3] +select a, round(l2_distance(v, "[0,1,2,3]"), 4) from u8t order by a; +a round(l2_distance(v, [0,1,2,3]), 4) +1 0.0 +2 380.3459 +3 51.1273 +select a, l2_distance_sq(v, "[0,1,2,3]") from u8t order by a; +a l2_distance_sq(v, [0,1,2,3]) +1 0.0 +2 144663.0 +3 2614.0 +select a, inner_product(v, "[1,1,1,1]") from u8t order by a; +a inner_product(v, [1,1,1,1]) +1 -6.0 +2 -637.0 +3 -100.0 +select a, round(cosine_distance(v, "[0,1,2,3]"), 4) from u8t order by a; +a round(cosine_distance(v, [0,1,2,3]), 4) +1 0.0 +2 0.5536 +3 0.0241 +select a, round(cosine_similarity(v, "[0,1,2,3]"), 4) from u8t order by a; +a round(cosine_similarity(v, [0,1,2,3]), 4) +1 1.0 +2 0.4464 +3 0.9759 +select normalize_l2(v) from u8t order by a; +normalize_l2(v) +[0, 0, 1, 1] +[1, 1, 0, 0] +[0, 0, 1, 1] +select a, round(l2_distance(v, cast("[10,20,30,40]" as vecuint8(4))), 4) from u8t order by a; +a round(l2_distance(v, cast([10,20,30,40] as vecuint8(4))), 4) +1 51.1273 +2 351.3189 +3 0.0 +select a from u8t order by l2_distance(v, '[0,1,2,3]') limit 3; +a +1 +3 +2 +select a from u8t order by inner_product(v, '[1,1,1,1]') limit 3; +a +2 +3 +1 +select a from u8t where v = "[10,20,30,40]"; +a +3 +select * from u8t order by v desc; +a v +2 [255, 254, 0, 128] +3 [10, 20, 30, 40] +1 [0, 1, 2, 3] +select distinct v from u8t order by v; +v +[0, 1, 2, 3] +[10, 20, 30, 40] +[255, 254, 0, 128] +select v + v from u8t; +invalid argument operator +, bad value [VECUINT8 VECUINT8] +select v * v from u8t; +invalid argument operator *, bad value [VECUINT8 VECUINT8] +select abs(v) from u8t; +invalid argument function abs, bad value [VECUINT8] +select summation(v) from u8t; +invalid argument function summation, bad value [VECUINT8] +select cast(v as vecf32(4)) + cast(v as vecf32(4)) from u8t order by a; +cast(v as vecf32(4)) + cast(v as vecf32(4)) +[0, 2, 4, 6] +[510, 508, 0, 256] +[20, 40, 60, 80] +select vecuint8_from_base64('ChQeKA=='); +vecuint8_from_base64(ChQeKA==) +[10, 20, 30, 40] +select vecuint8_from_base64('AP+AAQ=='); +vecuint8_from_base64(AP+AAQ==) +[0, 255, 128, 1] +drop database if exists vecu8db; diff --git a/test/distributed/cases/array/array_vecuint8.sql b/test/distributed/cases/array/array_vecuint8.sql new file mode 100644 index 0000000000000..3caba84a11999 --- /dev/null +++ b/test/distributed/cases/array/array_vecuint8.sql @@ -0,0 +1,78 @@ +-- vecuint8 narrow vector column type (unsigned 8-bit, [0,255]). +-- Mirrors array_vecnarrow for the int8 sibling; scope (per design): distance +-- functions + casts + storage only. Elementwise arithmetic is NOT supported and +-- must go through an explicit CAST to vecf32. + +drop database if exists vecu8db; +create database vecu8db; +use vecu8db; + +-- column type: create / desc / show create / insert / select +create table u8t(a int, v vecuint8(4)); +desc u8t; +show create table u8t; +insert into u8t values(1, "[0,1,2,3]"); +insert into u8t values(2, "[255,254,0,128]"); +insert into u8t values(3, "[10,20,30,40]"); +select * from u8t order by a; + +-- strict string parse: an integer in [0,255]; boundary values OK. +-- out-of-range and non-integer literals error (no silent round/clamp). +insert into u8t values(4, "[300,0,0,0]"); +insert into u8t values(5, "[-1,0,0,0]"); +insert into u8t values(6, "[1.4,0,0,0]"); + +-- rounding/clamping IS available, but only via the vecf32 -> vecuint8 CAST path +select cast(cast("[1.6,300,-5,200]" as vecf32(4)) as vecuint8(4)); + +-- string -> vecuint8 cast (strict) +select cast("[1,2,3]" as vecuint8(3)); +select cast("[1.4,2.6,-3.5]" as vecuint8(3)); + +-- vecuint8 -> vecf32 / vecf64 (explicit widening) +select cast(v as vecf32(4)), cast(v as vecf64(4)) from u8t order by a; + +-- vecuint8 <-> other narrow casts +select cast(cast("[1,2,3]" as vecf32(3)) as vecuint8(3)); +select cast(cast("[1,2,3]" as vecuint8(3)) as vecint8(3)); +select cast(cast("[1,2,3]" as vecuint8(3)) as vecbf16(3)); + +-- distance functions on vecuint8 +-- l2_distance / cosine_* involve sqrt/division and are computed in float64; +-- round to 4 digits so the low-order bits don't diverge across CPU SIMD kernels. +-- l2_distance_sq / inner_product are integer-exact for uint8 (no rounding). +select a, round(l2_distance(v, "[0,1,2,3]"), 4) from u8t order by a; +select a, l2_distance_sq(v, "[0,1,2,3]") from u8t order by a; +select a, inner_product(v, "[1,1,1,1]") from u8t order by a; +select a, round(cosine_distance(v, "[0,1,2,3]"), 4) from u8t order by a; +select a, round(cosine_similarity(v, "[0,1,2,3]"), 4) from u8t order by a; +select normalize_l2(v) from u8t order by a; + +-- distance between two vecuint8 values +select a, round(l2_distance(v, cast("[10,20,30,40]" as vecuint8(4))), 4) from u8t order by a; + +-- top-K (ORDER BY distance + LIMIT) +select a from u8t order by l2_distance(v, '[0,1,2,3]') limit 3; +select a from u8t order by inner_product(v, '[1,1,1,1]') limit 3; + +-- filtering / equality / ordering / distinct +select a from u8t where v = "[10,20,30,40]"; +select * from u8t order by v desc; +select distinct v from u8t order by v; + +-- negative: arithmetic is not allowed on vecuint8 (must CAST to vecf32 first) +select v + v from u8t; +select v * v from u8t; +select abs(v) from u8t; +select summation(v) from u8t; +-- arithmetic IS allowed after an explicit cast to vecf32 +select cast(v as vecf32(4)) + cast(v as vecf32(4)) from u8t order by a; + +-- vecuint8_from_base64: decode raw little-endian uint8 bytes. This is the builtin +-- the ivf search re-rank emits for the (quantized) query vector. A constant +-- argument constant-folds, so this is a direct regression for the elemSize +-- divide-by-zero panic that hid when the uint8 case was missing from the decoder. +select vecuint8_from_base64('ChQeKA=='); +select vecuint8_from_base64('AP+AAQ=='); + +drop database if exists vecu8db; diff --git a/test/distributed/cases/fulltext/bm25_coexist.result b/test/distributed/cases/fulltext/bm25_coexist.result new file mode 100644 index 0000000000000..a4ed188ff2a33 --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_coexist.result @@ -0,0 +1,37 @@ +drop database if exists bm25_coexist; +create database bm25_coexist; +use bm25_coexist; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +insert into t values (1,'apple banana cherry'),(2,'apple banana'),(3,'apple'),(4,'durian'),(5,'apple apple apple banana'); +create fulltext index ftc on t(txt); +create index ftr using bm25 on t(txt) with parser gojieba; +select id from t where match(txt) against('apple banana cherry' in boolean mode); +id +1 +2 +5 +3 +select id from t where match(txt) against('apple' in natural language mode); +id +5 +1 +2 +3 +select id from t where bm25(txt) against('apple'); +id +5 +3 +2 +1 +select id from t where bm25(txt) against('apple banana'); +id +2 +1 +5 +3 +create fulltext index ftc2 on t(txt); +not supported: Fulltext index are not allowed to use the same column +create index ftr2 using bm25 on t(txt) with parser gojieba; +not supported: Multiple bm25 indexes are not allowed to use the same column +drop database bm25_coexist; diff --git a/test/distributed/cases/fulltext/bm25_coexist.sql b/test/distributed/cases/fulltext/bm25_coexist.sql new file mode 100644 index 0000000000000..ca2f31b60fd84 --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_coexist.sql @@ -0,0 +1,24 @@ +-- A classic fulltext index and a bm25 index coexist on the SAME column, cleanly +-- disambiguated by query verb (no mode routing): MATCH(txt) -> the classic postings +-- index (supports boolean / natural language), BM25(txt) -> the bm25 ranked index. +-- Same-category duplicates (two classic, or two bm25) on one column stay rejected. +drop database if exists bm25_coexist; +create database bm25_coexist; +use bm25_coexist; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +insert into t values (1,'apple banana cherry'),(2,'apple banana'),(3,'apple'),(4,'durian'),(5,'apple apple apple banana'); +create fulltext index ftc on t(txt); +create index ftr using bm25 on t(txt) with parser gojieba; +-- MATCH -> classic fulltext (boolean mode: OR bag-of-words over any term) +select id from t where match(txt) against('apple banana cherry' in boolean mode); +-- MATCH -> classic fulltext (natural language) +select id from t where match(txt) against('apple' in natural language mode); +-- BM25 -> bm25 ranked top-K (BM25 score DESC) +select id from t where bm25(txt) against('apple'); +-- BM25 ranked bag-of-words, multi-term +select id from t where bm25(txt) against('apple banana'); +-- same-category duplicate on the same column is still rejected +create fulltext index ftc2 on t(txt); +create index ftr2 using bm25 on t(txt) with parser gojieba; +drop database bm25_coexist; diff --git a/test/distributed/cases/fulltext/bm25_datetime.result b/test/distributed/cases/fulltext/bm25_datetime.result new file mode 100644 index 0000000000000..038dc50cf2208 --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_datetime.result @@ -0,0 +1,25 @@ +drop database if exists bm25_datetime; +create database bm25_datetime; +use bm25_datetime; +set experimental_bm25_index = 1; +create table t (id datetime primary key, txt text); +insert into t values +('2020-06-01 10:00:01', '孩子 营养 早餐 视频 文案'), +('2020-06-01 10:00:02', '营养 早餐 健康 食谱'), +('2020-06-01 10:00:03', '视频 文案 创作 技巧'), +('2020-06-01 10:00:04', '孩子 教育 成长'); +create index ft using bm25 on t(txt) with parser gojieba; +select id from t where bm25(txt) against('营养 早餐'); +id +2020-06-01 10:00:02 +2020-06-01 10:00:01 +select id from t where bm25(txt) against('视频 文案'); +id +2020-06-01 10:00:03 +2020-06-01 10:00:01 +select id from t where bm25(txt) against('教育'); +id +2020-06-01 10:00:04 +select id from t where bm25(txt) against('不存在的词'); +id +drop database bm25_datetime; diff --git a/test/distributed/cases/fulltext/bm25_datetime.sql b/test/distributed/cases/fulltext/bm25_datetime.sql new file mode 100644 index 0000000000000..0ca7d4f129422 --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_datetime.sql @@ -0,0 +1,18 @@ +-- bm25 index on a DATETIME primary key (encodePk supports it). Ported from +-- fulltext_retrieval_datetime.sql. +drop database if exists bm25_datetime; +create database bm25_datetime; +use bm25_datetime; +set experimental_bm25_index = 1; +create table t (id datetime primary key, txt text); +insert into t values + ('2020-06-01 10:00:01', '孩子 营养 早餐 视频 文案'), + ('2020-06-01 10:00:02', '营养 早餐 健康 食谱'), + ('2020-06-01 10:00:03', '视频 文案 创作 技巧'), + ('2020-06-01 10:00:04', '孩子 教育 成长'); +create index ft using bm25 on t(txt) with parser gojieba; +select id from t where bm25(txt) against('营养 早餐'); +select id from t where bm25(txt) against('视频 文案'); +select id from t where bm25(txt) against('教育'); +select id from t where bm25(txt) against('不存在的词'); +drop database bm25_datetime; diff --git a/test/distributed/cases/fulltext/bm25_gate.result b/test/distributed/cases/fulltext/bm25_gate.result new file mode 100644 index 0000000000000..087174f5ea768 --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_gate.result @@ -0,0 +1,15 @@ +drop database if exists bm25_gate; +create database bm25_gate; +use bm25_gate; +create table t (id bigint primary key, body text); +insert into t values (1,'apple banana'),(2,'apple'); +set experimental_bm25_index = 0; +create index ft using bm25 on t(body) with parser gojieba; +internal error: experimental_bm25_index is not enabled +set experimental_bm25_index = 1; +create index ft using bm25 on t(body) with parser gojieba; +select id from t where bm25(body) against('apple'); +id +1 +2 +drop database bm25_gate; diff --git a/test/distributed/cases/fulltext/bm25_gate.sql b/test/distributed/cases/fulltext/bm25_gate.sql new file mode 100644 index 0000000000000..c40e066ab01f5 --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_gate.sql @@ -0,0 +1,18 @@ +-- CREATE INDEX ... USING bm25 is gated behind the experimental_bm25_index session +-- variable (off by default). Without it the DDL is rejected; enabling it allows the +-- index, and querying then works. +drop database if exists bm25_gate; +create database bm25_gate; +use bm25_gate; +create table t (id bigint primary key, body text); +insert into t values (1,'apple banana'),(2,'apple'); +-- explicitly turn the flag OFF (mo-tester reuses the session across files, so an +-- earlier bm25 case's `set experimental_bm25_index=1` can leak in). +set experimental_bm25_index = 0; +-- flag off -> rejected +create index ft using bm25 on t(body) with parser gojieba; +-- enable and retry -> succeeds +set experimental_bm25_index = 1; +create index ft using bm25 on t(body) with parser gojieba; +select id from t where bm25(body) against('apple'); +drop database bm25_gate; diff --git a/test/distributed/cases/fulltext/bm25_limit.result b/test/distributed/cases/fulltext/bm25_limit.result new file mode 100644 index 0000000000000..843610faacd33 --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_limit.result @@ -0,0 +1,44 @@ +drop database if exists bm25_limit; +create database bm25_limit; +use bm25_limit; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +insert into t values +(1, '营养 天气 城市'), +(2, '营养 早餐 城市'), +(3, '营养 早餐 视频'), +(4, '视频 天气 城市'), +(5, '教育 成长 学习'); +create index ft using bm25 on t(txt) with parser gojieba; +select id from t where bm25(txt) against('营养 早餐 视频'); +id +3 +2 +4 +1 +select id from t where bm25(txt) against('营养 早餐 视频') limit 1; +id +3 +select id from t where bm25(txt) against('营养 早餐 视频') limit 2; +id +3 +2 +select id from t where bm25(txt) against('营养 早餐 视频') limit 3; +id +3 +2 +4 +select id from t where bm25(txt) against('营养 早餐 视频') limit 10; +id +3 +2 +4 +1 +select id from t where bm25(txt) against('营养'); +id +1 +2 +3 +select id from t where bm25(txt) against('不存在') limit 5; +id +drop database bm25_limit; diff --git a/test/distributed/cases/fulltext/bm25_limit.sql b/test/distributed/cases/fulltext/bm25_limit.sql new file mode 100644 index 0000000000000..27192eefe378e --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_limit.sql @@ -0,0 +1,22 @@ +-- bm25 ranked top-K: LIMIT pushdown returns the highest-scored docs. Ported from +-- fulltext_retrieval_limit.sql (IN RETRIEVAL MODE -> default). +drop database if exists bm25_limit; +create database bm25_limit; +use bm25_limit; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +insert into t values + (1, '营养 天气 城市'), + (2, '营养 早餐 城市'), + (3, '营养 早餐 视频'), + (4, '视频 天气 城市'), + (5, '教育 成长 学习'); +create index ft using bm25 on t(txt) with parser gojieba; +select id from t where bm25(txt) against('营养 早餐 视频'); +select id from t where bm25(txt) against('营养 早餐 视频') limit 1; +select id from t where bm25(txt) against('营养 早餐 视频') limit 2; +select id from t where bm25(txt) against('营养 早餐 视频') limit 3; +select id from t where bm25(txt) against('营养 早餐 视频') limit 10; +select id from t where bm25(txt) against('营养'); +select id from t where bm25(txt) against('不存在') limit 5; +drop database bm25_limit; diff --git a/test/distributed/cases/fulltext/bm25_mixed.result b/test/distributed/cases/fulltext/bm25_mixed.result new file mode 100644 index 0000000000000..7d221eb181d29 --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_mixed.result @@ -0,0 +1,31 @@ +drop database if exists bm25_mixed; +create database bm25_mixed; +use bm25_mixed; +set experimental_bm25_index = 1; +create table t (id bigint primary key, a text, b text); +insert into t values +(1,'apple banana','cat dog'), +(2,'apple','dog'), +(3,'cherry','cat'), +(4,'apple apple','cat cat'), +(5,'banana','bird'); +create index ba using bm25 on t(a) with parser gojieba; +create fulltext index fb on t(b); +select id from t where bm25(a) against('apple') and match(b) against('cat'); +id +4 +1 +select id from t where match(b) against('cat') and bm25(a) against('apple'); +id +4 +1 +select id from t where bm25(a) against('apple') and match(b) against('dog'); +id +2 +1 +select id from t where bm25(a) against('apple') and match(b) against('cat') and id > 1; +id +4 +select id from t where bm25(a) against('cherry') and match(b) against('dog'); +id +drop database bm25_mixed; diff --git a/test/distributed/cases/fulltext/bm25_mixed.sql b/test/distributed/cases/fulltext/bm25_mixed.sql new file mode 100644 index 0000000000000..784402cbb5c19 --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_mixed.sql @@ -0,0 +1,28 @@ +-- A single query can combine BM25() and classic MATCH() — one on a bm25-indexed +-- column, the other on a fulltext-indexed column. The planner serves both in ONE +-- pass: one join chain (bm25_search + fulltext_index_scan) feeding one sort keyed by +-- BOTH scores. This is the payoff of the unified match-rewrite loop. +drop database if exists bm25_mixed; +create database bm25_mixed; +use bm25_mixed; +set experimental_bm25_index = 1; +create table t (id bigint primary key, a text, b text); +insert into t values +(1,'apple banana','cat dog'), +(2,'apple','dog'), +(3,'cherry','cat'), +(4,'apple apple','cat cat'), +(5,'banana','bird'); +create index ba using bm25 on t(a) with parser gojieba; +create fulltext index fb on t(b); +-- BM25(a) apple ∩ MATCH(b) cat -> docs 1,4 (apple in a AND cat in b) +select id from t where bm25(a) against('apple') and match(b) against('cat'); +-- order of the two verbs does not matter +select id from t where match(b) against('cat') and bm25(a) against('apple'); +-- BM25(a) apple ∩ MATCH(b) dog -> docs 1,2 +select id from t where bm25(a) against('apple') and match(b) against('dog'); +-- combined with a normal SQL filter +select id from t where bm25(a) against('apple') and match(b) against('cat') and id > 1; +-- no overlap -> empty (banana in a, but cat in b matches none of the banana docs) +select id from t where bm25(a) against('cherry') and match(b) against('dog'); +drop database bm25_mixed; diff --git a/test/distributed/cases/fulltext/bm25_mode_guard.result b/test/distributed/cases/fulltext/bm25_mode_guard.result new file mode 100644 index 0000000000000..d57d1769b8b1c --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_mode_guard.result @@ -0,0 +1,22 @@ +drop database if exists bm25_mode; +create database bm25_mode; +use bm25_mode; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +insert into t values (1,'apple banana'),(2,'banana cherry'),(3,'cherry date'); +create index ft using bm25 on t(txt) with parser gojieba; +select id from t where bm25(txt) against('apple'); +id +1 +select id from t where bm25(txt) against('banana'); +id +1 +2 +select id from t where bm25(txt) against('apple cherry'); +id +1 +2 +3 +select id from t where match(txt) against('apple'); +not supported: MATCH() AGAINST() function cannot be replaced by FULLTEXT INDEX and full table scan with fulltext search is not supported yet. +drop database bm25_mode; diff --git a/test/distributed/cases/fulltext/bm25_mode_guard.sql b/test/distributed/cases/fulltext/bm25_mode_guard.sql new file mode 100644 index 0000000000000..fb69833d4a1d4 --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_mode_guard.sql @@ -0,0 +1,21 @@ +-- bm25 query-surface contract: a bm25 index is queried ONLY through BM25(col) +-- AGAINST('query') — a distinct, mode-free ranked bag-of-words surface. It is NOT +-- queried through MATCH() (that is the classic fulltext surface); MATCH() on a +-- bm25-only column finds no fulltext index and errors. BM25() itself has no mode +-- modifiers (no boolean / natural language / query expansion) — those are not even +-- expressible in the grammar. +drop database if exists bm25_mode; +create database bm25_mode; +use bm25_mode; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +insert into t values (1,'apple banana'),(2,'banana cherry'),(3,'cherry date'); +create index ft using bm25 on t(txt) with parser gojieba; +-- BM25() ranked retrieval works +select id from t where bm25(txt) against('apple'); +select id from t where bm25(txt) against('banana'); +-- multi-term bag-of-words +select id from t where bm25(txt) against('apple cherry'); +-- MATCH() on a bm25-only column has no classic fulltext index -> error +select id from t where match(txt) against('apple'); +drop database bm25_mode; diff --git a/test/distributed/cases/fulltext/bm25_multibase.result b/test/distributed/cases/fulltext/bm25_multibase.result new file mode 100644 index 0000000000000..6a6d2ca1e9a46 --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_multibase.result @@ -0,0 +1,30 @@ +drop database if exists bm25_multibase; +create database bm25_multibase; +use bm25_multibase; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +insert into t values +(1,'apple banana'), +(2,'banana cherry'), +(3,'cherry date'), +(4,'date apple'), +(5,'elderberry fig'), +(6,'fig grape'); +create index ft using bm25 on t(txt) with parser gojieba max_index_capacity=2; +select id from t where bm25(txt) against('apple'); +id +1 +4 +select id from t where bm25(txt) against('cherry'); +id +2 +3 +select id from t where bm25(txt) against('banana'); +id +1 +2 +select id from t where bm25(txt) against('fig'); +id +5 +6 +drop database bm25_multibase; diff --git a/test/distributed/cases/fulltext/bm25_multibase.sql b/test/distributed/cases/fulltext/bm25_multibase.sql new file mode 100644 index 0000000000000..f73f425f74f68 --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_multibase.sql @@ -0,0 +1,21 @@ +-- bm25 with max_index_capacity=2 over 6 rows -> the tag=0 base is split into +-- multiple sub-indexes; the search loads + merges all subs. Ported from +-- fulltext_retrieval_multibase.sql. +drop database if exists bm25_multibase; +create database bm25_multibase; +use bm25_multibase; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +insert into t values + (1,'apple banana'), + (2,'banana cherry'), + (3,'cherry date'), + (4,'date apple'), + (5,'elderberry fig'), + (6,'fig grape'); +create index ft using bm25 on t(txt) with parser gojieba max_index_capacity=2; +select id from t where bm25(txt) against('apple'); +select id from t where bm25(txt) against('cherry'); +select id from t where bm25(txt) against('banana'); +select id from t where bm25(txt) against('fig'); +drop database bm25_multibase; diff --git a/test/distributed/cases/fulltext/bm25_pushdown.result b/test/distributed/cases/fulltext/bm25_pushdown.result new file mode 100644 index 0000000000000..35856b7cfae0a --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_pushdown.result @@ -0,0 +1,68 @@ +drop database if exists bm25_pushdown; +create database bm25_pushdown; +use bm25_pushdown; +set experimental_bm25_index = 1; +create table docs (id bigint primary key, body text, cat int); +insert into docs values +(1,'apple banana cherry',10), +(2,'apple banana',20), +(3,'apple',10), +(4,'durian mango',10), +(5,'apple apple apple banana',20), +(6,'apple cherry',10), +(7,'banana cherry',20); +create index ftx using bm25 on docs(body) with parser gojieba; +set fulltext_bloom_filter_pushdown=off; +select id from docs where bm25(body) against('apple') and cat=10; +id +3 +6 +1 +select id from docs where bm25(body) against('apple') and cat=20; +id +5 +2 +select id from docs where bm25(body) against('apple') and cat=10 limit 2; +id +3 +6 +select id from docs where bm25(body) against('apple banana') and cat=20; +id +2 +5 +7 +select id from docs where bm25(body) against('durian') and cat=20; +id +select id from docs where bm25(body) against('apple') and cat=99; +id +select count(*) from docs where bm25(body) against('apple') and cat=10; +count(*) +3 +set fulltext_bloom_filter_pushdown=on; +select id from docs where bm25(body) against('apple') and cat=10; +id +3 +6 +1 +select id from docs where bm25(body) against('apple') and cat=20; +id +5 +2 +select id from docs where bm25(body) against('apple') and cat=10 limit 2; +id +3 +6 +select id from docs where bm25(body) against('apple banana') and cat=20; +id +2 +5 +7 +select id from docs where bm25(body) against('durian') and cat=20; +id +select id from docs where bm25(body) against('apple') and cat=99; +id +select count(*) from docs where bm25(body) against('apple') and cat=10; +count(*) +3 +set fulltext_bloom_filter_pushdown=off; +drop database bm25_pushdown; diff --git a/test/distributed/cases/fulltext/bm25_pushdown.sql b/test/distributed/cases/fulltext/bm25_pushdown.sql new file mode 100644 index 0000000000000..956c3ac0012a0 --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_pushdown.sql @@ -0,0 +1,50 @@ +-- bm25 filtered retrieval with fulltext_bloom_filter_pushdown=ON. A MATCH combined +-- with an extra non-MATCH WHERE filter builds the pre-filter 2-JOIN that pushes the +-- predicate into the WAND walk as a membership bitset (so bm25_search only scores the +-- qualifying docs). The pushdown is a pure performance optimization: results MUST be +-- identical to the pushdown=OFF path. This exercises the 2-JOIN membership-bitset +-- prefilter, which the other bm25 cases (no extra WHERE) never reach. +drop database if exists bm25_pushdown; +create database bm25_pushdown; +use bm25_pushdown; +set experimental_bm25_index = 1; +create table docs (id bigint primary key, body text, cat int); +insert into docs values +(1,'apple banana cherry',10), +(2,'apple banana',20), +(3,'apple',10), +(4,'durian mango',10), +(5,'apple apple apple banana',20), +(6,'apple cherry',10), +(7,'banana cherry',20); +create index ftx using bm25 on docs(body) with parser gojieba; + +-- ===== baseline: pushdown OFF (single JOIN, filter applied after search) ===== +set fulltext_bloom_filter_pushdown=off; +-- apple ∩ cat=10 -> {1,3,6} +select id from docs where bm25(body) against('apple') and cat=10; +-- apple ∩ cat=20 -> {2,5} +select id from docs where bm25(body) against('apple') and cat=20; +-- ranked top-2 of apple ∩ cat=10 +select id from docs where bm25(body) against('apple') and cat=10 limit 2; +-- two-term MATCH + filter +select id from docs where bm25(body) against('apple banana') and cat=20; +-- filter selects rows the term does not match -> empty +select id from docs where bm25(body) against('durian') and cat=20; +-- filter matches nothing -> empty +select id from docs where bm25(body) against('apple') and cat=99; +-- count with MATCH + filter +select count(*) from docs where bm25(body) against('apple') and cat=10; + +-- ===== pushdown ON (2-JOIN membership-bitset prefilter): SAME rows ===== +set fulltext_bloom_filter_pushdown=on; +select id from docs where bm25(body) against('apple') and cat=10; +select id from docs where bm25(body) against('apple') and cat=20; +select id from docs where bm25(body) against('apple') and cat=10 limit 2; +select id from docs where bm25(body) against('apple banana') and cat=20; +select id from docs where bm25(body) against('durian') and cat=20; +select id from docs where bm25(body) against('apple') and cat=99; +select count(*) from docs where bm25(body) against('apple') and cat=10; + +set fulltext_bloom_filter_pushdown=off; +drop database bm25_pushdown; diff --git a/test/distributed/cases/fulltext/bm25_retrieval.result b/test/distributed/cases/fulltext/bm25_retrieval.result new file mode 100644 index 0000000000000..f39a7f32750af --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_retrieval.result @@ -0,0 +1,25 @@ +drop database if exists bm25_retrieval; +create database bm25_retrieval; +use bm25_retrieval; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +insert into t values +(1, '孩子 营养 早餐 视频 文案'), +(2, '营养 早餐 健康 食谱'), +(3, '视频 文案 创作 技巧'), +(4, '孩子 教育 成长'); +create index ft using bm25 on t(txt) with parser gojieba; +select id from t where bm25(txt) against('营养 早餐'); +id +2 +1 +select id from t where bm25(txt) against('视频 文案'); +id +3 +1 +select id from t where bm25(txt) against('教育'); +id +4 +select id from t where bm25(txt) against('不存在的词'); +id +drop database bm25_retrieval; diff --git a/test/distributed/cases/fulltext/bm25_retrieval.sql b/test/distributed/cases/fulltext/bm25_retrieval.sql new file mode 100644 index 0000000000000..0781e1d025110 --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_retrieval.sql @@ -0,0 +1,19 @@ +-- bm25 ranked-retrieval index: build from source + ranked MATCH over jieba tokens. +-- Ported from fulltext_retrieval.sql (with parser retrieval -> using bm25; +-- IN RETRIEVAL MODE -> default; bm25 build is synchronous so no settle sleep). +drop database if exists bm25_retrieval; +create database bm25_retrieval; +use bm25_retrieval; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +insert into t values + (1, '孩子 营养 早餐 视频 文案'), + (2, '营养 早餐 健康 食谱'), + (3, '视频 文案 创作 技巧'), + (4, '孩子 教育 成长'); +create index ft using bm25 on t(txt) with parser gojieba; +select id from t where bm25(txt) against('营养 早餐'); +select id from t where bm25(txt) against('视频 文案'); +select id from t where bm25(txt) against('教育'); +select id from t where bm25(txt) against('不存在的词'); +drop database bm25_retrieval; diff --git a/test/distributed/cases/fulltext/bm25_uuid.result b/test/distributed/cases/fulltext/bm25_uuid.result new file mode 100644 index 0000000000000..4c40e8076f363 --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_uuid.result @@ -0,0 +1,25 @@ +drop database if exists bm25_uuid; +create database bm25_uuid; +use bm25_uuid; +set experimental_bm25_index = 1; +create table t (id uuid primary key, txt text); +insert into t values +('00000000-0000-0000-0000-000000000001', '孩子 营养 早餐 视频 文案'), +('00000000-0000-0000-0000-000000000002', '营养 早餐 健康 食谱'), +('00000000-0000-0000-0000-000000000003', '视频 文案 创作 技巧'), +('00000000-0000-0000-0000-000000000004', '孩子 教育 成长'); +create index ft using bm25 on t(txt) with parser gojieba; +select id from t where bm25(txt) against('营养 早餐'); +id +00000000-0000-0000-0000-000000000002 +00000000-0000-0000-0000-000000000001 +select id from t where bm25(txt) against('视频 文案'); +id +00000000-0000-0000-0000-000000000003 +00000000-0000-0000-0000-000000000001 +select id from t where bm25(txt) against('教育'); +id +00000000-0000-0000-0000-000000000004 +select id from t where bm25(txt) against('不存在的词'); +id +drop database bm25_uuid; diff --git a/test/distributed/cases/fulltext/bm25_uuid.sql b/test/distributed/cases/fulltext/bm25_uuid.sql new file mode 100644 index 0000000000000..8baf012a0d025 --- /dev/null +++ b/test/distributed/cases/fulltext/bm25_uuid.sql @@ -0,0 +1,18 @@ +-- bm25 index on a UUID primary key (encodePk supports it). Ported from +-- fulltext_retrieval_uuid.sql. +drop database if exists bm25_uuid; +create database bm25_uuid; +use bm25_uuid; +set experimental_bm25_index = 1; +create table t (id uuid primary key, txt text); +insert into t values + ('00000000-0000-0000-0000-000000000001', '孩子 营养 早餐 视频 文案'), + ('00000000-0000-0000-0000-000000000002', '营养 早餐 健康 食谱'), + ('00000000-0000-0000-0000-000000000003', '视频 文案 创作 技巧'), + ('00000000-0000-0000-0000-000000000004', '孩子 教育 成长'); +create index ft using bm25 on t(txt) with parser gojieba; +select id from t where bm25(txt) against('营养 早餐'); +select id from t where bm25(txt) against('视频 文案'); +select id from t where bm25(txt) against('教育'); +select id from t where bm25(txt) against('不存在的词'); +drop database bm25_uuid; diff --git a/test/distributed/cases/load_data/load_data_narrow_vec.result b/test/distributed/cases/load_data/load_data_narrow_vec.result new file mode 100644 index 0000000000000..8a2c5e550399a --- /dev/null +++ b/test/distributed/cases/load_data/load_data_narrow_vec.result @@ -0,0 +1,32 @@ +drop database if exists load_narrow_vec; +create database load_narrow_vec; +use load_narrow_vec; +create table nvec(id int, a vecbf16(3), b vecf16(3), c vecint8(3), d vecuint8(3)); +load data infile '$resources/load_data/narrow_vec_array.csv' into table nvec fields terminated by ',' ignore 1 lines; +select * from nvec order by id; +id a b c d +1 [1, 2, 3] [0.5, 0.25, -0.5] [-128, 0, 127] [0, 128, 255] +2 [0.5, -0.25, 4] [1, 2, 3] [10, -10, 5] [1, 2, 3] +select id, round(l2_distance(c, '[0,0,0]'), 4) as dist from nvec order by id; +id dist +1 180.3136 +2 15.0 +create table nvec_oor(id int, c vecint8(3)); +load data infile '$resources/load_data/narrow_vec_int8_oor.csv' into table nvec_oor fields terminated by ',' ignore 1 lines; +internal error: error while casting 200 to VECINT8 +select count(*) as cnt from nvec_oor; +cnt +0 +create table nvec_frac(id int, c vecint8(3)); +load data infile '$resources/load_data/narrow_vec_int8_frac.csv' into table nvec_frac fields terminated by ',' ignore 1 lines; +internal error: error while casting 0.5 to VECINT8 +select count(*) as cnt from nvec_frac; +cnt +0 +create table nvec_dim(id int, d vecuint8(3)); +load data infile '$resources/load_data/narrow_vec_dim_bad.csv' into table nvec_dim fields terminated by ',' ignore 1 lines; +invalid input: expected vector dimension 3 != actual dimension 2. +select count(*) as cnt from nvec_dim; +cnt +0 +drop database load_narrow_vec; diff --git a/test/distributed/cases/load_data/load_data_narrow_vec.sql b/test/distributed/cases/load_data/load_data_narrow_vec.sql new file mode 100644 index 0000000000000..ac387d71e1f4c --- /dev/null +++ b/test/distributed/cases/load_data/load_data_narrow_vec.sql @@ -0,0 +1,51 @@ +-- Test: LOAD DATA INFILE into narrow vector base columns +-- (vecbf16 / vecf16 / vecint8 / vecuint8). +-- +-- Before the fix, the external/CSV import switches in external.go only handled +-- T_array_float32/float64, so loading a narrow vector column failed with +-- "the value type N is not support now". INSERT already worked; only the bulk +-- LOAD path was missing the narrow cases. +-- +-- int8/uint8 string parse is strict (integer, in range); fractional or +-- out-of-range values are rejected — mirroring INSERT. + +drop database if exists load_narrow_vec; +create database load_narrow_vec; +use load_narrow_vec; + +-- ============================================================ +-- 1. Happy path: load all four narrow types from one CSV. +-- Values are exactly representable in bf16/f16 (and integer +-- for int8/uint8), so the round-trip is loss-free. +-- ============================================================ +create table nvec(id int, a vecbf16(3), b vecf16(3), c vecint8(3), d vecuint8(3)); +load data infile '$resources/load_data/narrow_vec_array.csv' into table nvec fields terminated by ',' ignore 1 lines; +select * from nvec order by id; + +-- distance functions work on a loaded narrow column +-- round to 4 digits: l2_distance (sqrt, float64) low-order bits vary across SIMD kernels +select id, round(l2_distance(c, '[0,0,0]'), 4) as dist from nvec order by id; + +-- ============================================================ +-- 2. Strict int8 parse: out-of-range value (200) is rejected. +-- ============================================================ +create table nvec_oor(id int, c vecint8(3)); +load data infile '$resources/load_data/narrow_vec_int8_oor.csv' into table nvec_oor fields terminated by ',' ignore 1 lines; +select count(*) as cnt from nvec_oor; + +-- ============================================================ +-- 3. Strict int8 parse: fractional value (0.5) is rejected. +-- ============================================================ +create table nvec_frac(id int, c vecint8(3)); +load data infile '$resources/load_data/narrow_vec_int8_frac.csv' into table nvec_frac fields terminated by ',' ignore 1 lines; +select count(*) as cnt from nvec_frac; + +-- ============================================================ +-- 4. Dimension mismatch is rejected (vecuint8(3) given 2 elems). +-- ============================================================ +create table nvec_dim(id int, d vecuint8(3)); +load data infile '$resources/load_data/narrow_vec_dim_bad.csv' into table nvec_dim fields terminated by ',' ignore 1 lines; +select count(*) as cnt from nvec_dim; + +-- cleanup +drop database load_narrow_vec; diff --git a/test/distributed/cases/pessimistic_transaction/bm25/bm25_async.result b/test/distributed/cases/pessimistic_transaction/bm25/bm25_async.result new file mode 100644 index 0000000000000..592e53c691bb8 --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/bm25/bm25_async.result @@ -0,0 +1,22 @@ +drop database if exists bm25_async; +create database bm25_async; +use bm25_async; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +create index ft using bm25 on t(txt) with parser gojieba; +insert into t values (1, '营养 早餐'), (2, '视频 文案'); +insert into t values (3, '营养 健康 食谱'); +delete from t where id = 1; +select sleep(30); +sleep(30) +0 +select id from t where bm25(txt) against('营养'); +id +3 +select id from t where bm25(txt) against('视频'); +id +2 +select id from t where bm25(txt) against('健康'); +id +3 +drop database bm25_async; diff --git a/test/distributed/cases/pessimistic_transaction/bm25/bm25_async.sql b/test/distributed/cases/pessimistic_transaction/bm25/bm25_async.sql new file mode 100644 index 0000000000000..1deda22212104 --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/bm25/bm25_async.sql @@ -0,0 +1,19 @@ +-- bm25 ranked-retrieval index: post-create DML (INSERT/DELETE) flows into the +-- tag=1 CdcTail via the WAND sinker and is visible after the CDC settles. +-- Ported from pessimistic_transaction/fulltext/fulltext_retrieval_async.sql +-- (create fulltext ... with parser retrieval -> create index ... using bm25; +-- IN RETRIEVAL MODE -> default ranked mode). +drop database if exists bm25_async; +create database bm25_async; +use bm25_async; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +create index ft using bm25 on t(txt) with parser gojieba; +insert into t values (1, '营养 早餐'), (2, '视频 文案'); +insert into t values (3, '营养 健康 食谱'); +delete from t where id = 1; +select sleep(30); +select id from t where bm25(txt) against('营养'); +select id from t where bm25(txt) against('视频'); +select id from t where bm25(txt) against('健康'); +drop database bm25_async; diff --git a/test/distributed/cases/pessimistic_transaction/bm25/bm25_basic.result b/test/distributed/cases/pessimistic_transaction/bm25/bm25_basic.result new file mode 100644 index 0000000000000..62d8dcbec0011 --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/bm25/bm25_basic.result @@ -0,0 +1,32 @@ +drop database if exists bm25_basic; +create database bm25_basic; +use bm25_basic; +set experimental_bm25_index = 1; +create table docs (id bigint primary key, body text); +insert into docs values (1,'apple banana cherry'),(2,'apple banana'),(3,'apple'),(4,'durian mango'),(5,'apple apple apple banana'); +create index ftx using bm25 on docs(body) with parser gojieba; +select id from docs where bm25(body) against('apple'); +id +5 +3 +2 +1 +select id from docs where bm25(body) against('apple'); +id +5 +3 +2 +1 +select id from docs where bm25(body) against('apple banana'); +id +2 +1 +5 +3 +select id from docs where bm25(body) against('apple') limit 2; +id +5 +3 +select id from docs where match(body) against('apple'); +not supported: MATCH() AGAINST() function cannot be replaced by FULLTEXT INDEX and full table scan with fulltext search is not supported yet. +drop database bm25_basic; diff --git a/test/distributed/cases/pessimistic_transaction/bm25/bm25_basic.sql b/test/distributed/cases/pessimistic_transaction/bm25/bm25_basic.sql new file mode 100644 index 0000000000000..a31e895aa2971 --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/bm25/bm25_basic.sql @@ -0,0 +1,21 @@ +-- bm25 ranked-retrieval index: synchronous build from source + BM25() ranked +-- retrieval (bag-of-words BM25 top-K), LIMIT top-K pushdown, and the distinct +-- query surface (BM25(), not MATCH()). +drop database if exists bm25_basic; +create database bm25_basic; +use bm25_basic; +set experimental_bm25_index = 1; +create table docs (id bigint primary key, body text); +insert into docs values (1,'apple banana cherry'),(2,'apple banana'),(3,'apple'),(4,'durian mango'),(5,'apple apple apple banana'); +create index ftx using bm25 on docs(body) with parser gojieba; +-- membership: docs containing 'apple' (doc 4 excluded) +select id from docs where bm25(body) against('apple'); +-- ranked by BM25 score DESC (no ORDER BY): doc 5 (apple x3) first, doc 1 (longest) last +select id from docs where bm25(body) against('apple'); +-- multi-term bag-of-words +select id from docs where bm25(body) against('apple banana'); +-- LIMIT top-K pushdown: the two highest-scored docs +select id from docs where bm25(body) against('apple') limit 2; +-- distinct surface: MATCH() on a bm25-only column has no fulltext index -> error +select id from docs where match(body) against('apple'); +drop database bm25_basic; diff --git a/test/distributed/cases/pessimistic_transaction/bm25/bm25_clone.result b/test/distributed/cases/pessimistic_transaction/bm25/bm25_clone.result new file mode 100644 index 0000000000000..db605df47a188 --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/bm25/bm25_clone.result @@ -0,0 +1,25 @@ +drop database if exists bm25_clone; +create database bm25_clone; +use bm25_clone; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +insert into t values (1,'apple banana'),(2,'banana cherry'),(3,'cherry date'),(4,'date apple'); +create index ft using bm25 on t(txt) with parser gojieba max_index_capacity=2; +select id from t where bm25(txt) against('apple'); +id +1 +4 +create table t2 clone t; +insert into t2 values (5,'apple mango'); +select sleep(60); +sleep(60) +0 +select id from t2 where bm25(txt) against('apple'); +id +5 +1 +4 +select id from t2 where bm25(txt) against('mango'); +id +5 +drop database bm25_clone; diff --git a/test/distributed/cases/pessimistic_transaction/bm25/bm25_clone.sql b/test/distributed/cases/pessimistic_transaction/bm25/bm25_clone.sql new file mode 100644 index 0000000000000..a48c9606ee251 --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/bm25/bm25_clone.sql @@ -0,0 +1,17 @@ +-- CREATE TABLE ... CLONE of a bm25 index: the clone's index is rebuilt from the +-- cloned rows via the re-armed CDC's InitSQL (RestoreInitSQL), and post-clone +-- rows flow in via CDC. Ported from fulltext_retrieval_clone. +drop database if exists bm25_clone; +create database bm25_clone; +use bm25_clone; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +insert into t values (1,'apple banana'),(2,'banana cherry'),(3,'cherry date'),(4,'date apple'); +create index ft using bm25 on t(txt) with parser gojieba max_index_capacity=2; +select id from t where bm25(txt) against('apple'); +create table t2 clone t; +insert into t2 values (5,'apple mango'); +select sleep(60); +select id from t2 where bm25(txt) against('apple'); +select id from t2 where bm25(txt) against('mango'); +drop database bm25_clone; diff --git a/test/distributed/cases/pessimistic_transaction/bm25/bm25_merge.result b/test/distributed/cases/pessimistic_transaction/bm25/bm25_merge.result new file mode 100644 index 0000000000000..25aba67fdba07 --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/bm25/bm25_merge.result @@ -0,0 +1,33 @@ +drop database if exists bm25_merge; +create database bm25_merge; +use bm25_merge; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +insert into t values (1,'apple red'),(2,'apple green'),(3,'apple blue'); +create index ft using bm25 on t(txt) with parser gojieba; +insert into t values (4,'apple yellow'); +delete from t where id=2; +select sleep(30); +sleep(30) +0 +alter table t alter reindex ft bm25 merge; +insert into t values (5,'apple pink'); +update t set txt='apple orange' where id=4; +select sleep(30); +sleep(30) +0 +alter table t alter reindex ft bm25 merge; +select id from t where bm25(txt) against('apple'); +id +1 +3 +4 +5 +select id from t where bm25(txt) against('green'); +id +select id from t where bm25(txt) against('orange'); +id +4 +select id from t where bm25(txt) against('yellow'); +id +drop database bm25_merge; diff --git a/test/distributed/cases/pessimistic_transaction/bm25/bm25_merge.sql b/test/distributed/cases/pessimistic_transaction/bm25/bm25_merge.sql new file mode 100644 index 0000000000000..81c66ea0d7e9d --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/bm25/bm25_merge.sql @@ -0,0 +1,23 @@ +-- ALTER ... REINDEX ... BM25 MERGE folds the tag=1 CdcTail into the tag=0 base +-- (incremental compaction, no re-tokenize). Ported from fulltext_retrieval_merge +-- (bm25 part only; the classic-fulltext table is dropped). +drop database if exists bm25_merge; +create database bm25_merge; +use bm25_merge; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +insert into t values (1,'apple red'),(2,'apple green'),(3,'apple blue'); +create index ft using bm25 on t(txt) with parser gojieba; +insert into t values (4,'apple yellow'); +delete from t where id=2; +select sleep(30); +alter table t alter reindex ft bm25 merge; +insert into t values (5,'apple pink'); +update t set txt='apple orange' where id=4; +select sleep(30); +alter table t alter reindex ft bm25 merge; +select id from t where bm25(txt) against('apple'); +select id from t where bm25(txt) against('green'); +select id from t where bm25(txt) against('orange'); +select id from t where bm25(txt) against('yellow'); +drop database bm25_merge; diff --git a/test/distributed/cases/pessimistic_transaction/bm25/bm25_reindex.result b/test/distributed/cases/pessimistic_transaction/bm25/bm25_reindex.result new file mode 100644 index 0000000000000..056c0594d7cf8 --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/bm25/bm25_reindex.result @@ -0,0 +1,23 @@ +drop database if exists bm25_reindex; +create database bm25_reindex; +use bm25_reindex; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +insert into t values (1,'apple banana'),(2,'banana cherry'),(3,'cherry date'),(4,'date apple'); +create index ft using bm25 on t(txt) with parser gojieba max_index_capacity=2; +insert into t values (5,'fig grape'),(6,'grape apple'); +alter table t alter reindex ft bm25; +select id from t where bm25(txt) against('apple'); +id +1 +4 +6 +alter table t alter reindex ft bm25 max_index_capacity=3; +select id from t where bm25(txt) against('apple'); +id +1 +4 +6 +alter table t alter reindex ft bm25 lists=5; +not supported: bm25 reindex does not support option "lists" (only max_index_capacity) +drop database bm25_reindex; diff --git a/test/distributed/cases/pessimistic_transaction/bm25/bm25_reindex.sql b/test/distributed/cases/pessimistic_transaction/bm25/bm25_reindex.sql new file mode 100644 index 0000000000000..9f2cfd0551233 --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/bm25/bm25_reindex.sql @@ -0,0 +1,18 @@ +-- ALTER ... REINDEX ... BM25 rebuilds the whole index from source (re-tokenize), +-- and can change max_index_capacity; a non-bm25 option (lists) is rejected. +-- Ported from fulltext_retrieval_reindex (with parser retrieval -> using bm25; +-- reindex ... fulltext -> reindex ... bm25; IN RETRIEVAL MODE -> default). +drop database if exists bm25_reindex; +create database bm25_reindex; +use bm25_reindex; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +insert into t values (1,'apple banana'),(2,'banana cherry'),(3,'cherry date'),(4,'date apple'); +create index ft using bm25 on t(txt) with parser gojieba max_index_capacity=2; +insert into t values (5,'fig grape'),(6,'grape apple'); +alter table t alter reindex ft bm25; +select id from t where bm25(txt) against('apple'); +alter table t alter reindex ft bm25 max_index_capacity=3; +select id from t where bm25(txt) against('apple'); +alter table t alter reindex ft bm25 lists=5; +drop database bm25_reindex; diff --git a/test/distributed/cases/pessimistic_transaction/bm25/bm25_restore.result b/test/distributed/cases/pessimistic_transaction/bm25/bm25_restore.result new file mode 100644 index 0000000000000..71d17c09ef875 --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/bm25/bm25_restore.result @@ -0,0 +1,30 @@ +drop database if exists bm25_restore; +drop snapshot if exists sn_bm25_restore; +create database bm25_restore; +use bm25_restore; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +insert into t values (1,'apple banana'),(2,'banana cherry'),(3,'cherry date'),(4,'date apple'); +create index ft using bm25 on t(txt) with parser gojieba max_index_capacity=100; +select id from t where bm25(txt) against('apple'); +id +1 +4 +create snapshot sn_bm25_restore for account sys; +insert into t values (5,'fig apple'); +restore database bm25_restore {snapshot = "sn_bm25_restore"}; +use bm25_restore; +insert into t values (6,'grape apple'); +select sleep(30); +sleep(30) +0 +select id from t where bm25(txt) against('apple'); +id +6 +1 +4 +select id from t where bm25(txt) against('grape'); +id +6 +drop database bm25_restore; +drop snapshot if exists sn_bm25_restore; diff --git a/test/distributed/cases/pessimistic_transaction/bm25/bm25_restore.sql b/test/distributed/cases/pessimistic_transaction/bm25/bm25_restore.sql new file mode 100644 index 0000000000000..867599a53bbc2 --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/bm25/bm25_restore.sql @@ -0,0 +1,23 @@ +-- Snapshot + RESTORE of a bm25 index: the restore rebuilds the index from the +-- restored rows (RestoreInitSQL), rolling back the post-snapshot mutation; +-- post-restore rows flow in via the re-armed CDC. Ported from +-- fulltext_retrieval_restore. +drop database if exists bm25_restore; +drop snapshot if exists sn_bm25_restore; +create database bm25_restore; +use bm25_restore; +set experimental_bm25_index = 1; +create table t (id bigint primary key, txt text); +insert into t values (1,'apple banana'),(2,'banana cherry'),(3,'cherry date'),(4,'date apple'); +create index ft using bm25 on t(txt) with parser gojieba max_index_capacity=100; +select id from t where bm25(txt) against('apple'); +create snapshot sn_bm25_restore for account sys; +insert into t values (5,'fig apple'); +restore database bm25_restore {snapshot = "sn_bm25_restore"}; +use bm25_restore; +insert into t values (6,'grape apple'); +select sleep(30); +select id from t where bm25(txt) against('apple'); +select id from t where bm25(txt) against('grape'); +drop database bm25_restore; +drop snapshot if exists sn_bm25_restore; diff --git a/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_narrow_base_async.result b/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_narrow_base_async.result new file mode 100644 index 0000000000000..847dbab3501ff --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_narrow_base_async.result @@ -0,0 +1,134 @@ +SET probe_limit=10; +create table nbf(a int primary key, v vecbf16(4)); +insert into nbf values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xbf using ivfflat on nbf(v) lists=2 op_type 'vector_l2_ops' ASYNC; +create table nhf(a int primary key, v vecf16(4)); +insert into nhf values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xhf using ivfflat on nhf(v) lists=2 op_type 'vector_l2_ops' ASYNC; +create table ni8(a int primary key, v vecint8(4)); +insert into ni8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xi8 using ivfflat on ni8(v) lists=2 op_type 'vector_l2_ops' ASYNC; +create table nu8(a int primary key, v vecuint8(4)); +insert into nu8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xu8 using ivfflat on nu8(v) lists=2 op_type 'vector_l2_ops' ASYNC; +select sleep(30); +➤ sleep(30)[-6,8,0] 𝄀 +0 +select a from nbf order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +1 𝄀 +2 𝄀 +3 +select a from nbf order by l2_distance(v,'[54,54,54,54]') limit 3; +➤ a[4,32,0] 𝄀 +6 𝄀 +5 𝄀 +4 +select a from nhf order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +1 𝄀 +2 𝄀 +3 +select a from nhf order by l2_distance(v,'[54,54,54,54]') limit 3; +➤ a[4,32,0] 𝄀 +6 𝄀 +5 𝄀 +4 +select a from ni8 order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +1 𝄀 +2 𝄀 +3 +select a from ni8 order by l2_distance(v,'[54,54,54,54]') limit 3; +➤ a[4,32,0] 𝄀 +6 𝄀 +5 𝄀 +4 +select a from nu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +1 𝄀 +2 𝄀 +3 +select a from nu8 order by l2_distance(v,'[54,54,54,54]') limit 3; +➤ a[4,32,0] 𝄀 +6 𝄀 +5 𝄀 +4 +insert into nbf values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into nhf values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into ni8 values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into nu8 values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +select sleep(30); +➤ sleep(30)[-6,8,0] 𝄀 +0 +select a from nbf order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +1 𝄀 +7 𝄀 +2 +select a from nbf order by l2_distance(v,'[54,54,54,54]') limit 3; +➤ a[4,32,0] 𝄀 +6 𝄀 +8 𝄀 +5 +select a from nhf order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +1 𝄀 +7 𝄀 +2 +select a from nhf order by l2_distance(v,'[54,54,54,54]') limit 3; +➤ a[4,32,0] 𝄀 +6 𝄀 +8 𝄀 +5 +select a from ni8 order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +1 𝄀 +7 𝄀 +2 +select a from ni8 order by l2_distance(v,'[54,54,54,54]') limit 3; +➤ a[4,32,0] 𝄀 +6 𝄀 +8 𝄀 +5 +select a from nu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +1 𝄀 +7 𝄀 +2 +select a from nu8 order by l2_distance(v,'[54,54,54,54]') limit 3; +➤ a[4,32,0] 𝄀 +6 𝄀 +8 𝄀 +5 +update nbf set v = '[55,55,55,55]' where a = 1; +update nhf set v = '[55,55,55,55]' where a = 1; +update ni8 set v = '[55,55,55,55]' where a = 1; +update nu8 set v = '[55,55,55,55]' where a = 1; +select sleep(30); +➤ sleep(30)[-6,8,0] 𝄀 +0 +select a from nbf order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +7 𝄀 +2 𝄀 +3 +select a from nhf order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +7 𝄀 +2 𝄀 +3 +select a from ni8 order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +7 𝄀 +2 𝄀 +3 +select a from nu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +7 𝄀 +2 𝄀 +3 +drop table nbf; +drop table nhf; +drop table ni8; +drop table nu8; diff --git a/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_narrow_base_async.sql b/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_narrow_base_async.sql new file mode 100644 index 0000000000000..30e53262729e8 --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_narrow_base_async.sql @@ -0,0 +1,85 @@ +-- ivfflat NATIVE narrow base columns (vecbf16 / vecf16 / vecint8 / vecuint8) over the +-- ASYNC (ISCP/CDC) maintenance path. The synchronous CREATE INDEX on a narrow base +-- already works (vector_ivf_quantization.sql); this proves the ONGOING CDC delta path +-- carries a narrow base column too. +-- +-- The fix this guards: the ISCP row pipeline (pkg/iscp/util.go) extracts a narrow base +-- column to its native Go slice ([]types.Float16 / []types.BF16 / []int8 / []uint8) and +-- serializes it back into the IvfflatSqlWriter VALUES tuple as CAST('[...]' as vecXXX(n)). +-- Before the fix only vecf32/vecf64 had extract + serialize cases, so any DML on a +-- narrow-base ivfflat index errored ("extractRowFromVector: unsupported type") and the +-- CDC consumer could never apply the delta. +-- +-- Each index is built and maintained entirely by the CDC consumer (first iteration runs +-- the InitSQL build, later inserts/updates ride the delta path). Shared sleep(30) windows +-- let the 10s-tick consumer settle for all four indexes at once. +-- +-- Two well-separated clusters [1..5]/[50..54] and cluster-center query points keep every +-- top-k distance distinct (no ties) so the result is deterministic for every narrow type; +-- integer values 1..55 are exact in bf16/f16 and in int8/uint8 range, so the narrow base +-- stores them without ambiguity. Queries are `ORDER BY l2_distance LIMIT k` with no +-- secondary sort key so the ivfflat index pushdown (ivf_search) actually fires. +SET probe_limit=10; + +create table nbf(a int primary key, v vecbf16(4)); +insert into nbf values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xbf using ivfflat on nbf(v) lists=2 op_type 'vector_l2_ops' ASYNC; + +create table nhf(a int primary key, v vecf16(4)); +insert into nhf values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xhf using ivfflat on nhf(v) lists=2 op_type 'vector_l2_ops' ASYNC; + +create table ni8(a int primary key, v vecint8(4)); +insert into ni8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xi8 using ivfflat on ni8(v) lists=2 op_type 'vector_l2_ops' ASYNC; + +create table nu8(a int primary key, v vecuint8(4)); +insert into nu8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xu8 using ivfflat on nu8(v) lists=2 op_type 'vector_l2_ops' ASYNC; + +-- 1) initial async build (CDC reindex InitSQL). Low cluster -> 1,2,3 ; high -> 6,5,4. +select sleep(30); +select a from nbf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from nbf order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from nhf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from nhf order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from ni8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from ni8 order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from nu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from nu8 order by l2_distance(v,'[54,54,54,54]') limit 3; + +-- 2) incremental rows ride the CDC delta path through the narrow-base extract + +-- serialize. Row 7=[2,2,2,2] joins the low cluster (now 1,7,2) and row 8=[53,53,53,53] +-- the high cluster (now 6,8,5) -- their appearance proves the delta path indexed them. +insert into nbf values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into nhf values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into ni8 values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into nu8 values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +select sleep(30); +select a from nbf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from nbf order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from nhf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from nhf order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from ni8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from ni8 order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from nu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from nu8 order by l2_distance(v,'[54,54,54,54]') limit 3; + +-- 3) update row 1 across to the other cluster ([1,1,1,1] -> [55,55,55,55]); the delta path +-- must re-extract + re-bucket the narrow vector. Query the LOW cluster center [1,1,1,1]: +-- row 1 has LEFT it, so the top-3 is now 7,2,3 (row 1 absent) -- proving the delta UPDATE +-- moved it. +update nbf set v = '[55,55,55,55]' where a = 1; +update nhf set v = '[55,55,55,55]' where a = 1; +update ni8 set v = '[55,55,55,55]' where a = 1; +update nu8 set v = '[55,55,55,55]' where a = 1; +select sleep(30); +select a from nbf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from nhf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from ni8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from nu8 order by l2_distance(v,'[1,1,1,1]') limit 3; + +drop table nbf; +drop table nhf; +drop table ni8; +drop table nu8; diff --git a/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_quant_async.result b/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_quant_async.result new file mode 100644 index 0000000000000..6484c6f334bae --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_quant_async.result @@ -0,0 +1,134 @@ +SET probe_limit=10; +create table qi8(a int primary key, v vecf32(4)); +insert into qi8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xi8 using ivfflat on qi8(v) lists=2 op_type 'vector_l2_ops' quantization 'int8' ASYNC; +create table qu8(a int primary key, v vecf32(4)); +insert into qu8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xu8 using ivfflat on qu8(v) lists=2 op_type 'vector_l2_ops' quantization 'uint8' ASYNC; +create table qbf(a int primary key, v vecf32(4)); +insert into qbf values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xbf using ivfflat on qbf(v) lists=2 op_type 'vector_l2_ops' quantization 'bf16' ASYNC; +create table qf(a int primary key, v vecf32(4)); +insert into qf values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xf using ivfflat on qf(v) lists=2 op_type 'vector_l2_ops' quantization 'float16' ASYNC; +select sleep(30); +sleep(30) +0 +select a from qi8 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +select a from qi8 order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +5 +4 +select a from qu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +select a from qu8 order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +5 +4 +select a from qbf order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +select a from qbf order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +5 +4 +select a from qf order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +select a from qf order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +5 +4 +insert into qi8 values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into qu8 values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into qbf values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into qf values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +select sleep(30); +sleep(30) +0 +select a from qi8 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +7 +2 +select a from qi8 order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +8 +5 +select a from qu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +7 +2 +select a from qu8 order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +8 +5 +select a from qbf order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +7 +2 +select a from qbf order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +8 +5 +select a from qf order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +7 +2 +select a from qf order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +8 +5 +update qi8 set v = '[55,55,55,55]' where a = 1; +update qu8 set v = '[55,55,55,55]' where a = 1; +update qbf set v = '[55,55,55,55]' where a = 1; +update qf set v = '[55,55,55,55]' where a = 1; +select sleep(30); +sleep(30) +0 +select a from qi8 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +7 +2 +3 +select a from qu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +7 +2 +3 +select a from qbf order by l2_distance(v,'[1,1,1,1]') limit 3; +a +7 +2 +3 +select a from qf order by l2_distance(v,'[1,1,1,1]') limit 3; +a +7 +2 +3 +drop table qi8; +drop table qu8; +drop table qbf; +drop table qf; diff --git a/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_quant_async.sql b/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_quant_async.sql new file mode 100644 index 0000000000000..18ab53bd826f6 --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_quant_async.sql @@ -0,0 +1,81 @@ +-- ivfflat QUANTIZATION over the ASYNC (ISCP/CDC) maintenance path, for all four +-- narrow entry types: +-- * int8 / uint8 — the scaled quantizer: the CDC delta path (toIvfflatUpsert) +-- must re-apply the trained [min,max] q(x)=x*mul+add (int8 -> +-- [-128,127], uint8 -> [0,255]), not an identity cast; +-- * bf16 / float16 — lossless narrowing cast on the entry projection. +-- Every async index is built and maintained entirely by the CDC consumer (the +-- first iteration runs ALTER ... REINDEX ... FORCE_SYNC, later inserts/updates +-- ride the delta path). Three shared sleep(30) windows let the 10s-tick consumer +-- settle for all indexes at once. +-- +-- The queries are `ORDER BY l2_distance(v, q) LIMIT k` with NO secondary sort key, +-- so the ivfflat index pushdown fires (the ivf_search table function), actually +-- exercising the quantized re-rank. Two well-separated clusters [1..5]/[50..54] +-- and cluster-center query points keep every top-k distance distinct (no ties), +-- so the result is deterministic without a tiebreaker. +SET probe_limit=10; + +create table qi8(a int primary key, v vecf32(4)); +insert into qi8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xi8 using ivfflat on qi8(v) lists=2 op_type 'vector_l2_ops' quantization 'int8' ASYNC; + +create table qu8(a int primary key, v vecf32(4)); +insert into qu8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xu8 using ivfflat on qu8(v) lists=2 op_type 'vector_l2_ops' quantization 'uint8' ASYNC; + +create table qbf(a int primary key, v vecf32(4)); +insert into qbf values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xbf using ivfflat on qbf(v) lists=2 op_type 'vector_l2_ops' quantization 'bf16' ASYNC; + +create table qf(a int primary key, v vecf32(4)); +insert into qf values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xf using ivfflat on qf(v) lists=2 op_type 'vector_l2_ops' quantization 'float16' ASYNC; + +-- 1) initial async build (CDC reindex InitSQL). Low cluster -> 1,2,3 ; high -> 6,5,4. +select sleep(30); +select a from qi8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qi8 order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from qu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qu8 order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from qbf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qbf order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from qf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qf order by l2_distance(v,'[54,54,54,54]') limit 3; + +-- 2) incremental rows ride the CDC delta path (toIvfflatUpsert). Row 7=[2,2,2,2] +-- joins the low cluster (now 1,7,2) and row 8=[53,53,53,53] the high cluster +-- (now 6,8,5) -- their appearance proves the delta path indexed them. +insert into qi8 values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into qu8 values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into qbf values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into qf values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +select sleep(30); +select a from qi8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qi8 order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from qu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qu8 order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from qbf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qbf order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from qf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qf order by l2_distance(v,'[54,54,54,54]') limit 3; + +-- 3) update row 1 across to the other cluster ([1,1,1,1] -> [55,55,55,55]); the +-- delta path must re-quantize + re-bucket it. Query the LOW cluster center +-- [1,1,1,1]: row 1 has LEFT it, so the top-3 is now 7,2,3 (row 1 absent) -- +-- proving the delta UPDATE moved it. (Querying the high side instead would tie +-- under quantization: 55 clamps to the same code as 54.) +update qi8 set v = '[55,55,55,55]' where a = 1; +update qu8 set v = '[55,55,55,55]' where a = 1; +update qbf set v = '[55,55,55,55]' where a = 1; +update qf set v = '[55,55,55,55]' where a = 1; +select sleep(30); +select a from qi8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qbf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qf order by l2_distance(v,'[1,1,1,1]') limit 3; + +drop table qi8; +drop table qu8; +drop table qbf; +drop table qf; diff --git a/test/distributed/cases/vector/vector_ivf_mode.result b/test/distributed/cases/vector/vector_ivf_mode.result index 57f91db520454..64ce79e4c0bb9 100644 --- a/test/distributed/cases/vector/vector_ivf_mode.result +++ b/test/distributed/cases/vector/vector_ivf_mode.result @@ -372,22 +372,22 @@ id8 semantic item 0.7760798852372132 id1 hello world 1.0551303640156267 id2 greeting message 1.1550324705439516 id7 random note 1.191049991561001 -(SELECT id, text AS content, l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') AS dist +(SELECT id, text AS content, round(l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]'),4) AS dist FROM mini_vector_data ORDER BY id, l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') LIMIT 2 by rank with option 'mode=pre') UNION -(SELECT id, content, cosine_distance(embedding, '[0.9,0.8,0.7,0.6,0.5,0.4,0.3,0.2]') AS dist +(SELECT id, content, round(cosine_distance(embedding, '[0.9,0.8,0.7,0.6,0.5,0.4,0.3,0.2]'),4) AS dist FROM mini_embed_data ORDER BY cosine_distance(embedding, '[0.9,0.8,0.7,0.6,0.5,0.4,0.3,0.2]') LIMIT 2 by rank with option 'mode=pre') -ORDER BY id +ORDER BY dist, id LIMIT 4; id content dist id03 it stores high dimensional vectors 0.0 -id02 sql is structured query language 0.246478870511055 -id10 additional entry 1.4459599256515503 -id1 hello world 1.5163443088531494 +id02 sql is structured query language 0.2465 +id10 additional entry 1.446 +id1 hello world 1.5163 (SELECT id, category, l2_distance(vec, '[0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]') AS dist FROM vec_with_multi_idx WHERE category = 'A' AND status = 1 diff --git a/test/distributed/cases/vector/vector_ivf_mode.sql b/test/distributed/cases/vector/vector_ivf_mode.sql index 14f54a7a76b3a..b1e5d7e62bc72 100644 --- a/test/distributed/cases/vector/vector_ivf_mode.sql +++ b/test/distributed/cases/vector/vector_ivf_mode.sql @@ -293,16 +293,21 @@ UNION ORDER BY dist LIMIT 4; -- Test Case: UNION with mode=pre on different tables (mini_vector_data and mini_embed_data) -(SELECT id, text AS content, l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') AS dist - FROM mini_vector_data +-- round(dist,4): an exact-match cosine distance is 0 on scalar but ~1.1e-16 with the +-- arch-specific SIMD kernels (FMA); the result comparator treats 0-vs-nonzero as a hard +-- mismatch, so round it. round() wraps only the projection, leaving the ORDER BY on raw +-- distance, so the ivfflat index is still used. ORDER BY dist, id makes the outer row +-- order deterministic (it was under-determined for this UNION shape with ORDER BY id). +(SELECT id, text AS content, round(l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]'),4) AS dist + FROM mini_vector_data ORDER BY id, l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') LIMIT 2 by rank with option 'mode=pre') UNION -(SELECT id, content, cosine_distance(embedding, '[0.9,0.8,0.7,0.6,0.5,0.4,0.3,0.2]') AS dist - FROM mini_embed_data - ORDER BY cosine_distance(embedding, '[0.9,0.8,0.7,0.6,0.5,0.4,0.3,0.2]') +(SELECT id, content, round(cosine_distance(embedding, '[0.9,0.8,0.7,0.6,0.5,0.4,0.3,0.2]'),4) AS dist + FROM mini_embed_data + ORDER BY cosine_distance(embedding, '[0.9,0.8,0.7,0.6,0.5,0.4,0.3,0.2]') LIMIT 2 by rank with option 'mode=pre') -ORDER BY id +ORDER BY dist, id LIMIT 4; -- Test Case: UNION with mode=pre and complex WHERE conditions diff --git a/test/distributed/cases/vector/vector_ivf_quant_ddl.result b/test/distributed/cases/vector/vector_ivf_quant_ddl.result new file mode 100644 index 0000000000000..d5efd3aa7533b --- /dev/null +++ b/test/distributed/cases/vector/vector_ivf_quant_ddl.result @@ -0,0 +1,75 @@ +drop database if exists ivfqddl; +create database ivfqddl; +use ivfqddl; +create table q(a int primary key, v vecf32(4)); +insert into q values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index qi8 using ivfflat on q(v) lists=2 op_type 'vector_l2_ops' quantization 'int8'; +select a from q order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +select a from q order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +5 +4 +alter table q alter reindex qi8 ivfflat lists=2; +select a from q order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +select a from q order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +5 +4 +create table qc clone q; +select a from qc order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +select a from qc order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +5 +4 +drop snapshot if exists ivfqsp; +create snapshot ivfqsp for account sys; +drop database if exists ivfqddl2; +create database ivfqddl2 clone ivfqddl {snapshot='ivfqsp'}; +select a from ivfqddl2.q order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +select a from ivfqddl2.q order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +5 +4 +drop snapshot ivfqsp; +drop database ivfqddl2; +alter table q add column note varchar(10) default 'x'; +select a from q order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +alter table q drop index qi8; +create index qi8b using ivfflat on q(v) lists=2 op_type 'vector_l2_ops' quantization 'int8'; +select a from q order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +select a from q order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +5 +4 +drop table qc; +drop table q; +drop database ivfqddl; diff --git a/test/distributed/cases/vector/vector_ivf_quant_ddl.sql b/test/distributed/cases/vector/vector_ivf_quant_ddl.sql new file mode 100644 index 0000000000000..d92e270f1695c --- /dev/null +++ b/test/distributed/cases/vector/vector_ivf_quant_ddl.sql @@ -0,0 +1,54 @@ +-- ivfflat int8 QUANTIZATION across the DDL / maintenance matrix: +-- reindex, table clone, snapshot + db clone, alter table, drop index/table. +-- Two well-separated clusters [1..] and [50..]; queries probe each side and +-- must keep returning the right cluster after every operation (the int8 codes +-- and trained bounds must survive each path, not silently fall back to a raw +-- identity cast). +drop database if exists ivfqddl; +create database ivfqddl; +use ivfqddl; + +create table q(a int primary key, v vecf32(4)); +insert into q values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index qi8 using ivfflat on q(v) lists=2 op_type 'vector_l2_ops' quantization 'int8'; + +-- baseline: query near each cluster +select a from q order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from q order by l2_distance(v,'[54,54,54,54]') limit 3; + +-- 1) ALTER REINDEX: sync rebuild re-applies the quantizer + re-trains bounds. +alter table q alter reindex qi8 ivfflat lists=2; +select a from q order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from q order by l2_distance(v,'[54,54,54,54]') limit 3; + +-- 2) CLONE table: block-level physical copy of entries/centroids/metadata. +create table qc clone q; +select a from qc order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qc order by l2_distance(v,'[54,54,54,54]') limit 3; + +-- 3) SNAPSHOT + db clone-from-snapshot: RestoreTable path (empty seed, +-- block clone, FORCE_SYNC reindex InitSQL). +drop snapshot if exists ivfqsp; +create snapshot ivfqsp for account sys; +drop database if exists ivfqddl2; +create database ivfqddl2 clone ivfqddl {snapshot='ivfqsp'}; +select a from ivfqddl2.q order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from ivfqddl2.q order by l2_distance(v,'[54,54,54,54]') limit 3; +drop snapshot ivfqsp; +drop database ivfqddl2; + +-- 4) ALTER TABLE add a non-vector column: index must keep working. +alter table q add column note varchar(10) default 'x'; +select a from q order by l2_distance(v,'[1,1,1,1]') limit 3; + +-- 5) DROP INDEX then recreate int8 on the same column. +alter table q drop index qi8; +create index qi8b using ivfflat on q(v) lists=2 op_type 'vector_l2_ops' quantization 'int8'; +select a from q order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from q order by l2_distance(v,'[54,54,54,54]') limit 3; + +-- 6) DROP TABLE (clone first, then base). +drop table qc; +drop table q; + +drop database ivfqddl; diff --git a/test/distributed/cases/vector/vector_ivf_quant_upcast.result b/test/distributed/cases/vector/vector_ivf_quant_upcast.result new file mode 100644 index 0000000000000..e34aa152c4578 --- /dev/null +++ b/test/distributed/cases/vector/vector_ivf_quant_upcast.result @@ -0,0 +1,18 @@ +drop database if exists ivf_qup; +create database ivf_qup; +use ivf_qup; +create table i8(a int, v vecint8(4)); +create table bf(a int, v vecbf16(4)); +create table hf(a int, v vecf16(4)); +create table u8(a int, v vecuint8(4)); +create index x using ivfflat on i8(v) lists=1 op_type 'vector_l2_ops' quantization 'float32'; +not supported: ivfflat QUANTIZATION 'float32' (4 bytes/element) cannot upcast base column VECINT8 (1 bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type +create index x using ivfflat on i8(v) lists=1 op_type 'vector_l2_ops' quantization 'bf16'; +not supported: ivfflat QUANTIZATION 'bf16' (2 bytes/element) cannot upcast base column VECINT8 (1 bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type +create index x using ivfflat on bf(v) lists=1 op_type 'vector_l2_ops' quantization 'float32'; +not supported: ivfflat QUANTIZATION 'float32' (4 bytes/element) cannot upcast base column VECBF16 (2 bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type +create index x using ivfflat on hf(v) lists=1 op_type 'vector_l2_ops' quantization 'float32'; +not supported: ivfflat QUANTIZATION 'float32' (4 bytes/element) cannot upcast base column VECF16 (2 bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type +create index x using ivfflat on u8(v) lists=1 op_type 'vector_l2_ops' quantization 'float16'; +not supported: ivfflat QUANTIZATION 'float16' (2 bytes/element) cannot upcast base column VECUINT8 (1 bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type +drop database ivf_qup; diff --git a/test/distributed/cases/vector/vector_ivf_quant_upcast.sql b/test/distributed/cases/vector/vector_ivf_quant_upcast.sql new file mode 100644 index 0000000000000..264877db258ce --- /dev/null +++ b/test/distributed/cases/vector/vector_ivf_quant_upcast.sql @@ -0,0 +1,23 @@ +-- ivfflat QUANTIZATION is downcast-only. A narrow base column (vecbf16/vecf16/ +-- vecint8/vecuint8) with a QUANTIZATION wider than the base element is rejected at +-- plan time -- it would store upcast entries for no precision gain and force the +-- f32 distance kernel over narrow data. Regression for the schema.go upcast guard. +-- Equal-width / narrower quantization (and omitting it) are allowed. All cases here +-- fail before any index build, so no GPU is required. +drop database if exists ivf_qup; +create database ivf_qup; +use ivf_qup; +create table i8(a int, v vecint8(4)); +create table bf(a int, v vecbf16(4)); +create table hf(a int, v vecf16(4)); +create table u8(a int, v vecuint8(4)); +-- int8 base (1 byte) + wider quantization -> rejected +create index x using ivfflat on i8(v) lists=1 op_type 'vector_l2_ops' quantization 'float32'; +create index x using ivfflat on i8(v) lists=1 op_type 'vector_l2_ops' quantization 'bf16'; +-- bf16 base (2 bytes) + float32 (4 bytes) -> rejected +create index x using ivfflat on bf(v) lists=1 op_type 'vector_l2_ops' quantization 'float32'; +-- f16 base (2 bytes) + float32 (4 bytes) -> rejected +create index x using ivfflat on hf(v) lists=1 op_type 'vector_l2_ops' quantization 'float32'; +-- uint8 base (1 byte) + float16 (2 bytes) -> rejected +create index x using ivfflat on u8(v) lists=1 op_type 'vector_l2_ops' quantization 'float16'; +drop database ivf_qup; diff --git a/test/distributed/cases/vector/vector_ivf_quantization.result b/test/distributed/cases/vector/vector_ivf_quantization.result new file mode 100644 index 0000000000000..14cbe5732026f --- /dev/null +++ b/test/distributed/cases/vector/vector_ivf_quantization.result @@ -0,0 +1,82 @@ +drop database if exists ivfq; +create database ivfq; +use ivfq; +create table tbf16(a int primary key, v vecbf16(4)); +insert into tbf16 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index i_bf16 using ivfflat on tbf16(v) lists=2 op_type 'vector_l2_ops'; +select a from tbf16 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +create table tf16(a int primary key, v vecf16(4)); +insert into tf16 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index i_f16 using ivfflat on tf16(v) lists=2 op_type 'vector_l2_ops'; +select a from tf16 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +create table ti8(a int primary key, v vecint8(4)); +insert into ti8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index i_i8 using ivfflat on ti8(v) lists=2 op_type 'vector_l2_ops'; +select a from ti8 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +create table q32(a int primary key, v vecf32(4)); +insert into q32 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index q32f16 using ivfflat on q32(v) lists=2 op_type 'vector_l2_ops' quantization 'float16'; +select a from q32 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +alter table q32 drop index q32f16; +create index q32bf16 using ivfflat on q32(v) lists=2 op_type 'vector_l2_ops' quantization 'bf16'; +select a from q32 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +alter table q32 drop index q32bf16; +create index q32i8 using ivfflat on q32(v) lists=2 op_type 'vector_l2_ops' quantization 'int8'; +select a from q32 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +create table q64(a int primary key, v vecf64(4)); +insert into q64 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index q64f32 using ivfflat on q64(v) lists=2 op_type 'vector_l2_ops' quantization 'float32'; +select a from q64 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +alter table q64 drop index q64f32; +create index q64i8 using ivfflat on q64(v) lists=2 op_type 'vector_l2_ops' quantization 'int8'; +select a from q64 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +alter table q32 drop index q32i8; +create index q32u8 using ivfflat on q32(v) lists=2 op_type 'vector_l2_ops' quantization 'uint8'; +select a from q32 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +alter table q64 drop index q64i8; +create index q64u8 using ivfflat on q64(v) lists=2 op_type 'vector_l2_ops' quantization 'uint8'; +select a from q64 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +create table qbad(a int primary key, v vecf32(4)); +create index qb using ivfflat on qbad(v) lists=1 op_type 'vector_l2_ops' quantization 'int16'; +internal error: ivfflat: unsupported quantization 'int16' (supported: 'float32', 'float16', 'bf16', 'int8', 'uint8') +drop database ivfq; diff --git a/test/distributed/cases/vector/vector_ivf_quantization.sql b/test/distributed/cases/vector/vector_ivf_quantization.sql new file mode 100644 index 0000000000000..0709e854e2ada --- /dev/null +++ b/test/distributed/cases/vector/vector_ivf_quantization.sql @@ -0,0 +1,46 @@ +-- ivfflat: narrow base (direct match T->T) and QUANTIZATION down-cast of f32/f64 +-- bases to f32/f16/bf16/int8 entries. Two well-separated clusters (1..5 and 50..54) +-- make the top-3 stable under quantization; query an exact cluster-A point. +drop database if exists ivfq; +create database ivfq; +use ivfq; +create table tbf16(a int primary key, v vecbf16(4)); +insert into tbf16 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index i_bf16 using ivfflat on tbf16(v) lists=2 op_type 'vector_l2_ops'; +select a from tbf16 order by l2_distance(v,'[1,1,1,1]') limit 3; +create table tf16(a int primary key, v vecf16(4)); +insert into tf16 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index i_f16 using ivfflat on tf16(v) lists=2 op_type 'vector_l2_ops'; +select a from tf16 order by l2_distance(v,'[1,1,1,1]') limit 3; +create table ti8(a int primary key, v vecint8(4)); +insert into ti8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index i_i8 using ivfflat on ti8(v) lists=2 op_type 'vector_l2_ops'; +select a from ti8 order by l2_distance(v,'[1,1,1,1]') limit 3; +create table q32(a int primary key, v vecf32(4)); +insert into q32 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index q32f16 using ivfflat on q32(v) lists=2 op_type 'vector_l2_ops' quantization 'float16'; +select a from q32 order by l2_distance(v,'[1,1,1,1]') limit 3; +alter table q32 drop index q32f16; +create index q32bf16 using ivfflat on q32(v) lists=2 op_type 'vector_l2_ops' quantization 'bf16'; +select a from q32 order by l2_distance(v,'[1,1,1,1]') limit 3; +alter table q32 drop index q32bf16; +create index q32i8 using ivfflat on q32(v) lists=2 op_type 'vector_l2_ops' quantization 'int8'; +select a from q32 order by l2_distance(v,'[1,1,1,1]') limit 3; +create table q64(a int primary key, v vecf64(4)); +insert into q64 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index q64f32 using ivfflat on q64(v) lists=2 op_type 'vector_l2_ops' quantization 'float32'; +select a from q64 order by l2_distance(v,'[1,1,1,1]') limit 3; +alter table q64 drop index q64f32; +create index q64i8 using ivfflat on q64(v) lists=2 op_type 'vector_l2_ops' quantization 'int8'; +select a from q64 order by l2_distance(v,'[1,1,1,1]') limit 3; +-- uint8 QUANTIZATION (unsigned [0,255]) on f32 and f64 bases +alter table q32 drop index q32i8; +create index q32u8 using ivfflat on q32(v) lists=2 op_type 'vector_l2_ops' quantization 'uint8'; +select a from q32 order by l2_distance(v,'[1,1,1,1]') limit 3; +alter table q64 drop index q64i8; +create index q64u8 using ivfflat on q64(v) lists=2 op_type 'vector_l2_ops' quantization 'uint8'; +select a from q64 order by l2_distance(v,'[1,1,1,1]') limit 3; +-- an unsupported quantization name still errors +create table qbad(a int primary key, v vecf32(4)); +create index qb using ivfflat on qbad(v) lists=1 op_type 'vector_l2_ops' quantization 'int16'; +drop database ivfq; diff --git a/test/distributed/cases/vector/vector_reindex_options.result b/test/distributed/cases/vector/vector_reindex_options.result index 180d0f9b7489b..0c1aed165fe9e 100644 --- a/test/distributed/cases/vector/vector_reindex_options.result +++ b/test/distributed/cases/vector/vector_reindex_options.result @@ -19,6 +19,12 @@ alter table ivf_t alter reindex idx ivfflat ef_construction=200; not supported: ALTER REINDEX option "ef_construction" on a ivfflat index alter table ivf_t alter reindex idx ivfflat graph_degree=64; not supported: ALTER REINDEX option "graph_degree" on a ivfflat index +alter table ivf_t alter reindex idx ivfflat quantization 'Float16'; +show create table ivf_t; +Table Create Table +ivf_t CREATE TABLE `ivf_t` (\n `a` int NOT NULL,\n `b` vecf32(4) DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `idx` USING ivfflat (`b`) lists = 4 op_type 'vector_l2_ops' quantization 'float16' kmeans_train_percent = 80 kmeans_max_iteration = 50 \n) +alter table ivf_t alter reindex idx ivfflat quantization 'garbage'; +not supported: ivfflat quantization "garbage" (supported: float32, float16, bf16, int8, uint8) create table hnsw_t(a bigint primary key, b vecf32(4)); insert into hnsw_t values(1,"[1,2,3,4]"),(2,"[5,6,7,8]"),(3,"[9,10,11,12]"),(4,"[2,1,4,3]"); create index hidx using hnsw on hnsw_t(b) op_type "vector_l2_ops" m=48 ef_construction=64 ef_search=64; diff --git a/test/distributed/cases/vector/vector_reindex_options.sql b/test/distributed/cases/vector/vector_reindex_options.sql index dc45c03dd9228..4087f49cfe25b 100644 --- a/test/distributed/cases/vector/vector_reindex_options.sql +++ b/test/distributed/cases/vector/vector_reindex_options.sql @@ -31,6 +31,12 @@ alter table ivf_t alter reindex idx ivfflat m=16; alter table ivf_t alter reindex idx ivfflat ef_construction=200; alter table ivf_t alter reindex idx ivfflat graph_degree=64; +-- IVF-FLAT honors quantization (narrow-type entries); value is normalized to +-- lowercase and an unsupported name is rejected. +alter table ivf_t alter reindex idx ivfflat quantization 'Float16'; +show create table ivf_t; +alter table ivf_t alter reindex idx ivfflat quantization 'garbage'; + -- ---------------------------------------------------------------------------- -- HNSW: honors m + ef_construction + ef_search + max_index_capacity -- ---------------------------------------------------------------------------- diff --git a/test/distributed/gpu_cases/README.md b/test/distributed/gpu_cases/README.md index fe2e7eccffcdd..295ee559f9ac4 100644 --- a/test/distributed/gpu_cases/README.md +++ b/test/distributed/gpu_cases/README.md @@ -11,6 +11,8 @@ CPU-only BVT run is not gated on a GPU. | `vector_ivfpq.sql` | IVF-PQ | `gpu_cases/vector/` | sync CREATE INDEX, DDL surface, exact-match search, drop/recreate lifecycle | | `vector_cagra_quantization.sql` | CAGRA | `gpu_cases/vector/` | `QUANTIZATION 'float16'`, `'int8'` and `'uint8'` — each round-trips through the catalog + exact-match search | | `vector_ivfpq_quantization.sql` | IVF-PQ | `gpu_cases/vector/` | `QUANTIZATION 'float16'`, `'int8'` and `'uint8'` — each round-trips through the catalog + exact-match search | +| `vector_cagra_f16.sql` | CAGRA | `gpu_cases/vector/` | **vecf16 BASE column** (native half end-to-end): direct (half storage), `QUANTIZATION 'int8'`/`'uint8'` (native half→int8/uint8, no f32 detour) — catalog round-trip + exact-match search; query cast to vecf16(8) | +| `vector_ivfpq_f16.sql` | IVF-PQ | `gpu_cases/vector/` | same vecf16 BASE coverage as `vector_cagra_f16.sql` | | `vector_pairwise_scan.sql` | (none) | `gpu_cases/vector/` | GPU **pairwise distance** on a NON-INDEX table scan: `ORDER BY l2_distance/l2_distance_sq/cosine_distance(col, query)` over 10k×128 SIFT rows routes the batch through `metric.PairwiseDistanceLaunch` (exact, deterministic) | | `vector_pairwise_mode.sql` | (none) | `gpu_cases/vector/` | same non-index pairwise scan run under **`gpu_mode=1` (GPU) and `gpu_mode=0` (CPU)** for l2/l2sq/cosine/**inner_product** — results are byte-identical (GPU==CPU), and inner_product shows the negated score | | `vector_ivfflat_mode.sql` | IVF-FLAT | `gpu_cases/vector/` | IVF-FLAT search under **`gpu_mode=1`/`0`** — the productl2 centroid-assignment brute-force (GPU vs CPU) returns identical results | @@ -19,7 +21,9 @@ CPU-only BVT run is not gated on a GPU. | `vector_ivfpq_metric.sql` | IVF-PQ | `gpu_cases/vector/` | same per-metric build/search/score coverage as `vector_cagra_metric.sql` | | `vector_cagra_filter.sql` | CAGRA | `gpu_cases/vector/` | **INCLUDE-column pre-filter** across all 4 supported INCLUDE types — `INCLUDE (c_i32 int, c_i64 bigint, c_f32 float, c_f64 double)`; single- and multi-column `WHERE` predicates are pushed into the GPU search (predsJSON) and restrict the ANN candidate set — verifies both columns round-trip and the filter changes the nearest neighbor | | `vector_ivfpq_filter.sql` | IVF-PQ | `gpu_cases/vector/` | same 4-type INCLUDE pre-filter coverage as `vector_cagra_filter.sql` | -| `vector_gpu_negative.sql` | CAGRA + IVF-PQ | `gpu_cases/vector/` | **validation guard rails** (expected errors): `op_type 'vector_l1_ops'` / unknown op_type rejected, `vecf64` column rejected, `QUANTIZATION 'float64'` rejected, **VARCHAR INCLUDE column** rejected, search dimension-mismatch rejected | +| `vector_cagra_postfilter.sql` | CAGRA | `gpu_cases/vector/` | **post-filter on a NON-INCLUDE column** — a `WHERE` on a column absent from `INCLUDE` cannot be GPU-pushed, so the planner runs the ANN search for a candidate window then JOINs+filters at the DB. Verifies the post-filtered result equals the unfiltered ranked result ∩ predicate (exact when `LIMIT` ≥ rows so the window covers all), plus the mixed pre+post case and the small-`LIMIT` approximate window | +| `vector_ivfpq_postfilter.sql` | IVF-PQ | `gpu_cases/vector/` | same non-INCLUDE post-filter coverage as `vector_cagra_postfilter.sql` | +| `vector_gpu_negative.sql` | CAGRA + IVF-PQ | `gpu_cases/vector/` | **validation guard rails** (expected errors): `op_type 'vector_l1_ops'` / unknown op_type rejected, `vecf64` column rejected, `QUANTIZATION 'float64'` rejected, **VARCHAR INCLUDE column** rejected, search dimension-mismatch rejected, **`vecbf16` base column rejected**, **`vecf16` base + `QUANTIZATION 'float32'` upcast rejected** | | `vector_cagra_delete.sql` | CAGRA | `gpu_cases/pessimistic_transaction/vector/` | **soft-delete**: `DELETE` a row, after CDC catch-up search excludes it and returns the next survivor (per-device deleted bitset) | | `vector_ivfpq_delete.sql` | IVF-PQ | `gpu_cases/pessimistic_transaction/vector/` | same soft-delete coverage as `vector_cagra_delete.sql` | | `vector_cagra_ddl.sql` | CAGRA | `gpu_cases/pessimistic_transaction/vector/` | **DDL/DML lifecycle** on an indexed table: ALTER ADD/DROP COLUMN, TRUNCATE, re-INSERT, reindex — each table-rewrite triggers a CDC rebuild (SLEEP(30)) after which search recovers | diff --git a/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_cagra_f16_async.result b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_cagra_f16_async.result new file mode 100644 index 0000000000000..9396df8d10a5a --- /dev/null +++ b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_cagra_f16_async.result @@ -0,0 +1,63 @@ +SET experimental_cagra_index = 1; +SET cagra_threads_build = 7; +SET cagra_max_index_capacity = 99999; +drop database if exists cagra_f16_cdc; +create database cagra_f16_cdc; +use cagra_f16_cdc; +create table t (id bigint primary key, v vecf16(8)); +insert into t values +(1, '[1,1,1,1,1,1,1,1]'), (2, '[2,2,2,2,2,2,2,2]'), +(3, '[3,3,3,3,3,3,3,3]'), (4, '[4,4,4,4,4,4,4,4]'), +(5, '[5,5,5,5,5,5,5,5]'), (6, '[6,6,6,6,6,6,6,6]'), +(7, '[7,7,7,7,7,7,7,7]'), (8, '[8,8,8,8,8,8,8,8]'), +(9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'); +create index ix using cagra on t (v) +op_type 'vector_l2_ops' intermediate_graph_degree=8 graph_degree=4 ASYNC; +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' async quantization 'float32' distribution_mode 'single' intermediate_graph_degree = 8 graph_degree = 4 +) +insert into t values (100, '[100,100,100,100,100,100,100,100]'); +insert into t values (105, '[105,105,105,105,105,105,105,105]'); +insert into t values (300, '[300,300,300,300,300,300,300,300]'); +insert into t values (700, '[700,700,700,700,700,700,700,700]'); +delete from t where id=105; +delete from t where id=3; +update t set v = '[500,500,500,500,500,500,500,500]' where id=5; +update t set v = '[305,305,305,305,305,305,305,305]' where id=300; +insert into t values (800, '[800,800,800,800,800,800,800,800]'); +select sleep(30); +➤ sleep(30)[-6,8,0] 𝄀 +0 +select id from t order by l2_distance(v, cast('[100,100,100,100,100,100,100,100]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +100 +select id from t order by l2_distance(v, cast('[700,700,700,700,700,700,700,700]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +700 +select id from t order by l2_distance(v, cast('[800,800,800,800,800,800,800,800]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +800 +select id from t order by l2_distance(v, cast('[500,500,500,500,500,500,500,500]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t order by l2_distance(v, cast('[305,305,305,305,305,305,305,305]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +300 +select id from t order by l2_distance(v, cast('[105,105,105,105,105,105,105,105]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +100 +set @stbl = (select index_table_name from mo_catalog.mo_indexes +where table_id=(select rel_id from mo_catalog.mo_tables +where relname='t' and reldatabase='cagra_f16_cdc') +and name='ix' and algo_table_type='cagra_index'); +set @q = concat('select distinct tag from `', @stbl, '` order by tag'); +prepare s from @q; execute s; deallocate prepare s; +select count(*) from t; +➤ count(*)[-5,64,0] 𝄀 +13 +drop database cagra_f16_cdc; diff --git a/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_cagra_f16_async.sql b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_cagra_f16_async.sql new file mode 100644 index 0000000000000..a34b88ec51357 --- /dev/null +++ b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_cagra_f16_async.sql @@ -0,0 +1,96 @@ +-- ===================================================================== +-- vector_cagra_f16_async.sql — CAGRA vecf16 base + ISCP CDC INSERT/DELETE/UPDATE +-- +-- GPU REQUIRED. The vecf16 twin of vector_cagra_async.sql. It proves the +-- ongoing CDC ingestion path carries a vecf16 base column NATIVELY (2 bytes +-- per element, no f32 widening): the iscp CuvsCdcWriter extracts the source +-- column as []types.Float16, encodes each event record at 2*dim bytes, and +-- CagraSync.AppendRecords steps the stream by that same width; the search-side +-- replayEventChunks[cuvs.Float16] then reinterprets the bytes back to half and +-- feeds the f16 brute-force overflow. +-- +-- ASYNC CREATE INDEX: cagra_create is deferred to the first CDC iteration +-- (stashed as ConsumerInfo.InitSQL). The 10 initial rows build the tag=0 +-- model (native half); every DML below rides the CDC tail into the tag=1 +-- overflow. Like vector_cagra_async / vector_hnsw_async, ALL ops run first, +-- then a single SELECT SLEEP(30) lets the 10s-interval consumer apply the +-- whole batch, and only then do we search. +-- +-- The query literal is cast to vecf16(8) so the native half query path is +-- exercised (CagraSearch over base B == cuvs.Float16). +-- +-- Determinism notes (CAGRA is an approximate index; values are exact in f16): +-- * Every sentinel value (100..800, 305, 500, 105) is an integer < 2048, so +-- it is represented exactly in float16 — no rounding ambiguity. +-- * Exact-match probes always return that row as top-1 — verifies INSERT and +-- UPDATE (new vec replaces old). +-- * The deleted-sentinel probe [105,...] resolves to id=100: once id=105 is +-- gone, id=100 lives in the exact f16 overflow and is the unique nearest +-- neighbor by a wide margin, stable regardless of graph approximation. +-- * id=3 (deleted) is verified via COUNT(*) only — its integer neighbors are +-- L2-equidistant, so a search over them would not be reproducible. +-- ===================================================================== + +SET experimental_cagra_index = 1; +SET cagra_threads_build = 7; +SET cagra_max_index_capacity = 99999; + +drop database if exists cagra_f16_cdc; +create database cagra_f16_cdc; +use cagra_f16_cdc; + +create table t (id bigint primary key, v vecf16(8)); +insert into t values + (1, '[1,1,1,1,1,1,1,1]'), (2, '[2,2,2,2,2,2,2,2]'), + (3, '[3,3,3,3,3,3,3,3]'), (4, '[4,4,4,4,4,4,4,4]'), + (5, '[5,5,5,5,5,5,5,5]'), (6, '[6,6,6,6,6,6,6,6]'), + (7, '[7,7,7,7,7,7,7,7]'), (8, '[8,8,8,8,8,8,8,8]'), + (9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'); + +-- Async build: cagra_create deferred to first CDC iteration via InitSQL. +create index ix using cagra on t (v) + op_type 'vector_l2_ops' intermediate_graph_degree=8 graph_degree=4 ASYNC; +show create table t; + +-- Batch of DML — all applied before any search. The 10 initial rows go +-- through the InitSQL build (tag=0); everything below rides the f16 CDC tail +-- into the tag=1 overflow. +insert into t values (100, '[100,100,100,100,100,100,100,100]'); +insert into t values (105, '[105,105,105,105,105,105,105,105]'); +insert into t values (300, '[300,300,300,300,300,300,300,300]'); +insert into t values (700, '[700,700,700,700,700,700,700,700]'); +delete from t where id=105; +delete from t where id=3; +update t set v = '[500,500,500,500,500,500,500,500]' where id=5; +update t set v = '[305,305,305,305,305,305,305,305]' where id=300; +insert into t values (800, '[800,800,800,800,800,800,800,800]'); + +-- Single wait for the whole batch to flow through CDC. +select sleep(30); + +-- Surviving inserted sentinels — exact match → that row. +select id from t order by l2_distance(v, cast('[100,100,100,100,100,100,100,100]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[700,700,700,700,700,700,700,700]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[800,800,800,800,800,800,800,800]' as vecf16(8))) limit 1; + +-- Updated rows — exact match on the NEW value returns the moved row. +select id from t order by l2_distance(v, cast('[500,500,500,500,500,500,500,500]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[305,305,305,305,305,305,305,305]' as vecf16(8))) limit 1; + +-- Deleted sentinel — probe its old value; id=105 is gone so the unique +-- nearest survivor id=100 (exact f16 overflow) comes back instead. +select id from t order by l2_distance(v, cast('[105,105,105,105,105,105,105,105]' as vecf16(8))) limit 1; + +-- Storage layout: tag=0 (model from initial build) + tag=1 (CDC overflow). +set @stbl = (select index_table_name from mo_catalog.mo_indexes + where table_id=(select rel_id from mo_catalog.mo_tables + where relname='t' and reldatabase='cagra_f16_cdc') + and name='ix' and algo_table_type='cagra_index'); +set @q = concat('select distinct tag from `', @stbl, '` order by tag'); +prepare s from @q; execute s; deallocate prepare s; + +-- Row count: 10 initial + 5 inserts (100,105,300,700,800) - 2 deletes +-- (105,3) = 13. Confirms id=3 and id=105 are gone. +select count(*) from t; + +drop database cagra_f16_cdc; diff --git a/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_int8_overflow_scale.result b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_int8_overflow_scale.result new file mode 100644 index 0000000000000..aa8bab83c0d5e --- /dev/null +++ b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_int8_overflow_scale.result @@ -0,0 +1,60 @@ +SET experimental_cagra_index = 1; +SET experimental_ivfpq_index = 1; +SET cagra_threads_build = 7; +drop database if exists int8scale_cagra; +create database int8scale_cagra; +use int8scale_cagra; +create table t (id bigint primary key, v vecf32(8)); +insert into t values +(1,'[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]'),(2,'[0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4]'), +(3,'[0.6,0.6,0.6,0.6,0.6,0.6,0.6,0.6]'),(4,'[0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8]'), +(5,'[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]'),(6,'[1.2,1.2,1.2,1.2,1.2,1.2,1.2,1.2]'), +(7,'[1.4,1.4,1.4,1.4,1.4,1.4,1.4,1.4]'),(8,'[1.6,1.6,1.6,1.6,1.6,1.6,1.6,1.6]'), +(9,'[1.8,1.8,1.8,1.8,1.8,1.8,1.8,1.8]'),(10,'[2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0]'); +create index ix using cagra on t (v) op_type 'vector_l2_ops' +intermediate_graph_degree=8 graph_degree=4 itopk_size=16 QUANTIZATION 'int8'; +select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 1; +➤ id[-5,64,0] 𝄀 +1 +insert into t values (999, '[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]'); +select sleep(30); +➤ sleep(30)[-6,8,0] 𝄀 +0 +select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 1; +➤ id[-5,64,0] 𝄀 +1 +select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 3; +➤ id[-5,64,0] 𝄀 +1 𝄀 +2 𝄀 +3 +drop database int8scale_cagra; +drop database if exists int8scale_ivfpq; +create database int8scale_ivfpq; +use int8scale_ivfpq; +create table t (id bigint primary key, v vecf32(8)); +insert into t values +(1,'[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]'),(2,'[0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4]'), +(3,'[0.6,0.6,0.6,0.6,0.6,0.6,0.6,0.6]'),(4,'[0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8]'), +(5,'[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]'),(6,'[1.2,1.2,1.2,1.2,1.2,1.2,1.2,1.2]'), +(7,'[1.4,1.4,1.4,1.4,1.4,1.4,1.4,1.4]'),(8,'[1.6,1.6,1.6,1.6,1.6,1.6,1.6,1.6]'), +(9,'[1.8,1.8,1.8,1.8,1.8,1.8,1.8,1.8]'),(10,'[2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0]'); +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' +lists=2 m=2 bits_per_code=8 QUANTIZATION 'uint8'; +insert into t values (999, '[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]'); +select sleep(30); +➤ sleep(30)[-6,8,0] 𝄀 +0 +select count(*) as top1_is_overflow from +(select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 1) x +where x.id = 999; +➤ top1_is_overflow[-5,64,0] 𝄀 +0 +select count(*) as top3_has_overflow from +(select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 3) x +where x.id = 999; +➤ top3_has_overflow[-5,64,0] 𝄀 +0 +drop database int8scale_ivfpq; +SET experimental_cagra_index = 0; +SET experimental_ivfpq_index = 0; diff --git a/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_int8_overflow_scale.sql b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_int8_overflow_scale.sql new file mode 100644 index 0000000000000..77c442ff47b89 --- /dev/null +++ b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_int8_overflow_scale.sql @@ -0,0 +1,79 @@ +-- ===================================================================== +-- vector_int8_overflow_scale.sql — int8/uint8 quantized index + CDC overflow +-- must merge on the SAME distance scale (break-review CRITICAL fix). +-- +-- GPU REQUIRED. A 1-byte (int8/uint8) main index computes L2 over the quantized +-- vectors, i.e. scalar^2 * true_L2 (scalar = 255/(max-min)). The base-typed CDC +-- overflow brute force computes true (base-scale) L2. Before the fix, +-- mergeMultiResults compared the two raw, so a moderately-distant overflow row +-- (small base-scale distance) out-ranked the true-nearest main row (large +-- scalar^2-scaled distance). The fix dequantizes the main distances by +-- 1/scalar^2 inside transform_distance so both tiers share the base scale. +-- +-- Values in [0.2, 2.0] => scalar ~= 255/1.8 ~= 141, scalar^2 ~= 20000. Main rows +-- id 1..10 at [i*0.2]*8. Overflow row id 999 at [1.0]*8 (true dist 4.5 from the +-- query, ~225x farther than id 1). Query [0.25]*8: the TRUE nearest is id 1 +-- (dist ~0.02). Correct top-1 (with the fix) is 1; the bug returned 999. +-- ===================================================================== + +SET experimental_cagra_index = 1; +SET experimental_ivfpq_index = 1; +SET cagra_threads_build = 7; + +-- --------------------------------------------------------------------- +-- CAGRA, QUANTIZATION 'int8' +-- --------------------------------------------------------------------- +drop database if exists int8scale_cagra; +create database int8scale_cagra; +use int8scale_cagra; +create table t (id bigint primary key, v vecf32(8)); +insert into t values + (1,'[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]'),(2,'[0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4]'), + (3,'[0.6,0.6,0.6,0.6,0.6,0.6,0.6,0.6]'),(4,'[0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8]'), + (5,'[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]'),(6,'[1.2,1.2,1.2,1.2,1.2,1.2,1.2,1.2]'), + (7,'[1.4,1.4,1.4,1.4,1.4,1.4,1.4,1.4]'),(8,'[1.6,1.6,1.6,1.6,1.6,1.6,1.6,1.6]'), + (9,'[1.8,1.8,1.8,1.8,1.8,1.8,1.8,1.8]'),(10,'[2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0]'); +create index ix using cagra on t (v) op_type 'vector_l2_ops' + intermediate_graph_degree=8 graph_degree=4 itopk_size=16 QUANTIZATION 'int8'; +-- no overflow yet -> id 1 +select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 1; +-- add the overflow row, then the true nearest is STILL id 1 (not 999) +insert into t values (999, '[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]'); +select sleep(30); +select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 1; +select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 3; +drop database int8scale_cagra; + +-- --------------------------------------------------------------------- +-- IVF-PQ, QUANTIZATION 'uint8' (same scale issue; +128 shift also cancels in L2) +-- --------------------------------------------------------------------- +drop database if exists int8scale_ivfpq; +create database int8scale_ivfpq; +use int8scale_ivfpq; +create table t (id bigint primary key, v vecf32(8)); +insert into t values + (1,'[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]'),(2,'[0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4]'), + (3,'[0.6,0.6,0.6,0.6,0.6,0.6,0.6,0.6]'),(4,'[0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8]'), + (5,'[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]'),(6,'[1.2,1.2,1.2,1.2,1.2,1.2,1.2,1.2]'), + (7,'[1.4,1.4,1.4,1.4,1.4,1.4,1.4,1.4]'),(8,'[1.6,1.6,1.6,1.6,1.6,1.6,1.6,1.6]'), + (9,'[1.8,1.8,1.8,1.8,1.8,1.8,1.8,1.8]'),(10,'[2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0]'); +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' + lists=2 m=2 bits_per_code=8 QUANTIZATION 'uint8'; +insert into t values (999, '[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]'); +select sleep(30); +-- IVF-PQ recall on this tiny PQ config picks among the near main ids (1..4) +-- non-deterministically, so the exact top-1 is not stable. The scale bug is +-- about the moderately-distant overflow row 999 (true dist 4.5) out-ranking the +-- near main rows, so assert the stable invariant directly: 999 must NEVER appear +-- in the top results. Pre-fix this returned 1 (999 was top-1); with the +-- dequant fix it is 0. +select count(*) as top1_is_overflow from + (select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 1) x + where x.id = 999; +select count(*) as top3_has_overflow from + (select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 3) x + where x.id = 999; +drop database int8scale_ivfpq; + +SET experimental_cagra_index = 0; +SET experimental_ivfpq_index = 0; diff --git a/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_async.result b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_async.result index f14dd68f70aff..9016ce842452c 100644 --- a/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_async.result +++ b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_async.result @@ -21,7 +21,7 @@ t ¦ CREATE TABLE `t` ( `id` bigint NOT NULL, `v` vecf32(8) DEFAULT NULL, PRIMARY KEY (`id`), - KEY `ix` USING ivfpq (`v`) lists = 2 m = 2 op_type 'vector_l2_ops' quantization 'float32' distribution_mode 'single' bits_per_code = 8 + KEY `ix` USING ivfpq (`v`) lists = 2 m = 2 op_type 'vector_l2_ops' async quantization 'float32' distribution_mode 'single' bits_per_code = 8 ) insert into t values (100, '[100,100,100,100,100,100,100,100]'); insert into t values (105, '[105,105,105,105,105,105,105,105]'); diff --git a/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_f16_async.result b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_f16_async.result new file mode 100644 index 0000000000000..ca5395ded29b7 --- /dev/null +++ b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_f16_async.result @@ -0,0 +1,65 @@ +SET experimental_ivfpq_index = 1; +SET ivfpq_threads_build = 6; +SET ivfpq_max_index_capacity = 99999; +SET kmeans_train_percent = 37; +SET kmeans_max_iteration = 12; +drop database if exists ivfpq_f16_cdc; +create database ivfpq_f16_cdc; +use ivfpq_f16_cdc; +create table t (id bigint primary key, v vecf16(8)); +insert into t values +(1, '[1,1,1,1,1,1,1,1]'), (2, '[2,2,2,2,2,2,2,2]'), +(3, '[3,3,3,3,3,3,3,3]'), (4, '[4,4,4,4,4,4,4,4]'), +(5, '[5,5,5,5,5,5,5,5]'), (6, '[6,6,6,6,6,6,6,6]'), +(7, '[7,7,7,7,7,7,7,7]'), (8, '[8,8,8,8,8,8,8,8]'), +(9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'); +create index ix using ivfpq on t (v) +op_type 'vector_l2_ops' lists=2 m=2 bits_per_code=8 ASYNC; +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 2 m = 2 op_type 'vector_l2_ops' async quantization 'float32' distribution_mode 'single' bits_per_code = 8 +) +insert into t values (100, '[100,100,100,100,100,100,100,100]'); +insert into t values (105, '[105,105,105,105,105,105,105,105]'); +insert into t values (300, '[300,300,300,300,300,300,300,300]'); +insert into t values (700, '[700,700,700,700,700,700,700,700]'); +delete from t where id=105; +delete from t where id=3; +update t set v = '[500,500,500,500,500,500,500,500]' where id=5; +update t set v = '[305,305,305,305,305,305,305,305]' where id=300; +insert into t values (800, '[800,800,800,800,800,800,800,800]'); +select sleep(30); +➤ sleep(30)[-6,8,0] 𝄀 +0 +select id from t order by l2_distance(v, cast('[100,100,100,100,100,100,100,100]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +100 +select id from t order by l2_distance(v, cast('[700,700,700,700,700,700,700,700]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +700 +select id from t order by l2_distance(v, cast('[800,800,800,800,800,800,800,800]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +800 +select id from t order by l2_distance(v, cast('[500,500,500,500,500,500,500,500]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t order by l2_distance(v, cast('[305,305,305,305,305,305,305,305]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +300 +select id from t order by l2_distance(v, cast('[105,105,105,105,105,105,105,105]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +100 +set @stbl = (select index_table_name from mo_catalog.mo_indexes +where table_id=(select rel_id from mo_catalog.mo_tables +where relname='t' and reldatabase='ivfpq_f16_cdc') +and name='ix' and algo_table_type='ivfpq_index'); +set @q = concat('select distinct tag from `', @stbl, '` order by tag'); +prepare s from @q; execute s; deallocate prepare s; +select count(*) from t; +➤ count(*)[-5,64,0] 𝄀 +13 +drop database ivfpq_f16_cdc; diff --git a/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_f16_async.sql b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_f16_async.sql new file mode 100644 index 0000000000000..53d62f760559c --- /dev/null +++ b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_f16_async.sql @@ -0,0 +1,99 @@ +-- ===================================================================== +-- vector_ivfpq_f16_async.sql — IVF-PQ vecf16 base + ISCP CDC INSERT/DELETE/UPDATE +-- +-- GPU REQUIRED. The vecf16 twin of vector_ivfpq_async.sql. It proves the +-- ongoing CDC ingestion path carries a vecf16 base column NATIVELY (2 bytes +-- per element, no f32 widening): the iscp CuvsCdcWriter extracts the source +-- column as []types.Float16, encodes each event record at 2*dim bytes, and +-- IvfpqSync.AppendRecords steps the stream by that same width; the search-side +-- replayEventChunks[cuvs.Float16] then reinterprets the bytes back to half and +-- feeds the f16 brute-force overflow. +-- +-- ASYNC CREATE INDEX: ivfpq_create is deferred to the first CDC iteration +-- (stashed as ConsumerInfo.InitSQL). The 10 initial rows build the tag=0 +-- model (native half); every DML below rides the CDC tail into the tag=1 +-- overflow. Like vector_ivfpq_async / vector_hnsw_async, ALL ops run first, +-- then a single SELECT SLEEP(30) lets the 10s-interval consumer apply the +-- whole batch, and only then do we search. +-- +-- The query literal is cast to vecf16(8) so the native half query path is +-- exercised (IvfpqSearch over base B == cuvs.Float16). +-- +-- Determinism notes (IVF-PQ is a quantized + clustered approximate index; +-- values are exact in f16): +-- * Every sentinel value (100..800, 305, 500, 105) is an integer < 2048, so +-- it is represented exactly in float16 — no rounding ambiguity. +-- * Exact-match probes always return that row as top-1 — verifies INSERT and +-- UPDATE (new vec replaces old). +-- * The deleted-sentinel probe [105,...] resolves to id=100: once id=105 is +-- gone, id=100 lives in the exact f16 overflow and is the unique nearest +-- neighbor by a wide margin, stable regardless of PQ approximation. +-- * id=3 (deleted) is verified via COUNT(*) only — its integer neighbors are +-- L2-equidistant, so a search over them would not be reproducible. +-- ===================================================================== + +SET experimental_ivfpq_index = 1; +SET ivfpq_threads_build = 6; +SET ivfpq_max_index_capacity = 99999; +SET kmeans_train_percent = 37; +SET kmeans_max_iteration = 12; + +drop database if exists ivfpq_f16_cdc; +create database ivfpq_f16_cdc; +use ivfpq_f16_cdc; + +create table t (id bigint primary key, v vecf16(8)); +insert into t values + (1, '[1,1,1,1,1,1,1,1]'), (2, '[2,2,2,2,2,2,2,2]'), + (3, '[3,3,3,3,3,3,3,3]'), (4, '[4,4,4,4,4,4,4,4]'), + (5, '[5,5,5,5,5,5,5,5]'), (6, '[6,6,6,6,6,6,6,6]'), + (7, '[7,7,7,7,7,7,7,7]'), (8, '[8,8,8,8,8,8,8,8]'), + (9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'); + +-- Async build: ivfpq_create deferred to first CDC iteration via InitSQL. +create index ix using ivfpq on t (v) + op_type 'vector_l2_ops' lists=2 m=2 bits_per_code=8 ASYNC; +show create table t; + +-- Batch of DML — all applied before any search. The 10 initial rows go +-- through the InitSQL build (tag=0); everything below rides the f16 CDC tail +-- into the tag=1 overflow. +insert into t values (100, '[100,100,100,100,100,100,100,100]'); +insert into t values (105, '[105,105,105,105,105,105,105,105]'); +insert into t values (300, '[300,300,300,300,300,300,300,300]'); +insert into t values (700, '[700,700,700,700,700,700,700,700]'); +delete from t where id=105; +delete from t where id=3; +update t set v = '[500,500,500,500,500,500,500,500]' where id=5; +update t set v = '[305,305,305,305,305,305,305,305]' where id=300; +insert into t values (800, '[800,800,800,800,800,800,800,800]'); + +-- Single wait for the whole batch to flow through CDC. +select sleep(30); + +-- Surviving inserted sentinels — exact match → that row. +select id from t order by l2_distance(v, cast('[100,100,100,100,100,100,100,100]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[700,700,700,700,700,700,700,700]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[800,800,800,800,800,800,800,800]' as vecf16(8))) limit 1; + +-- Updated rows — exact match on the NEW value returns the moved row. +select id from t order by l2_distance(v, cast('[500,500,500,500,500,500,500,500]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[305,305,305,305,305,305,305,305]' as vecf16(8))) limit 1; + +-- Deleted sentinel — probe its old value; id=105 is gone so the unique +-- nearest survivor id=100 (exact f16 overflow) comes back instead. +select id from t order by l2_distance(v, cast('[105,105,105,105,105,105,105,105]' as vecf16(8))) limit 1; + +-- Storage layout: tag=0 (model from initial build) + tag=1 (CDC overflow). +set @stbl = (select index_table_name from mo_catalog.mo_indexes + where table_id=(select rel_id from mo_catalog.mo_tables + where relname='t' and reldatabase='ivfpq_f16_cdc') + and name='ix' and algo_table_type='ivfpq_index'); +set @q = concat('select distinct tag from `', @stbl, '` order by tag'); +prepare s from @q; execute s; deallocate prepare s; + +-- Row count: 10 initial + 5 inserts (100,105,300,700,800) - 2 deletes +-- (105,3) = 13. Confirms id=3 and id=105 are gone. +select count(*) from t; + +drop database ivfpq_f16_cdc; diff --git a/test/distributed/gpu_cases/vector/vector_cagra_f16.result b/test/distributed/gpu_cases/vector/vector_cagra_f16.result new file mode 100644 index 0000000000000..041c967e2d9e9 --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_cagra_f16.result @@ -0,0 +1,119 @@ +SET experimental_cagra_index = 1; +SET cagra_threads_build = 7; +SET cagra_max_index_capacity = 99999; +drop database if exists cagra_f16_direct; +create database cagra_f16_direct; +use cagra_f16_direct; +create table t (id bigint primary key, v vecf16(8)); +insert into t values +( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), +( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), +( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), +( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), +( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), +(11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), +(13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), +(15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), +(17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), +(19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); +create index ix using cagra on t (v) +op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32; +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'float32' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 32 +) +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +1 +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +10 +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +15 +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +20 +drop database cagra_f16_direct; +drop database if exists cagra_f16_int8; +create database cagra_f16_int8; +use cagra_f16_int8; +create table t (id bigint primary key, v vecf16(8)); +insert into t values +( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), +( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), +( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), +( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), +( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), +(11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), +(13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), +(15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), +(17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), +(19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); +create index ix using cagra on t (v) +op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 +QUANTIZATION 'int8'; +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'int8' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 32 +) +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +1 +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +10 +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +15 +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +20 +drop database cagra_f16_int8; +drop database if exists cagra_f16_uint8; +create database cagra_f16_uint8; +use cagra_f16_uint8; +create table t (id bigint primary key, v vecf16(8)); +insert into t values +( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), +( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), +( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), +( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), +( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), +(11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), +(13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), +(15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), +(17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), +(19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); +create index ix using cagra on t (v) +op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 +QUANTIZATION 'uint8'; +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'uint8' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 32 +) +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +1 +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +10 +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +15 +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +20 +drop database cagra_f16_uint8; diff --git a/test/distributed/gpu_cases/vector/vector_cagra_f16.sql b/test/distributed/gpu_cases/vector/vector_cagra_f16.sql new file mode 100644 index 0000000000000..fe1c0c6ab5568 --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_cagra_f16.sql @@ -0,0 +1,120 @@ +-- ===================================================================== +-- vector_cagra_f16.sql — CAGRA over a vecf16 (half) BASE column +-- +-- GPU REQUIRED. Unlike vector_cagra_quantization.sql (vecf32 base, the +-- QUANTIZATION clause only changes internal storage), here the COLUMN itself +-- is vecf16 — the native base/query type is half end-to-end: +-- * direct — no QUANTIZATION: the index stores half natively (Q == base). +-- * int8 — vecf16 base quantized half->int8 via the native half-source +-- scalar quantizer (no f32 detour). +-- * uint8 — same, half->uint8. +-- +-- Three databases, one per storage. Each builds a sync CAGRA index and asserts +-- (a) the vecf16 column + index round-trip through SHOW CREATE TABLE and +-- (b) exact-match search returns the right row. The query literal is cast to +-- vecf16(8) so the half query path is exercised. +-- +-- Determinism: integers 1..20 — every value is exact in half, and the int8/ +-- uint8 quantizer trains on [1,20] so each integer maps to a distinct level; +-- the exact-match probe is always the unique top-1. Do not widen the range +-- under int8/uint8 (adjacent levels would collapse). +-- ===================================================================== + +SET experimental_cagra_index = 1; +SET cagra_threads_build = 7; +SET cagra_max_index_capacity = 99999; + +-- ===================================================================== +-- vecf16 base, direct (no QUANTIZATION — stored as half) +-- ===================================================================== +drop database if exists cagra_f16_direct; +create database cagra_f16_direct; +use cagra_f16_direct; + +create table t (id bigint primary key, v vecf16(8)); +insert into t values + ( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), + ( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), + ( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), + ( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), + ( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), + (11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), + (13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), + (15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), + (17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), + (19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); + +create index ix using cagra on t (v) + op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32; + +show create table t; +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; + +drop database cagra_f16_direct; + +-- ===================================================================== +-- vecf16 base, QUANTIZATION int8 (native half->int8) +-- ===================================================================== +drop database if exists cagra_f16_int8; +create database cagra_f16_int8; +use cagra_f16_int8; + +create table t (id bigint primary key, v vecf16(8)); +insert into t values + ( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), + ( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), + ( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), + ( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), + ( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), + (11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), + (13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), + (15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), + (17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), + (19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); + +create index ix using cagra on t (v) + op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 + QUANTIZATION 'int8'; + +show create table t; +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; + +drop database cagra_f16_int8; + +-- ===================================================================== +-- vecf16 base, QUANTIZATION uint8 (native half->uint8) +-- ===================================================================== +drop database if exists cagra_f16_uint8; +create database cagra_f16_uint8; +use cagra_f16_uint8; + +create table t (id bigint primary key, v vecf16(8)); +insert into t values + ( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), + ( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), + ( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), + ( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), + ( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), + (11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), + (13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), + (15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), + (17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), + (19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); + +create index ix using cagra on t (v) + op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 + QUANTIZATION 'uint8'; + +show create table t; +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; + +drop database cagra_f16_uint8; diff --git a/test/distributed/gpu_cases/vector/vector_cagra_filter.result b/test/distributed/gpu_cases/vector/vector_cagra_filter.result index 20135d00515c3..4d01e9de8e4f3 100644 --- a/test/distributed/gpu_cases/vector/vector_cagra_filter.result +++ b/test/distributed/gpu_cases/vector/vector_cagra_filter.result @@ -44,7 +44,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_filter') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","included_columns":"c_i32,c_i64,c_f32,c_f64","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"cagra_max_index_capacity":{"t":"I","v":99999},"cagra_threads_build":{"t":"I","v":7},"experimental_cagra_index":{"t":"I8","v":1},"lower_case_table_names":{"t":"I","v":1}}}} +cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","included_columns":"c_i32,c_i64,c_f32,c_f64","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; ➤ id[-5,64,0] 𝄀 9 diff --git a/test/distributed/gpu_cases/vector/vector_cagra_filter_quant.result b/test/distributed/gpu_cases/vector/vector_cagra_filter_quant.result new file mode 100644 index 0000000000000..eab105f78ca43 --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_cagra_filter_quant.result @@ -0,0 +1,345 @@ +SET experimental_cagra_index = 1; +SET cagra_threads_build = 7; +SET cagra_max_index_capacity = 99999; +drop database if exists cagra_fq_f16; +create database cagra_fq_f16; +use cagra_fq_f16; +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'float16' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf32(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'float16' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 32 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database cagra_fq_f16; +drop database if exists cagra_fq_int8; +create database cagra_fq_int8; +use cagra_fq_int8; +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'int8' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf32(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'int8' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 32 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database cagra_fq_int8; +drop database if exists cagra_fq_uint8; +create database cagra_fq_uint8; +use cagra_fq_uint8; +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'uint8' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf32(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'uint8' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 32 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database cagra_fq_uint8; +drop database if exists cagra_fq_f16base; +create database cagra_fq_f16base; +use cagra_fq_f16base; +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'float32' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 32 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database cagra_fq_f16base; +drop database if exists cagra_fq_f16int8; +create database cagra_fq_f16int8; +use cagra_fq_f16int8; +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'int8' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'int8' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 32 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database cagra_fq_f16int8; +drop database if exists cagra_fq_f16uint8; +create database cagra_fq_f16uint8; +use cagra_fq_f16uint8; +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'uint8' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'uint8' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 32 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database cagra_fq_f16uint8; diff --git a/test/distributed/gpu_cases/vector/vector_cagra_filter_quant.sql b/test/distributed/gpu_cases/vector/vector_cagra_filter_quant.sql new file mode 100644 index 0000000000000..fad4badfb721e --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_cagra_filter_quant.sql @@ -0,0 +1,295 @@ +-- ===================================================================== +-- vector_cagra_filter_quant.sql — CAGRA INCLUDE-column pre-filter combined +-- with quantization and a vecf16 base column. +-- +-- GPU REQUIRED. vector_cagra_filter.sql already covers the INCLUDE pre-filter +-- over a plain vecf32 base (quantization 'float32'). This file proves the SAME +-- predsJSON pre-filter path stays correct when the index storage is compressed +-- or the base column is half: +-- * f32 base + QUANTIZATION 'float16' — supported (query quantized to T) +-- * f32 base + QUANTIZATION 'int8' — supported (learned scalar quantizer) +-- * f32 base + QUANTIZATION 'uint8' — supported +-- * vecf16 base, direct (no QUANTIZATION) — supported (native half query) +-- * vecf16 base + QUANTIZATION 'int8' + filter — supported (the native half +-- query is quantized to int8 inside +-- cuVS via search_quantize_with_filter) +-- * vecf16 base + QUANTIZATION 'uint8' + filter — supported (same path) +-- Every storage routes the SAME predsJSON pre-filter through the const-B* +-- quantize search, so the expected nearest neighbor per predicate is identical. +-- +-- Data/predicates are identical to vector_cagra_filter.sql so the expected +-- nearest neighbor per predicate is unchanged across every storage: +-- id=i -> [i]*8; c_i32=i, c_i64=i*10, c_f32=i.25, c_f64=i.5 (all monotone). +-- Query [12]*8: +-- * c_i32 < 10 -> id 9 +-- * c_i64 >= 100 -> id 12 +-- * c_f32 > 15.25 -> id 16 +-- * c_f64 = 5.5 -> id 5 +-- * c_i32 >= 10 AND c_f64 < 15.5 -> id 12 +-- * c_i64 < 100 AND c_f32 > 5.25 -> id 9 +-- +-- Determinism note: integers 1..20 are exact in float16 and the int8/uint8 +-- quantizer trains on [1,20] so each integer maps to a distinct level; each +-- predicate band keeps a unique nearest. Do NOT widen the range under int8/ +-- uint8 (adjacent levels would collapse and the top-1 would become ambiguous). +-- ===================================================================== + +SET experimental_cagra_index = 1; +SET cagra_threads_build = 7; +SET cagra_max_index_capacity = 99999; + +-- ===================================================================== +-- f32 base + QUANTIZATION 'float16' + INCLUDE pre-filter +-- ===================================================================== +drop database if exists cagra_fq_f16; +create database cagra_fq_f16; +use cagra_fq_f16; + +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'float16' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; + +drop database cagra_fq_f16; + +-- ===================================================================== +-- f32 base + QUANTIZATION 'int8' + INCLUDE pre-filter +-- ===================================================================== +drop database if exists cagra_fq_int8; +create database cagra_fq_int8; +use cagra_fq_int8; + +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'int8' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; + +drop database cagra_fq_int8; + +-- ===================================================================== +-- f32 base + QUANTIZATION 'uint8' + INCLUDE pre-filter +-- ===================================================================== +drop database if exists cagra_fq_uint8; +create database cagra_fq_uint8; +use cagra_fq_uint8; + +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'uint8' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; + +drop database cagra_fq_uint8; + +-- ===================================================================== +-- vecf16 base, direct (no QUANTIZATION) + INCLUDE pre-filter +-- The query literal is cast to vecf16(8) so the half query path is exercised. +-- ===================================================================== +drop database if exists cagra_fq_f16base; +create database cagra_fq_f16base; +use cagra_fq_f16base; + +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; + +drop database cagra_fq_f16base; + +-- ===================================================================== +-- vecf16 base + QUANTIZATION 'int8' + INCLUDE pre-filter +-- The native half query is quantized to int8 inside cuVS (the const-B* +-- search_quantize_with_filter path); same predicates and nearest neighbors +-- as every storage above. +-- ===================================================================== +drop database if exists cagra_fq_f16int8; +create database cagra_fq_f16int8; +use cagra_fq_f16int8; + +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'int8' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; + +drop database cagra_fq_f16int8; + +-- ===================================================================== +-- vecf16 base + QUANTIZATION 'uint8' + INCLUDE pre-filter (same path as int8) +-- ===================================================================== +drop database if exists cagra_fq_f16uint8; +create database cagra_fq_f16uint8; +use cagra_fq_f16uint8; + +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'uint8' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; + +drop database cagra_fq_f16uint8; diff --git a/test/distributed/gpu_cases/vector/vector_cagra_metric.result b/test/distributed/gpu_cases/vector/vector_cagra_metric.result index 4aa3089270840..1f7dec42ec106 100644 --- a/test/distributed/gpu_cases/vector/vector_cagra_metric.result +++ b/test/distributed/gpu_cases/vector/vector_cagra_metric.result @@ -23,7 +23,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_metric') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"cagra_max_index_capacity":{"t":"I","v":99999},"cagra_threads_build":{"t":"I","v":7},"experimental_cagra_index":{"t":"I8","v":1},"lower_case_table_names":{"t":"I","v":1}}}} +cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id, l2_distance(v, '[16,15,14,13,12,11,10,9]') as score from t order by score asc limit 1; ➤ id[-5,64,0] ¦ score[8,9,0] 𝄀 1 ¦ 0.0 @@ -34,7 +34,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_metric') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2sq_ops","quantization":"float32","session_vars":{"cfg":{"cagra_max_index_capacity":{"t":"I","v":99999},"cagra_threads_build":{"t":"I","v":7},"experimental_cagra_index":{"t":"I8","v":1},"lower_case_table_names":{"t":"I","v":1}}}} +cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2sq_ops","quantization":"float32","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id, l2_distance_sq(v, '[16,15,14,13,12,11,10,9]') as score from t order by score asc limit 1; ➤ id[-5,64,0] ¦ score[8,54,0] 𝄀 1 ¦ 0.0 @@ -45,7 +45,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_metric') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_ip_ops","quantization":"float32","session_vars":{"cfg":{"cagra_max_index_capacity":{"t":"I","v":99999},"cagra_threads_build":{"t":"I","v":7},"experimental_cagra_index":{"t":"I8","v":1},"lower_case_table_names":{"t":"I","v":1}}}} +cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_ip_ops","quantization":"float32","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id, inner_product(v, '[16,15,14,13,12,11,10,9]') as score from t order by score asc limit 1; ➤ id[-5,64,0] ¦ score[8,9,0] 𝄀 1 ¦ -1292.0 @@ -56,7 +56,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_metric') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_cosine_ops","quantization":"float32","session_vars":{"cfg":{"cagra_max_index_capacity":{"t":"I","v":99999},"cagra_threads_build":{"t":"I","v":7},"experimental_cagra_index":{"t":"I8","v":1},"lower_case_table_names":{"t":"I","v":1}}}} +cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_cosine_ops","quantization":"float32","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id, cosine_distance(v, '[16,15,14,13,12,11,10,9]') as score from t order by score asc limit 1; ➤ id[-5,64,0] ¦ score[8,9,0] 𝄀 1 ¦ -1.1920928955078125E-7 diff --git a/test/distributed/gpu_cases/vector/vector_cagra_postfilter.result b/test/distributed/gpu_cases/vector/vector_cagra_postfilter.result new file mode 100644 index 0000000000000..08801f85f71ef --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_cagra_postfilter.result @@ -0,0 +1,89 @@ +SET experimental_cagra_index = 1; +SET cagra_threads_build = 7; +SET cagra_max_index_capacity = 99999; +drop database if exists cagra_postfilter; +create database cagra_postfilter; +use cagra_postfilter; +create table t (id bigint primary key, v vecf32(8), c_inc int, c_noinc int); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 101), +(2, '[2,2,2,2,2,2,2,2]', 2, 102), +(3, '[3,3,3,3,3,3,3,3]', 3, 103), +(4, '[4,4,4,4,4,4,4,4]', 4, 104), +(5, '[5,5,5,5,5,5,5,5]', 5, 105), +(6, '[6,6,6,6,6,6,6,6]', 6, 106), +(7, '[7,7,7,7,7,7,7,7]', 7, 107), +(8, '[8,8,8,8,8,8,8,8]', 8, 108), +(9, '[9,9,9,9,9,9,9,9]', 9, 109), +(10, '[10,10,10,10,10,10,10,10]', 10, 110), +(11, '[11,11,11,11,11,11,11,11]', 11, 111), +(12, '[12,12,12,12,12,12,12,12]', 12, 112), +(13, '[13,13,13,13,13,13,13,13]', 13, 113), +(14, '[14,14,14,14,14,14,14,14]', 14, 114), +(15, '[15,15,15,15,15,15,15,15]', 15, 115), +(16, '[16,16,16,16,16,16,16,16]', 16, 116), +(17, '[17,17,17,17,17,17,17,17]', 17, 117), +(18, '[18,18,18,18,18,18,18,18]', 18, 118), +(19, '[19,19,19,19,19,19,19,19]', 19, 119), +(20, '[20,20,20,20,20,20,20,20]', 20, 120); +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=64 INCLUDE (c_inc); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf32(8) DEFAULT NULL, + `c_inc` int DEFAULT NULL, + `c_noinc` int DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'float32' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 64 INCLUDE (c_inc) +) +select id, c_noinc from t order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] ¦ c_noinc[4,32,0] 𝄀 +12 ¦ 112 𝄀 +13 ¦ 113 𝄀 +11 ¦ 111 𝄀 +10 ¦ 110 𝄀 +14 ¦ 114 𝄀 +9 ¦ 109 𝄀 +15 ¦ 115 𝄀 +16 ¦ 116 𝄀 +8 ¦ 108 𝄀 +7 ¦ 107 𝄀 +17 ¦ 117 𝄀 +6 ¦ 106 𝄀 +18 ¦ 118 𝄀 +5 ¦ 105 𝄀 +19 ¦ 119 𝄀 +20 ¦ 120 𝄀 +4 ¦ 104 𝄀 +3 ¦ 103 𝄀 +2 ¦ 102 𝄀 +1 ¦ 101 +select id from t where c_noinc < 105 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] 𝄀 +4 𝄀 +3 𝄀 +2 𝄀 +1 +select id from t where c_noinc >= 116 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] 𝄀 +16 𝄀 +17 𝄀 +18 𝄀 +19 𝄀 +20 +select id from t where c_noinc = 102 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] 𝄀 +2 +select id from t where c_inc <= 14 and c_noinc >= 108 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] 𝄀 +12 𝄀 +13 𝄀 +11 𝄀 +10 𝄀 +14 𝄀 +9 𝄀 +8 +select id from t where c_noinc = 102 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] +drop database cagra_postfilter; diff --git a/test/distributed/gpu_cases/vector/vector_cagra_postfilter.sql b/test/distributed/gpu_cases/vector/vector_cagra_postfilter.sql new file mode 100644 index 0000000000000..d894215970443 --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_cagra_postfilter.sql @@ -0,0 +1,75 @@ +-- ===================================================================== +-- vector_cagra_postfilter.sql — CAGRA search filtering on a NON-INCLUDE column +-- +-- GPU REQUIRED. A WHERE predicate on a column that is NOT in the index INCLUDE +-- list cannot be pushed into the GPU bitset pre-filter. Instead the planner runs +-- the ANN search to get a candidate window, then JOINs + filters the predicate +-- at the database (post-filter). See the plan: cagra_search (candidate window) +-- INNER JOIN (table scan Filter: ) -> Sort -> Limit. +-- +-- Methodology: first take the UNFILTERED ranked result, then verify the +-- post-filtered result equals exactly the unfiltered rows that satisfy the +-- predicate. The candidate window grows with the query LIMIT, so a LIMIT >= row +-- count makes the window cover every row -> the post-filter is exact. +-- +-- Data: id=i -> [i]*8; c_inc=i (INCLUDE, int), c_noinc=100+i (NOT included). +-- Query [12]*8. With LIMIT 20 the window is all 20 rows, so post-filter is exact: +-- * c_noinc < 105 -> i in 1..4 -> 4,3,2,1 (nearest-first) +-- * c_noinc >= 116 -> i in 16..20 -> 16,17,18,19,20 +-- * c_noinc = 102 -> id 2 (far row still found, full window) +-- * c_inc <= 14 (PRE) AND c_noinc >= 108 (POST) -> i in 8..14 +-- ===================================================================== + +SET experimental_cagra_index = 1; +SET cagra_threads_build = 7; +SET cagra_max_index_capacity = 99999; + +drop database if exists cagra_postfilter; +create database cagra_postfilter; +use cagra_postfilter; + +create table t (id bigint primary key, v vecf32(8), c_inc int, c_noinc int); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 101), + (2, '[2,2,2,2,2,2,2,2]', 2, 102), + (3, '[3,3,3,3,3,3,3,3]', 3, 103), + (4, '[4,4,4,4,4,4,4,4]', 4, 104), + (5, '[5,5,5,5,5,5,5,5]', 5, 105), + (6, '[6,6,6,6,6,6,6,6]', 6, 106), + (7, '[7,7,7,7,7,7,7,7]', 7, 107), + (8, '[8,8,8,8,8,8,8,8]', 8, 108), + (9, '[9,9,9,9,9,9,9,9]', 9, 109), + (10, '[10,10,10,10,10,10,10,10]', 10, 110), + (11, '[11,11,11,11,11,11,11,11]', 11, 111), + (12, '[12,12,12,12,12,12,12,12]', 12, 112), + (13, '[13,13,13,13,13,13,13,13]', 13, 113), + (14, '[14,14,14,14,14,14,14,14]', 14, 114), + (15, '[15,15,15,15,15,15,15,15]', 15, 115), + (16, '[16,16,16,16,16,16,16,16]', 16, 116), + (17, '[17,17,17,17,17,17,17,17]', 17, 117), + (18, '[18,18,18,18,18,18,18,18]', 18, 118), + (19, '[19,19,19,19,19,19,19,19]', 19, 119), + (20, '[20,20,20,20,20,20,20,20]', 20, 120); + +-- Only c_inc is pushed into the GPU pre-filter; c_noinc is post-filtered. +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=64 INCLUDE (c_inc); + +show create table t; + +-- (1) UNFILTERED ranked baseline (window covers all rows at LIMIT 20). +select id, c_noinc from t order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; + +-- (2) POST-FILTER on the non-INCLUDE column — must equal the baseline rows that +-- satisfy the predicate, in the same distance order. +select id from t where c_noinc < 105 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +select id from t where c_noinc >= 116 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +select id from t where c_noinc = 102 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; + +-- (3) MIXED: c_inc is pushed (pre-filter), c_noinc is post-filtered. Both apply. +select id from t where c_inc <= 14 and c_noinc >= 108 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; + +-- (4) Small LIMIT shrinks the candidate window: a far post-filter match (id 2, +-- c_noinc=102) falls outside the window and is not returned (approximate). +select id from t where c_noinc = 102 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; + +drop database cagra_postfilter; diff --git a/test/distributed/gpu_cases/vector/vector_cagra_quantization.result b/test/distributed/gpu_cases/vector/vector_cagra_quantization.result index 5fb295dc10b53..1b9462fc33ed4 100644 --- a/test/distributed/gpu_cases/vector/vector_cagra_quantization.result +++ b/test/distributed/gpu_cases/vector/vector_cagra_quantization.result @@ -32,7 +32,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_q_f16') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float16","session_vars":{"cfg":{"cagra_max_index_capacity":{"t":"I","v":99999},"cagra_threads_build":{"t":"I","v":7},"experimental_cagra_index":{"t":"I8","v":1},"lower_case_table_names":{"t":"I","v":1}}}} +cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float16","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 @@ -77,7 +77,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_q_int8') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"int8","session_vars":{"cfg":{"cagra_max_index_capacity":{"t":"I","v":99999},"cagra_threads_build":{"t":"I","v":7},"experimental_cagra_index":{"t":"I8","v":1},"lower_case_table_names":{"t":"I","v":1}}}} +cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"int8","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 @@ -122,7 +122,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_q_uint8') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"uint8","session_vars":{"cfg":{"cagra_max_index_capacity":{"t":"I","v":99999},"cagra_threads_build":{"t":"I","v":7},"experimental_cagra_index":{"t":"I8","v":1},"lower_case_table_names":{"t":"I","v":1}}}} +cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"uint8","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 diff --git a/test/distributed/gpu_cases/vector/vector_cagra_replicated.result b/test/distributed/gpu_cases/vector/vector_cagra_replicated.result index 3c812efe292c7..558f4206f5e66 100644 --- a/test/distributed/gpu_cases/vector/vector_cagra_replicated.result +++ b/test/distributed/gpu_cases/vector/vector_cagra_replicated.result @@ -33,7 +33,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_replicated') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"replicated","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float32"} +cagra ¦ cagra_index ¦ {"distribution_mode":"replicated","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 diff --git a/test/distributed/gpu_cases/vector/vector_cagra_sharded.result b/test/distributed/gpu_cases/vector/vector_cagra_sharded.result index 1ff22c7fa8a39..61548c7deea79 100644 --- a/test/distributed/gpu_cases/vector/vector_cagra_sharded.result +++ b/test/distributed/gpu_cases/vector/vector_cagra_sharded.result @@ -87,7 +87,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_sharded') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"sharded","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float32"} +cagra ¦ cagra_index ¦ {"distribution_mode":"sharded","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 diff --git a/test/distributed/gpu_cases/vector/vector_gpu_negative.result b/test/distributed/gpu_cases/vector/vector_gpu_negative.result index 5214d70421899..03b13ddea0c79 100644 --- a/test/distributed/gpu_cases/vector/vector_gpu_negative.result +++ b/test/distributed/gpu_cases/vector/vector_gpu_negative.result @@ -18,15 +18,49 @@ internal error: invalid op_type. 'vector_l1_ops' create index ix using cagra on t (v) op_type 'vector_bogus_ops'; internal error: invalid op_type. 'vector_bogus_ops' create index ixf using cagra on tf (v) op_type 'vector_l2_ops'; -not supported: Cagra only supports VECF32 column types +not supported: Cagra only supports VECF32 / VECF16 base column types create index ixf using ivfpq on tf (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8; -not supported: IvfPQ only supports VECF32 column types +not supported: IvfPQ only supports VECF32 / VECF16 base column types create index ixq using cagra on t (v) op_type 'vector_l2_ops' QUANTIZATION 'float64'; internal error: invalid quantization. quantization is invalid. f32, f16, int8, uint8 +create index ixbq using cagra on t (v) op_type 'vector_l2_ops' QUANTIZATION 'bf16'; +not supported: Cagra does not support 'bf16' quantization (no GPU bfloat16 storage); use 'float16', 'int8', or 'uint8' +create index ixbq using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'bf16'; +not supported: IvfPQ does not support 'bf16' quantization (no GPU bfloat16 storage); use 'float16', 'int8', or 'uint8' create index ixv using cagra on t (v) op_type 'vector_l2_ops' INCLUDE (lbl); not supported: INCLUDE column 'lbl' has unsupported type VARCHAR (supported: int32, int64, float32, float64) create index ixok using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=8 graph_degree=4 itopk_size=16; select id from t order by l2_distance(v, '[1,2,3]') asc limit 1; invalid input: vector ops between different dimensions (8, 3) is not permitted. +create table tbf (id bigint primary key, v vecbf16(8)); +create index ixbf using cagra on tbf (v) op_type 'vector_l2_ops'; +not supported: Cagra only supports VECF32 / VECF16 base column types +create index ixbf using ivfpq on tbf (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8; +not supported: IvfPQ only supports VECF32 / VECF16 base column types +create table th (id bigint primary key, v vecf16(8)); +create index ixup using cagra on th (v) op_type 'vector_l2_ops' QUANTIZATION 'float32'; +not supported: Cagra QUANTIZATION 'float32' (4 bytes/element) cannot upcast base column VECF16 (2 bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type +create index ixup using ivfpq on th (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'float32'; +not supported: IvfPQ QUANTIZATION 'float32' (4 bytes/element) cannot upcast base column VECF16 (2 bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type +create index ixqi using cagra on t (v) op_type 'vector_ip_ops' QUANTIZATION 'int8'; +not supported: cagra quantization "int8" is only supported with L2 (op_type 'vector_l2_ops'); the int8/uint8 affine quantizer does not preserve inner-product / cosine geometry +create index ixqi using cagra on t (v) op_type 'vector_cosine_ops' QUANTIZATION 'int8'; +not supported: cagra quantization "int8" is only supported with L2 (op_type 'vector_l2_ops'); the int8/uint8 affine quantizer does not preserve inner-product / cosine geometry +create index ixqi using ivfpq on t (v) op_type 'vector_ip_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'uint8'; +not supported: ivfpq quantization "uint8" is only supported with L2 (op_type 'vector_l2_ops'); the int8/uint8 affine quantizer does not preserve inner-product / cosine geometry +create table tre (id bigint primary key, v vecf32(8)); +insert into tre values +(1, '[1,1,1,1,1,1,1,1]'), (2, '[2,2,2,2,2,2,2,2]'), (3, '[3,3,3,3,3,3,3,3]'), +(4, '[4,4,4,4,4,4,4,4]'), (5, '[5,5,5,5,5,5,5,5]'), (6, '[6,6,6,6,6,6,6,6]'), +(7, '[7,7,7,7,7,7,7,7]'), (8, '[8,8,8,8,8,8,8,8]'), (9, '[9,9,9,9,9,9,9,9]'), +(10, '[10,10,10,10,10,10,10,10]'); +create index ixre using cagra on tre (v) op_type 'vector_ip_ops' +intermediate_graph_degree=8 graph_degree=4 itopk_size=16; +alter table tre alter reindex ixre cagra QUANTIZATION 'int8'; +not supported: cagra quantization "int8" is only supported with L2 (op_type 'vector_l2_ops'); the int8/uint8 affine quantizer does not preserve inner-product / cosine geometry +alter table tre alter reindex ixre cagra QUANTIZATION 'bf16'; +not supported: cagra quantization "bf16" (supported: float32, float16, int8, uint8) +alter table tre alter reindex ixre cagra QUANTIZATION 'float64'; +not supported: cagra quantization "float64" (supported: float32, float16, int8, uint8) drop database gpu_negative; diff --git a/test/distributed/gpu_cases/vector/vector_gpu_negative.sql b/test/distributed/gpu_cases/vector/vector_gpu_negative.sql index d6f86f61bd9cf..238a2c7ce9270 100644 --- a/test/distributed/gpu_cases/vector/vector_gpu_negative.sql +++ b/test/distributed/gpu_cases/vector/vector_gpu_negative.sql @@ -9,6 +9,11 @@ -- * op_type 'vector_bogus_ops' — unknown op_type -- * vecf64 column — cuvs has no float64; only VECF32 allowed -- * QUANTIZATION 'float64' — cuvs quantization is f32/f16/int8/uint8 only +-- * QUANTIZATION 'bf16' — no GPU bfloat16 storage; must not silent-fallback to f32 +-- * int8/uint8 + ip/cosine — affine quantizer breaks dot-product/angle (L2-only) at CREATE +-- * REINDEX QUANTIZATION — the (quantization, op_type) pair is gated via the per-algo +-- ValidQuantization hook on the merged config: bad values +-- (bf16/float64) and int8/uint8 on a non-L2 index are rejected -- * dimension mismatch at search — query dim must equal the column dim -- ===================================================================== @@ -42,6 +47,12 @@ create index ixf using ivfpq on tf (v) op_type 'vector_l2_ops' lists=2 m=8 bits_ -- Unsupported QUANTIZATION value. create index ixq using cagra on t (v) op_type 'vector_l2_ops' QUANTIZATION 'float64'; +-- QUANTIZATION 'bf16' has no GPU bfloat16 storage (cuvs has no bfloat16 index or +-- quantizer). It passes the downcast width guard (bf16 is 2 bytes <= f32's 4), +-- so it must be rejected explicitly rather than silently building f32 storage. +create index ixbq using cagra on t (v) op_type 'vector_l2_ops' QUANTIZATION 'bf16'; +create index ixbq using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'bf16'; + -- VARCHAR is not a supported INCLUDE column type (only int32/int64/float32/float64). create index ixv using cagra on t (v) op_type 'vector_l2_ops' INCLUDE (lbl); @@ -50,4 +61,45 @@ create index ixok using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=8 graph_degree=4 itopk_size=16; select id from t order by l2_distance(v, '[1,2,3]') asc limit 1; +-- Base-column type guard: only vecf32 / vecf16 are valid base columns; +-- vecbf16 (like int8/uint8) is rejected. +create table tbf (id bigint primary key, v vecbf16(8)); +create index ixbf using cagra on tbf (v) op_type 'vector_l2_ops'; +create index ixbf using ivfpq on tbf (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8; + +-- QUANTIZATION is downcast-only: a vecf16 base (2 bytes/element) cannot be +-- upcast to float32 storage (4 bytes/element). +create table th (id bigint primary key, v vecf16(8)); +create index ixup using cagra on th (v) op_type 'vector_l2_ops' QUANTIZATION 'float32'; +create index ixup using ivfpq on th (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'float32'; + +-- int8/uint8 QUANTIZATION is L2-only. The scalar quantizer applies a per-element +-- affine map q(x)=scalar*x+offset; the constant offset is a translation that +-- cancels in an L2 difference but NOT in a dot product (biases IP by component +-- sum) or norm (rotates cosine angles). So int8/uint8 + inner-product / cosine +-- returns wrong rankings and is rejected. (L2 is fine; the scale is corrected +-- on search.) +create index ixqi using cagra on t (v) op_type 'vector_ip_ops' QUANTIZATION 'int8'; +create index ixqi using cagra on t (v) op_type 'vector_cosine_ops' QUANTIZATION 'int8'; +create index ixqi using ivfpq on t (v) op_type 'vector_ip_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'uint8'; + +-- REINDEX gates the (quantization, op_type) pair through the per-algo +-- ValidQuantization hook, evaluated on the MERGED config: the value must be a +-- cuvs storage name (float32/float16/int8/uint8 — bf16/float64 rejected), and +-- int8/uint8 require L2. op_type is immutable across a reindex, so the merged +-- op_type is the index's stored inner-product — hence int8 is rejected here too. +-- Built on its own table, since t already has a CAGRA index on v and two CAGRA +-- indexes may not share a column. +create table tre (id bigint primary key, v vecf32(8)); +insert into tre values + (1, '[1,1,1,1,1,1,1,1]'), (2, '[2,2,2,2,2,2,2,2]'), (3, '[3,3,3,3,3,3,3,3]'), + (4, '[4,4,4,4,4,4,4,4]'), (5, '[5,5,5,5,5,5,5,5]'), (6, '[6,6,6,6,6,6,6,6]'), + (7, '[7,7,7,7,7,7,7,7]'), (8, '[8,8,8,8,8,8,8,8]'), (9, '[9,9,9,9,9,9,9,9]'), + (10, '[10,10,10,10,10,10,10,10]'); +create index ixre using cagra on tre (v) op_type 'vector_ip_ops' + intermediate_graph_degree=8 graph_degree=4 itopk_size=16; +alter table tre alter reindex ixre cagra QUANTIZATION 'int8'; +alter table tre alter reindex ixre cagra QUANTIZATION 'bf16'; +alter table tre alter reindex ixre cagra QUANTIZATION 'float64'; + drop database gpu_negative; diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_f16.result b/test/distributed/gpu_cases/vector/vector_ivfpq_f16.result new file mode 100644 index 0000000000000..540d9e48a0097 --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_f16.result @@ -0,0 +1,122 @@ +SET experimental_ivfpq_index = 1; +SET ivfpq_threads_build = 6; +SET ivfpq_max_index_capacity = 99999; +SET kmeans_train_percent = 100; +SET kmeans_max_iteration = 12; +SET probe_limit = 16; +drop database if exists ivfpq_f16_direct; +create database ivfpq_f16_direct; +use ivfpq_f16_direct; +create table t (id bigint primary key, v vecf16(8)); +insert into t values +( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), +( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), +( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), +( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), +( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), +(11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), +(13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), +(15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), +(17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), +(19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); +create index ix using ivfpq on t (v) +op_type 'vector_l2_ops' lists=10 m=8 bits_per_code=8; +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 10 m = 8 op_type 'vector_l2_ops' quantization 'float32' distribution_mode 'single' bits_per_code = 8 +) +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +1 +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +10 +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +15 +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +20 +drop database ivfpq_f16_direct; +drop database if exists ivfpq_f16_int8; +create database ivfpq_f16_int8; +use ivfpq_f16_int8; +create table t (id bigint primary key, v vecf16(8)); +insert into t values +( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), +( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), +( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), +( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), +( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), +(11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), +(13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), +(15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), +(17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), +(19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); +create index ix using ivfpq on t (v) +op_type 'vector_l2_ops' lists=10 m=8 bits_per_code=8 +QUANTIZATION 'int8'; +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 10 m = 8 op_type 'vector_l2_ops' quantization 'int8' distribution_mode 'single' bits_per_code = 8 +) +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +1 +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +10 +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +15 +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +20 +drop database ivfpq_f16_int8; +drop database if exists ivfpq_f16_uint8; +create database ivfpq_f16_uint8; +use ivfpq_f16_uint8; +create table t (id bigint primary key, v vecf16(8)); +insert into t values +( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), +( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), +( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), +( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), +( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), +(11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), +(13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), +(15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), +(17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), +(19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); +create index ix using ivfpq on t (v) +op_type 'vector_l2_ops' lists=10 m=8 bits_per_code=8 +QUANTIZATION 'uint8'; +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 10 m = 8 op_type 'vector_l2_ops' quantization 'uint8' distribution_mode 'single' bits_per_code = 8 +) +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +1 +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +10 +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +15 +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +20 +drop database ivfpq_f16_uint8; diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_f16.sql b/test/distributed/gpu_cases/vector/vector_ivfpq_f16.sql new file mode 100644 index 0000000000000..74bf43f9d7b76 --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_f16.sql @@ -0,0 +1,123 @@ +-- ===================================================================== +-- vector_ivfpq_f16.sql — IVF-PQ over a vecf16 (half) BASE column +-- +-- GPU REQUIRED. Unlike vector_ivfpq_quantization.sql (vecf32 base, the +-- QUANTIZATION clause only changes internal storage), here the COLUMN itself +-- is vecf16 — the native base/query type is half end-to-end: +-- * direct — no QUANTIZATION: the index stores half natively (Q == base). +-- * int8 — vecf16 base quantized half->int8 via the native half-source +-- scalar quantizer (no f32 detour). +-- * uint8 — same, half->uint8. +-- +-- Three databases, one per storage. Each builds a sync IVF-PQ index and +-- asserts (a) the vecf16 column + index round-trip through SHOW CREATE TABLE / +-- the catalog and (b) exact-match search returns the right row. The query +-- literal is cast to vecf16(8) so the half query path is exercised. +-- +-- Determinism: integers 1..20 — every value is exact in half, and the int8/ +-- uint8 quantizer trains on [1,20] so each integer maps to a distinct level; +-- the exact-match probe is always the unique top-1. Do not widen the range +-- under int8/uint8 (adjacent levels would collapse). +-- ===================================================================== + +SET experimental_ivfpq_index = 1; +SET ivfpq_threads_build = 6; +SET ivfpq_max_index_capacity = 99999; +SET kmeans_train_percent = 100; +SET kmeans_max_iteration = 12; +SET probe_limit = 16; + +-- ===================================================================== +-- vecf16 base, direct (no QUANTIZATION — stored as half) +-- ===================================================================== +drop database if exists ivfpq_f16_direct; +create database ivfpq_f16_direct; +use ivfpq_f16_direct; + +create table t (id bigint primary key, v vecf16(8)); +insert into t values + ( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), + ( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), + ( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), + ( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), + ( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), + (11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), + (13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), + (15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), + (17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), + (19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); + +create index ix using ivfpq on t (v) + op_type 'vector_l2_ops' lists=10 m=8 bits_per_code=8; + +show create table t; +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; + +drop database ivfpq_f16_direct; + +-- ===================================================================== +-- vecf16 base, QUANTIZATION int8 (native half->int8) +-- ===================================================================== +drop database if exists ivfpq_f16_int8; +create database ivfpq_f16_int8; +use ivfpq_f16_int8; + +create table t (id bigint primary key, v vecf16(8)); +insert into t values + ( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), + ( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), + ( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), + ( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), + ( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), + (11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), + (13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), + (15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), + (17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), + (19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); + +create index ix using ivfpq on t (v) + op_type 'vector_l2_ops' lists=10 m=8 bits_per_code=8 + QUANTIZATION 'int8'; + +show create table t; +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; + +drop database ivfpq_f16_int8; + +-- ===================================================================== +-- vecf16 base, QUANTIZATION uint8 (native half->uint8) +-- ===================================================================== +drop database if exists ivfpq_f16_uint8; +create database ivfpq_f16_uint8; +use ivfpq_f16_uint8; + +create table t (id bigint primary key, v vecf16(8)); +insert into t values + ( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), + ( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), + ( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), + ( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), + ( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), + (11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), + (13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), + (15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), + (17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), + (19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); + +create index ix using ivfpq on t (v) + op_type 'vector_l2_ops' lists=10 m=8 bits_per_code=8 + QUANTIZATION 'uint8'; + +show create table t; +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; + +drop database ivfpq_f16_uint8; diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_filter.result b/test/distributed/gpu_cases/vector/vector_ivfpq_filter.result index f9ff94aec9cea..2ff55e1f04494 100644 --- a/test/distributed/gpu_cases/vector/vector_ivfpq_filter.result +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_filter.result @@ -46,7 +46,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_filter') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","included_columns":"c_i32,c_i64,c_f32,c_f64","lists":"2","m":"8","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"experimental_ivfpq_index":{"t":"I8","v":1},"ivfpq_max_index_capacity":{"t":"I","v":99999},"ivfpq_threads_build":{"t":"I","v":6},"kmeans_max_iteration":{"t":"I","v":20},"kmeans_train_percent":{"t":"F","v":100},"lower_case_table_names":{"t":"I","v":1}}}} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","included_columns":"c_i32,c_i64,c_f32,c_f64","lists":"2","m":"8","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; ➤ id[-5,64,0] 𝄀 9 diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_filter_quant.result b/test/distributed/gpu_cases/vector/vector_ivfpq_filter_quant.result new file mode 100644 index 0000000000000..e057a9306517d --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_filter_quant.result @@ -0,0 +1,347 @@ +SET experimental_ivfpq_index = 1; +SET ivfpq_threads_build = 6; +SET ivfpq_max_index_capacity = 99999; +SET kmeans_train_percent = 100; +SET probe_limit = 16; +drop database if exists ivfpq_fq_f16; +create database ivfpq_fq_f16; +use ivfpq_fq_f16; +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'float16' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf32(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 2 m = 8 op_type 'vector_l2_ops' quantization 'float16' distribution_mode 'single' bits_per_code = 8 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database ivfpq_fq_f16; +drop database if exists ivfpq_fq_int8; +create database ivfpq_fq_int8; +use ivfpq_fq_int8; +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'int8' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf32(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 2 m = 8 op_type 'vector_l2_ops' quantization 'int8' distribution_mode 'single' bits_per_code = 8 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database ivfpq_fq_int8; +drop database if exists ivfpq_fq_uint8; +create database ivfpq_fq_uint8; +use ivfpq_fq_uint8; +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'uint8' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf32(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 2 m = 8 op_type 'vector_l2_ops' quantization 'uint8' distribution_mode 'single' bits_per_code = 8 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database ivfpq_fq_uint8; +drop database if exists ivfpq_fq_f16base; +create database ivfpq_fq_f16base; +use ivfpq_fq_f16base; +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 2 m = 8 op_type 'vector_l2_ops' quantization 'float32' distribution_mode 'single' bits_per_code = 8 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database ivfpq_fq_f16base; +drop database if exists ivfpq_fq_f16int8; +create database ivfpq_fq_f16int8; +use ivfpq_fq_f16int8; +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'int8' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 2 m = 8 op_type 'vector_l2_ops' quantization 'int8' distribution_mode 'single' bits_per_code = 8 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database ivfpq_fq_f16int8; +drop database if exists ivfpq_fq_f16uint8; +create database ivfpq_fq_f16uint8; +use ivfpq_fq_f16uint8; +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'uint8' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 2 m = 8 op_type 'vector_l2_ops' quantization 'uint8' distribution_mode 'single' bits_per_code = 8 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database ivfpq_fq_f16uint8; diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_filter_quant.sql b/test/distributed/gpu_cases/vector/vector_ivfpq_filter_quant.sql new file mode 100644 index 0000000000000..c208b1e2d38e3 --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_filter_quant.sql @@ -0,0 +1,297 @@ +-- ===================================================================== +-- vector_ivfpq_filter_quant.sql — IVFPQ INCLUDE-column pre-filter combined +-- with quantization and a vecf16 base column. +-- +-- GPU REQUIRED. vector_ivfpq_filter.sql already covers the INCLUDE pre-filter +-- over a plain vecf32 base (quantization 'float32'). This file proves the SAME +-- predsJSON pre-filter path stays correct when the index storage is compressed +-- or the base column is half: +-- * f32 base + QUANTIZATION 'float16' — supported (query quantized to T) +-- * f32 base + QUANTIZATION 'int8' — supported (learned scalar quantizer) +-- * f32 base + QUANTIZATION 'uint8' — supported +-- * vecf16 base, direct (no QUANTIZATION) — supported (native half query) +-- * vecf16 base + QUANTIZATION 'int8' + filter — supported (the native half +-- query is quantized to int8 inside +-- cuVS via search_quantize_with_filter) +-- * vecf16 base + QUANTIZATION 'uint8' + filter — supported (same path) +-- Every storage routes the SAME predsJSON pre-filter through the const-B* +-- quantize search, so the expected nearest neighbor per predicate is identical. +-- +-- Data/predicates are identical to vector_ivfpq_filter.sql so the expected +-- nearest neighbor per predicate is unchanged across every storage: +-- id=i -> [i]*8; c_i32=i, c_i64=i*10, c_f32=i.25, c_f64=i.5 (all monotone). +-- Query [12]*8: +-- * c_i32 < 10 -> id 9 +-- * c_i64 >= 100 -> id 12 +-- * c_f32 > 15.25 -> id 16 +-- * c_f64 = 5.5 -> id 5 +-- * c_i32 >= 10 AND c_f64 < 15.5 -> id 12 +-- * c_i64 < 100 AND c_f32 > 5.25 -> id 9 +-- +-- Determinism note: integers 1..20 are exact in float16 and the int8/uint8 +-- quantizer trains on [1,20] so each integer maps to a distinct level; each +-- predicate band keeps a unique nearest. Do NOT widen the range under int8/ +-- uint8 (adjacent levels would collapse and the top-1 would become ambiguous). +-- ===================================================================== + +SET experimental_ivfpq_index = 1; +SET ivfpq_threads_build = 6; +SET ivfpq_max_index_capacity = 99999; +SET kmeans_train_percent = 100; +SET probe_limit = 16; + +-- ===================================================================== +-- f32 base + QUANTIZATION 'float16' + INCLUDE pre-filter +-- ===================================================================== +drop database if exists ivfpq_fq_f16; +create database ivfpq_fq_f16; +use ivfpq_fq_f16; + +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'float16' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; + +drop database ivfpq_fq_f16; + +-- ===================================================================== +-- f32 base + QUANTIZATION 'int8' + INCLUDE pre-filter +-- ===================================================================== +drop database if exists ivfpq_fq_int8; +create database ivfpq_fq_int8; +use ivfpq_fq_int8; + +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'int8' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; + +drop database ivfpq_fq_int8; + +-- ===================================================================== +-- f32 base + QUANTIZATION 'uint8' + INCLUDE pre-filter +-- ===================================================================== +drop database if exists ivfpq_fq_uint8; +create database ivfpq_fq_uint8; +use ivfpq_fq_uint8; + +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'uint8' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; + +drop database ivfpq_fq_uint8; + +-- ===================================================================== +-- vecf16 base, direct (no QUANTIZATION) + INCLUDE pre-filter +-- The query literal is cast to vecf16(8) so the half query path is exercised. +-- ===================================================================== +drop database if exists ivfpq_fq_f16base; +create database ivfpq_fq_f16base; +use ivfpq_fq_f16base; + +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; + +drop database ivfpq_fq_f16base; + +-- ===================================================================== +-- vecf16 base + QUANTIZATION 'int8' + INCLUDE pre-filter +-- The native half query is quantized to int8 inside cuVS (the const-B* +-- search_quantize_with_filter path); same predicates and nearest neighbors +-- as every storage above. +-- ===================================================================== +drop database if exists ivfpq_fq_f16int8; +create database ivfpq_fq_f16int8; +use ivfpq_fq_f16int8; + +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'int8' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; + +drop database ivfpq_fq_f16int8; + +-- ===================================================================== +-- vecf16 base + QUANTIZATION 'uint8' + INCLUDE pre-filter (same path as int8) +-- ===================================================================== +drop database if exists ivfpq_fq_f16uint8; +create database ivfpq_fq_f16uint8; +use ivfpq_fq_f16uint8; + +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'uint8' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; + +drop database ivfpq_fq_f16uint8; diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_metric.result b/test/distributed/gpu_cases/vector/vector_ivfpq_metric.result index 2b22fc117e0e7..4323a72d60785 100644 --- a/test/distributed/gpu_cases/vector/vector_ivfpq_metric.result +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_metric.result @@ -25,7 +25,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_metric') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"2","m":"8","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"experimental_ivfpq_index":{"t":"I8","v":1},"ivfpq_max_index_capacity":{"t":"I","v":99999},"ivfpq_threads_build":{"t":"I","v":6},"kmeans_max_iteration":{"t":"I","v":20},"kmeans_train_percent":{"t":"F","v":100},"lower_case_table_names":{"t":"I","v":1}}}} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"2","m":"8","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id, l2_distance(v, '[16,15,14,13,12,11,10,9]') as score from t order by score asc limit 1; ➤ id[-5,64,0] ¦ score[8,9,0] 𝄀 1 ¦ 0.0 @@ -36,7 +36,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_metric') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"2","m":"8","op_type":"vector_l2sq_ops","quantization":"float32","session_vars":{"cfg":{"experimental_ivfpq_index":{"t":"I8","v":1},"ivfpq_max_index_capacity":{"t":"I","v":99999},"ivfpq_threads_build":{"t":"I","v":6},"kmeans_max_iteration":{"t":"I","v":20},"kmeans_train_percent":{"t":"F","v":100},"lower_case_table_names":{"t":"I","v":1}}}} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"2","m":"8","op_type":"vector_l2sq_ops","quantization":"float32","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id, l2_distance_sq(v, '[16,15,14,13,12,11,10,9]') as score from t order by score asc limit 1; ➤ id[-5,64,0] ¦ score[8,54,0] 𝄀 1 ¦ 0.0 @@ -47,7 +47,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_metric') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"2","m":"8","op_type":"vector_ip_ops","quantization":"float32","session_vars":{"cfg":{"experimental_ivfpq_index":{"t":"I8","v":1},"ivfpq_max_index_capacity":{"t":"I","v":99999},"ivfpq_threads_build":{"t":"I","v":6},"kmeans_max_iteration":{"t":"I","v":20},"kmeans_train_percent":{"t":"F","v":100},"lower_case_table_names":{"t":"I","v":1}}}} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"2","m":"8","op_type":"vector_ip_ops","quantization":"float32","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id, inner_product(v, '[16,15,14,13,12,11,10,9]') as score from t order by score asc limit 1; ➤ id[-5,64,0] ¦ score[8,9,0] 𝄀 1 ¦ -1292.0 @@ -58,7 +58,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_metric') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"2","m":"8","op_type":"vector_cosine_ops","quantization":"float32","session_vars":{"cfg":{"experimental_ivfpq_index":{"t":"I8","v":1},"ivfpq_max_index_capacity":{"t":"I","v":99999},"ivfpq_threads_build":{"t":"I","v":6},"kmeans_max_iteration":{"t":"I","v":20},"kmeans_train_percent":{"t":"F","v":100},"lower_case_table_names":{"t":"I","v":1}}}} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"2","m":"8","op_type":"vector_cosine_ops","quantization":"float32","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id, cosine_distance(v, '[16,15,14,13,12,11,10,9]') as score from t order by score asc limit 1; ➤ id[-5,64,0] ¦ score[8,9,0] 𝄀 1 ¦ 0.0 diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_postfilter.result b/test/distributed/gpu_cases/vector/vector_ivfpq_postfilter.result new file mode 100644 index 0000000000000..c6e69e9be959f --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_postfilter.result @@ -0,0 +1,91 @@ +SET experimental_ivfpq_index = 1; +SET ivfpq_threads_build = 6; +SET ivfpq_max_index_capacity = 99999; +SET kmeans_train_percent = 100; +SET probe_limit = 16; +drop database if exists ivfpq_postfilter; +create database ivfpq_postfilter; +use ivfpq_postfilter; +create table t (id bigint primary key, v vecf32(8), c_inc int, c_noinc int); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 101), +(2, '[2,2,2,2,2,2,2,2]', 2, 102), +(3, '[3,3,3,3,3,3,3,3]', 3, 103), +(4, '[4,4,4,4,4,4,4,4]', 4, 104), +(5, '[5,5,5,5,5,5,5,5]', 5, 105), +(6, '[6,6,6,6,6,6,6,6]', 6, 106), +(7, '[7,7,7,7,7,7,7,7]', 7, 107), +(8, '[8,8,8,8,8,8,8,8]', 8, 108), +(9, '[9,9,9,9,9,9,9,9]', 9, 109), +(10, '[10,10,10,10,10,10,10,10]', 10, 110), +(11, '[11,11,11,11,11,11,11,11]', 11, 111), +(12, '[12,12,12,12,12,12,12,12]', 12, 112), +(13, '[13,13,13,13,13,13,13,13]', 13, 113), +(14, '[14,14,14,14,14,14,14,14]', 14, 114), +(15, '[15,15,15,15,15,15,15,15]', 15, 115), +(16, '[16,16,16,16,16,16,16,16]', 16, 116), +(17, '[17,17,17,17,17,17,17,17]', 17, 117), +(18, '[18,18,18,18,18,18,18,18]', 18, 118), +(19, '[19,19,19,19,19,19,19,19]', 19, 119), +(20, '[20,20,20,20,20,20,20,20]', 20, 120); +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 INCLUDE (c_inc); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf32(8) DEFAULT NULL, + `c_inc` int DEFAULT NULL, + `c_noinc` int DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 2 m = 8 op_type 'vector_l2_ops' quantization 'float32' distribution_mode 'single' bits_per_code = 8 INCLUDE (c_inc) +) +select id, c_noinc from t order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] ¦ c_noinc[4,32,0] 𝄀 +12 ¦ 112 𝄀 +13 ¦ 113 𝄀 +11 ¦ 111 𝄀 +10 ¦ 110 𝄀 +14 ¦ 114 𝄀 +9 ¦ 109 𝄀 +15 ¦ 115 𝄀 +16 ¦ 116 𝄀 +8 ¦ 108 𝄀 +7 ¦ 107 𝄀 +17 ¦ 117 𝄀 +6 ¦ 106 𝄀 +18 ¦ 118 𝄀 +5 ¦ 105 𝄀 +19 ¦ 119 𝄀 +20 ¦ 120 𝄀 +4 ¦ 104 𝄀 +3 ¦ 103 𝄀 +2 ¦ 102 𝄀 +1 ¦ 101 +select id from t where c_noinc < 105 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] 𝄀 +4 𝄀 +3 𝄀 +2 𝄀 +1 +select id from t where c_noinc >= 116 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] 𝄀 +16 𝄀 +17 𝄀 +18 𝄀 +19 𝄀 +20 +select id from t where c_noinc = 102 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] 𝄀 +2 +select id from t where c_inc <= 14 and c_noinc >= 108 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] 𝄀 +12 𝄀 +13 𝄀 +11 𝄀 +10 𝄀 +14 𝄀 +9 𝄀 +8 +select id from t where c_noinc = 102 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] +drop database ivfpq_postfilter; diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_postfilter.sql b/test/distributed/gpu_cases/vector/vector_ivfpq_postfilter.sql new file mode 100644 index 0000000000000..9b8a648ec70f3 --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_postfilter.sql @@ -0,0 +1,77 @@ +-- ===================================================================== +-- vector_ivfpq_postfilter.sql — IVF-PQ search filtering on a NON-INCLUDE column +-- +-- GPU REQUIRED. A WHERE predicate on a column that is NOT in the index INCLUDE +-- list cannot be pushed into the GPU bitset pre-filter. Instead the planner runs +-- the ANN search to get a candidate window, then JOINs + filters the predicate +-- at the database (post-filter): ivfpq_search (candidate window) INNER JOIN +-- (table scan Filter: ) -> Sort -> Limit. +-- +-- Methodology: first take the UNFILTERED ranked result, then verify the +-- post-filtered result equals exactly the unfiltered rows that satisfy the +-- predicate. The candidate window grows with the query LIMIT, so a LIMIT >= row +-- count makes the window cover every row -> the post-filter is exact. +-- +-- Data: id=i -> [i]*8; c_inc=i (INCLUDE, int), c_noinc=100+i (NOT included). +-- Query [12]*8. With LIMIT 20 the window is all 20 rows, so post-filter is exact: +-- * c_noinc < 105 -> i in 1..4 -> 4,3,2,1 (nearest-first) +-- * c_noinc >= 116 -> i in 16..20 -> 16,17,18,19,20 +-- * c_noinc = 102 -> id 2 (far row still found, full window) +-- * c_inc <= 14 (PRE) AND c_noinc >= 108 (POST) -> i in 8..14 +-- ===================================================================== + +SET experimental_ivfpq_index = 1; +SET ivfpq_threads_build = 6; +SET ivfpq_max_index_capacity = 99999; +SET kmeans_train_percent = 100; +SET probe_limit = 16; + +drop database if exists ivfpq_postfilter; +create database ivfpq_postfilter; +use ivfpq_postfilter; + +create table t (id bigint primary key, v vecf32(8), c_inc int, c_noinc int); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 101), + (2, '[2,2,2,2,2,2,2,2]', 2, 102), + (3, '[3,3,3,3,3,3,3,3]', 3, 103), + (4, '[4,4,4,4,4,4,4,4]', 4, 104), + (5, '[5,5,5,5,5,5,5,5]', 5, 105), + (6, '[6,6,6,6,6,6,6,6]', 6, 106), + (7, '[7,7,7,7,7,7,7,7]', 7, 107), + (8, '[8,8,8,8,8,8,8,8]', 8, 108), + (9, '[9,9,9,9,9,9,9,9]', 9, 109), + (10, '[10,10,10,10,10,10,10,10]', 10, 110), + (11, '[11,11,11,11,11,11,11,11]', 11, 111), + (12, '[12,12,12,12,12,12,12,12]', 12, 112), + (13, '[13,13,13,13,13,13,13,13]', 13, 113), + (14, '[14,14,14,14,14,14,14,14]', 14, 114), + (15, '[15,15,15,15,15,15,15,15]', 15, 115), + (16, '[16,16,16,16,16,16,16,16]', 16, 116), + (17, '[17,17,17,17,17,17,17,17]', 17, 117), + (18, '[18,18,18,18,18,18,18,18]', 18, 118), + (19, '[19,19,19,19,19,19,19,19]', 19, 119), + (20, '[20,20,20,20,20,20,20,20]', 20, 120); + +-- Only c_inc is pushed into the GPU pre-filter; c_noinc is post-filtered. +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 INCLUDE (c_inc); + +show create table t; + +-- (1) UNFILTERED ranked baseline (window covers all rows at LIMIT 20). +select id, c_noinc from t order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; + +-- (2) POST-FILTER on the non-INCLUDE column — must equal the baseline rows that +-- satisfy the predicate, in the same distance order. +select id from t where c_noinc < 105 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +select id from t where c_noinc >= 116 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +select id from t where c_noinc = 102 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; + +-- (3) MIXED: c_inc is pushed (pre-filter), c_noinc is post-filtered. Both apply. +select id from t where c_inc <= 14 and c_noinc >= 108 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; + +-- (4) Small LIMIT shrinks the candidate window: a far post-filter match (id 2, +-- c_noinc=102) falls outside the window and is not returned (approximate). +select id from t where c_noinc = 102 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; + +drop database ivfpq_postfilter; diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_quantization.result b/test/distributed/gpu_cases/vector/vector_ivfpq_quantization.result index 0e3d09db10d83..8bc834c012133 100644 --- a/test/distributed/gpu_cases/vector/vector_ivfpq_quantization.result +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_quantization.result @@ -35,7 +35,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_q_f16') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"float16","session_vars":{"cfg":{"experimental_ivfpq_index":{"t":"I8","v":1},"ivfpq_max_index_capacity":{"t":"I","v":99999},"ivfpq_threads_build":{"t":"I","v":6},"kmeans_max_iteration":{"t":"I","v":12},"kmeans_train_percent":{"t":"F","v":100},"lower_case_table_names":{"t":"I","v":1}}}} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"float16","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 @@ -80,7 +80,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_q_int8') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"int8","session_vars":{"cfg":{"experimental_ivfpq_index":{"t":"I8","v":1},"ivfpq_max_index_capacity":{"t":"I","v":99999},"ivfpq_threads_build":{"t":"I","v":6},"kmeans_max_iteration":{"t":"I","v":12},"kmeans_train_percent":{"t":"F","v":100},"lower_case_table_names":{"t":"I","v":1}}}} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"int8","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 @@ -125,7 +125,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_q_uint8') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"uint8","session_vars":{"cfg":{"experimental_ivfpq_index":{"t":"I8","v":1},"ivfpq_max_index_capacity":{"t":"I","v":99999},"ivfpq_threads_build":{"t":"I","v":6},"kmeans_max_iteration":{"t":"I","v":12},"kmeans_train_percent":{"t":"F","v":100},"lower_case_table_names":{"t":"I","v":1}}}} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"uint8","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_replicated.result b/test/distributed/gpu_cases/vector/vector_ivfpq_replicated.result index f3e211c4185b1..c446eed512e7f 100644 --- a/test/distributed/gpu_cases/vector/vector_ivfpq_replicated.result +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_replicated.result @@ -36,7 +36,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_replicated') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"replicated","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"float32"} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"replicated","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_sharded.result b/test/distributed/gpu_cases/vector/vector_ivfpq_sharded.result index 58f990e2fcda9..8bf09f422700e 100644 --- a/test/distributed/gpu_cases/vector/vector_ivfpq_sharded.result +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_sharded.result @@ -90,7 +90,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_sharded') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"sharded","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"float32"} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"sharded","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 diff --git a/test/distributed/resources/load_data/narrow_vec_array.csv b/test/distributed/resources/load_data/narrow_vec_array.csv new file mode 100644 index 0000000000000..802127603ecab --- /dev/null +++ b/test/distributed/resources/load_data/narrow_vec_array.csv @@ -0,0 +1,3 @@ +id,a,b,c,d +1,"[1, 2, 3]","[0.5, 0.25, -0.5]","[-128, 0, 127]","[0, 128, 255]" +2,"[0.5, -0.25, 4]","[1, 2, 3]","[10, -10, 5]","[1, 2, 3]" diff --git a/test/distributed/resources/load_data/narrow_vec_dim_bad.csv b/test/distributed/resources/load_data/narrow_vec_dim_bad.csv new file mode 100644 index 0000000000000..36c7272447d8b --- /dev/null +++ b/test/distributed/resources/load_data/narrow_vec_dim_bad.csv @@ -0,0 +1,2 @@ +id,d +9,"[1, 2]" diff --git a/test/distributed/resources/load_data/narrow_vec_int8_frac.csv b/test/distributed/resources/load_data/narrow_vec_int8_frac.csv new file mode 100644 index 0000000000000..59001a14f0553 --- /dev/null +++ b/test/distributed/resources/load_data/narrow_vec_int8_frac.csv @@ -0,0 +1,2 @@ +id,c +9,"[0.5, 0, 0]" diff --git a/test/distributed/resources/load_data/narrow_vec_int8_oor.csv b/test/distributed/resources/load_data/narrow_vec_int8_oor.csv new file mode 100644 index 0000000000000..dff5c34d3181f --- /dev/null +++ b/test/distributed/resources/load_data/narrow_vec_int8_oor.csv @@ -0,0 +1,2 @@ +id,c +9,"[200, 0, 0]"