diff --git a/cmake/svs.cmake b/cmake/svs.cmake index 173b759c2..38ff79477 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -68,7 +68,7 @@ if(USE_SVS) elseif(GLIBCXX_VERSION VERSION_GREATER_EQUAL "12") set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-nightly-reduced-clang21-gcc12-2026-03-31-1147.tar.gz" CACHE STRING "SVS URL") elseif(GLIBCXX_VERSION VERSION_GREATER_EQUAL "11") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-nightly-reduced-clang21-gcc11-2026-03-31-1147.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-reduced-clang21-gcc11-2026-06-18-1414.tar.gz" CACHE STRING "SVS URL") else() message(STATUS "libstdc++ >= GCC 11 is required for Clang SVS binaries - disabling SVS_SHARED_LIB") set(SVS_SHARED_LIB OFF) @@ -80,9 +80,9 @@ if(USE_SVS) else() if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.28") if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "14.0") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.3.0/svs-shared-library-0.3.0-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-reduced-gcc14-2026-06-18-1414.tar.gz" CACHE STRING "SVS URL") else() - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.3.0/svs-shared-library-0.3.0-reduced.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-reduced-2026-06-18-1414.tar.gz" CACHE STRING "SVS URL") endif() elseif(GLIBC_VERSION VERSION_GREATER_EQUAL "2.26") set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.3.0/svs-shared-library-0.3.0-reduced-glibc2_26.tar.gz" CACHE STRING "SVS URL") diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch index 02dea9d9c..bbe38c82e 160000 --- a/deps/ScalableVectorSearch +++ b/deps/ScalableVectorSearch @@ -1 +1 @@ -Subproject commit 02dea9d9cd1c73dc03dc0fd91d14a548f1aece70 +Subproject commit bbe38c82e2e6dc2c65749ae115fe486a5892049e diff --git a/src/VecSim/algorithms/svs/svs_extensions.h b/src/VecSim/algorithms/svs/svs_extensions.h index 3903d2289..2e93a6a58 100644 --- a/src/VecSim/algorithms/svs/svs_extensions.h +++ b/src/VecSim/algorithms/svs/svs_extensions.h @@ -115,7 +115,8 @@ struct SVSStorageTraits allocator) { // SVS block size is a power of two, so we can use it directly - auto svs_bs = svs_details::SVSBlockSize(block_size, element_size(dim)); + auto elem_size = std::max(primary_element_size(dim), residual_element_size(dim)); + auto svs_bs = svs_details::SVSBlockSize(block_size, elem_size); allocator_type data_allocator{std::move(allocator)}; return svs::make_blocked_allocator_handle({svs_bs}, data_allocator); } @@ -144,8 +145,8 @@ struct SVSStorageTraits(path, /*alignment=*/0, blocked_alloc); } - static constexpr size_t element_size(size_t dims, size_t alignment = 0, - size_t /*leanvec_dim*/ = 0) { + static constexpr size_t primary_element_size(size_t dims, size_t alignment = 0, + size_t /*leanvec_dim*/ = 0) { using primary_type = typename index_storage_type::primary_type; using layout_type = typename primary_type::helper_type; using layout_dims_type = svs::lib::MaybeStatic; @@ -153,6 +154,23 @@ struct SVSStorageTraits; + auto residual_dims = dims_type{dims}; + return residual_type::total_bytes(residual_dims); + } + } + + static constexpr size_t element_size(size_t dims, size_t alignment = 0, + size_t leanvec_dim = 0) { + return primary_element_size(dims, alignment, leanvec_dim) + + residual_element_size(dims, alignment); + } + static size_t storage_capacity(const index_storage_type &storage) { // LVQDataset does not provide a capacity method return storage.size(); @@ -191,7 +209,8 @@ struct SVSStorageTraits { static auto make_blocked_allocator(size_t block_size, size_t dim, std::shared_ptr allocator) { // SVS block size is a power of two, so we can use it directly - auto svs_bs = svs_details::SVSBlockSize(block_size, element_size(dim)); + auto elem_size = std::max(primary_element_size(dim), secondary_element_size(dim)); + auto svs_bs = svs_details::SVSBlockSize(block_size, elem_size); allocator_type data_allocator{std::move(allocator)}; return svs::make_blocked_allocator_handle({svs_bs}, data_allocator); } @@ -223,11 +242,20 @@ struct SVSStorageTraits { return svs::lib::load_from_disk(path, /*alignment=*/0, blocked_alloc); } + static constexpr size_t primary_element_size(size_t dims, size_t alignment = 0, + size_t leanvec_dim = 0) { + return SVSStorageTraits::element_size( + check_leanvec_dim(dims, leanvec_dim), alignment); + } + + static constexpr size_t secondary_element_size(size_t dims, size_t alignment = 0) { + return SVSStorageTraits::element_size(dims, alignment); + } + static constexpr size_t element_size(size_t dims, size_t alignment = 0, size_t leanvec_dim = 0) { - return SVSStorageTraits::element_size( - check_leanvec_dim(dims, leanvec_dim), alignment) + - SVSStorageTraits::element_size(dims, alignment); + return primary_element_size(dims, alignment, leanvec_dim) + + secondary_element_size(dims, alignment); } static size_t storage_capacity(const index_storage_type &storage) { diff --git a/src/VecSim/algorithms/svs/svs_utils.h b/src/VecSim/algorithms/svs/svs_utils.h index 8dfa23d53..7d90f97c9 100644 --- a/src/VecSim/algorithms/svs/svs_utils.h +++ b/src/VecSim/algorithms/svs/svs_utils.h @@ -167,13 +167,14 @@ joinSearchParams(svs::index::vamana::VamanaSearchParameters &&sp, // @param bs VecSim block size // @param elem_size SVS storage element size // @return block size in type of SVS `PowerOfTwo` -inline svs::lib::PowerOfTwo SVSBlockSize(size_t bs, size_t elem_size) { +inline svs::data::BlockingParameters SVSBlockSize(size_t bs, size_t elem_size) { auto svs_bs = svs::lib::prevpow2(bs * elem_size); // block size should not be less than element size while (svs_bs.value() < elem_size) { svs_bs = svs::lib::PowerOfTwo{svs_bs.raw() + 1}; } - return svs_bs; + auto svs_bs_elems = svs::lib::prevpow2(bs); + return svs::data::BlockingParameters{svs_bs, svs_bs_elems}; } // Check if the SVS implementation supports Quantization mode diff --git a/tests/unit/test_svs.cpp b/tests/unit/test_svs.cpp index 415add1fc..764f17316 100644 --- a/tests/unit/test_svs.cpp +++ b/tests/unit/test_svs.cpp @@ -1970,22 +1970,6 @@ TYPED_TEST(SVSTest, svs_vector_search_test_cosine) { TYPED_TEST(SVSTest, testSizeEstimation) { size_t dim = 64; auto constexpr quantBits = TypeParam::get_quant_bits(); -#if HAVE_SVS_LVQ - // SVS block sizes always rounded to a power of 2 - // This why, in case of quantization, actual block size can be differ than requested - // In addition, block size to be passed to graph and dataset counted in bytes, - // converted then to a number of elements. - // IMHO, would be better to always interpret block size to a number of elements - // rather than conversion to-from number of bytes - if constexpr (quantBits != VecSimSvsQuant_NONE) { // constexpr eliminates div-by-zero warning; - // inner condition is runtime-only - if (!this->isFallbackToSQ()) { - // Extra data in LVQ vector - const auto lvq_vector_extra = sizeof(svs::quantization::lvq::ScalarBundle); - dim -= (lvq_vector_extra * 8) / quantBits; - } - } -#endif size_t n = 0; size_t bs = DEFAULT_BLOCK_SIZE; @@ -2016,8 +2000,7 @@ TYPED_TEST(SVSTest, testSizeEstimation) { GenerateAndAddVector(index, dim, 0); actual = index->getAllocationSize() - actual; // get the delta ASSERT_GT(actual, 0); - // LVQ element estimation accuracy is low - double estimation_accuracy = (quantBits != VecSimSvsQuant_NONE) ? 0.1 : 0.01; + double estimation_accuracy = 0.01; ASSERT_GE(estimation * (1.0 + estimation_accuracy), actual); ASSERT_LE(estimation * (1.0 - estimation_accuracy), actual); diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 88f5b715a..9542e7cd6 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -3199,7 +3199,13 @@ TYPED_TEST(SVSTieredIndexTestBasic, runGCAPI) { size_t dim = 4; size_t threshold = 1024; const size_t n = threshold * 3; - SVSParams params = {.type = TypeParam::get_index_type(), .dim = dim, .metric = VecSimMetric_L2}; + // svs::data::SimpleData::resize() keeps 1 empty block in case of size reducing, so we need to + // make sure that we have at least 2 blocks of vectors to be deleted: + const size_t block_size = threshold / 2; + SVSParams params = {.type = TypeParam::get_index_type(), + .dim = dim, + .metric = VecSimMetric_L2, + .blockSize = block_size}; VecSimParams svs_params = CreateParams(params); auto mock_thread_pool = tieredIndexMock();