Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions cmake/svs.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ if(USE_SVS)
elseif(GLIBCXX_VERSION VERSION_GREATER_EQUAL "12")
set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-nightly-reduced-clang21-gcc12-2026-03-31-1147.tar.gz" CACHE STRING "SVS URL")
elseif(GLIBCXX_VERSION VERSION_GREATER_EQUAL "11")
set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-nightly-reduced-clang21-gcc11-2026-03-31-1147.tar.gz" CACHE STRING "SVS URL")
set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-reduced-clang21-gcc11-2026-06-18-1414.tar.gz" CACHE STRING "SVS URL")
else()
message(STATUS "libstdc++ >= GCC 11 is required for Clang SVS binaries - disabling SVS_SHARED_LIB")
set(SVS_SHARED_LIB OFF)
Expand All @@ -80,9 +80,9 @@ if(USE_SVS)
else()
if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.28")
if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "14.0")
set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.3.0/svs-shared-library-0.3.0-reduced-gcc14.tar.gz" CACHE STRING "SVS URL")
set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-reduced-gcc14-2026-06-18-1414.tar.gz" CACHE STRING "SVS URL")
else()
set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.3.0/svs-shared-library-0.3.0-reduced.tar.gz" CACHE STRING "SVS URL")
set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-reduced-2026-06-18-1414.tar.gz" CACHE STRING "SVS URL")
endif()
elseif(GLIBC_VERSION VERSION_GREATER_EQUAL "2.26")
set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.3.0/svs-shared-library-0.3.0-reduced-glibc2_26.tar.gz" CACHE STRING "SVS URL")
Expand Down
2 changes: 1 addition & 1 deletion deps/ScalableVectorSearch
42 changes: 35 additions & 7 deletions src/VecSim/algorithms/svs/svs_extensions.h
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,8 @@ struct SVSStorageTraits<DataType, QuantBits, ResidualBits, false,
static auto make_blocked_allocator(size_t block_size, size_t dim,
std::shared_ptr<VecSimAllocator> allocator) {
// SVS block size is a power of two, so we can use it directly
auto svs_bs = svs_details::SVSBlockSize(block_size, element_size(dim));
auto elem_size = std::max(primary_element_size(dim), residual_element_size(dim));
auto svs_bs = svs_details::SVSBlockSize(block_size, elem_size);
allocator_type data_allocator{std::move(allocator)};
return svs::make_blocked_allocator_handle({svs_bs}, data_allocator);
}
Expand Down Expand Up @@ -144,15 +145,32 @@ struct SVSStorageTraits<DataType, QuantBits, ResidualBits, false,
return svs::lib::load_from_disk<index_storage_type>(path, /*alignment=*/0, blocked_alloc);
}

static constexpr size_t element_size(size_t dims, size_t alignment = 0,
size_t /*leanvec_dim*/ = 0) {
static constexpr size_t primary_element_size(size_t dims, size_t alignment = 0,
size_t /*leanvec_dim*/ = 0) {
using primary_type = typename index_storage_type::primary_type;
using layout_type = typename primary_type::helper_type;
using layout_dims_type = svs::lib::MaybeStatic<index_storage_type::extent>;
const auto layout_dims = layout_dims_type{dims};
return primary_type::compute_data_dimensions(layout_type{layout_dims}, alignment);
}

static constexpr size_t residual_element_size(size_t dims, size_t alignment = 0) {
if constexpr (ResidualBits == 0) {
return 0;
} else {
using residual_type = typename index_storage_type::residual_type;
using dims_type = svs::lib::MaybeStatic<index_storage_type::extent>;
auto residual_dims = dims_type{dims};
return residual_type::total_bytes(residual_dims);
}
}

static constexpr size_t element_size(size_t dims, size_t alignment = 0,
size_t leanvec_dim = 0) {
return primary_element_size(dims, alignment, leanvec_dim) +
residual_element_size(dims, alignment);
}

static size_t storage_capacity(const index_storage_type &storage) {
// LVQDataset does not provide a capacity method
return storage.size();
Expand Down Expand Up @@ -191,7 +209,8 @@ struct SVSStorageTraits<DataType, QuantBits, ResidualBits, true> {
static auto make_blocked_allocator(size_t block_size, size_t dim,
std::shared_ptr<VecSimAllocator> allocator) {
// SVS block size is a power of two, so we can use it directly
auto svs_bs = svs_details::SVSBlockSize(block_size, element_size(dim));
auto elem_size = std::max(primary_element_size(dim), secondary_element_size(dim));
auto svs_bs = svs_details::SVSBlockSize(block_size, elem_size);
allocator_type data_allocator{std::move(allocator)};
return svs::make_blocked_allocator_handle({svs_bs}, data_allocator);
}
Expand Down Expand Up @@ -223,11 +242,20 @@ struct SVSStorageTraits<DataType, QuantBits, ResidualBits, true> {
return svs::lib::load_from_disk<index_storage_type>(path, /*alignment=*/0, blocked_alloc);
}

static constexpr size_t primary_element_size(size_t dims, size_t alignment = 0,
size_t leanvec_dim = 0) {
return SVSStorageTraits<DataType, QuantBits, 0, false>::element_size(
check_leanvec_dim(dims, leanvec_dim), alignment);
}

static constexpr size_t secondary_element_size(size_t dims, size_t alignment = 0) {
return SVSStorageTraits<DataType, ResidualBits, 0, false>::element_size(dims, alignment);
}

static constexpr size_t element_size(size_t dims, size_t alignment = 0,
size_t leanvec_dim = 0) {
return SVSStorageTraits<DataType, QuantBits, 0, false>::element_size(
check_leanvec_dim(dims, leanvec_dim), alignment) +
SVSStorageTraits<DataType, ResidualBits, 0, false>::element_size(dims, alignment);
return primary_element_size(dims, alignment, leanvec_dim) +
secondary_element_size(dims, alignment);
}

static size_t storage_capacity(const index_storage_type &storage) {
Expand Down
5 changes: 3 additions & 2 deletions src/VecSim/algorithms/svs/svs_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -167,13 +167,14 @@ joinSearchParams(svs::index::vamana::VamanaSearchParameters &&sp,
// @param bs VecSim block size
// @param elem_size SVS storage element size
// @return block size in type of SVS `PowerOfTwo`
inline svs::lib::PowerOfTwo SVSBlockSize(size_t bs, size_t elem_size) {
inline svs::data::BlockingParameters SVSBlockSize(size_t bs, size_t elem_size) {
auto svs_bs = svs::lib::prevpow2(bs * elem_size);
// block size should not be less than element size
while (svs_bs.value() < elem_size) {
svs_bs = svs::lib::PowerOfTwo{svs_bs.raw() + 1};
}
return svs_bs;
auto svs_bs_elems = svs::lib::prevpow2(bs);
return svs::data::BlockingParameters{svs_bs, svs_bs_elems};
}

// Check if the SVS implementation supports Quantization mode
Expand Down
19 changes: 1 addition & 18 deletions tests/unit/test_svs.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1970,22 +1970,6 @@ TYPED_TEST(SVSTest, svs_vector_search_test_cosine) {
TYPED_TEST(SVSTest, testSizeEstimation) {
size_t dim = 64;
auto constexpr quantBits = TypeParam::get_quant_bits();
#if HAVE_SVS_LVQ
// SVS block sizes always rounded to a power of 2
// This why, in case of quantization, actual block size can be differ than requested
// In addition, block size to be passed to graph and dataset counted in bytes,
// converted then to a number of elements.
// IMHO, would be better to always interpret block size to a number of elements
// rather than conversion to-from number of bytes
if constexpr (quantBits != VecSimSvsQuant_NONE) { // constexpr eliminates div-by-zero warning;
// inner condition is runtime-only
if (!this->isFallbackToSQ()) {
// Extra data in LVQ vector
const auto lvq_vector_extra = sizeof(svs::quantization::lvq::ScalarBundle);
dim -= (lvq_vector_extra * 8) / quantBits;
}
}
#endif
size_t n = 0;
size_t bs = DEFAULT_BLOCK_SIZE;

Expand Down Expand Up @@ -2016,8 +2000,7 @@ TYPED_TEST(SVSTest, testSizeEstimation) {
GenerateAndAddVector<TEST_DATA_T>(index, dim, 0);
actual = index->getAllocationSize() - actual; // get the delta
ASSERT_GT(actual, 0);
// LVQ element estimation accuracy is low
double estimation_accuracy = (quantBits != VecSimSvsQuant_NONE) ? 0.1 : 0.01;
double estimation_accuracy = 0.01;
ASSERT_GE(estimation * (1.0 + estimation_accuracy), actual);
ASSERT_LE(estimation * (1.0 - estimation_accuracy), actual);

Expand Down
8 changes: 7 additions & 1 deletion tests/unit/test_svs_tiered.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3199,7 +3199,13 @@ TYPED_TEST(SVSTieredIndexTestBasic, runGCAPI) {
size_t dim = 4;
size_t threshold = 1024;
const size_t n = threshold * 3;
SVSParams params = {.type = TypeParam::get_index_type(), .dim = dim, .metric = VecSimMetric_L2};
// svs::data::SimpleData::resize() keeps 1 empty block in case of size reducing, so we need to
// make sure that we have at least 2 blocks of vectors to be deleted:
const size_t block_size = threshold / 2;
SVSParams params = {.type = TypeParam::get_index_type(),
.dim = dim,
.metric = VecSimMetric_L2,
.blockSize = block_size};
VecSimParams svs_params = CreateParams(params);
auto mock_thread_pool = tieredIndexMock();

Expand Down
Loading