Support for IVF index in cpp runtime

.github/scripts/build-cpp-runtime-bindings.sh

@@ -15,9 +15,20 @@
 
 set -e  # Exit on error
 
-# Source environment setup (for compiler and MKL)
+# Source environment setup (for compiler)
 source /etc/bashrc || true
 
+# Source MKL environment if IVF is enabled
+if [ "${ENABLE_IVF:-OFF}" = "ON" ]; then
+    if [ -f /opt/intel/oneapi/setvars.sh ]; then
+        source /opt/intel/oneapi/setvars.sh --include-intel-llvm 2>/dev/null || true
+        echo "MKL sourced: MKLROOT=${MKLROOT}"
+    else
+        echo "ERROR: IVF enabled but MKL setvars.sh not found"
+        exit 1
+    fi
+fi
+
 # Create build+install directories for cpp runtime bindings
 rm -rf /workspace/bindings/cpp/build_cpp_bindings /workspace/install_cpp_bindings
 mkdir -p /workspace/bindings/cpp/build_cpp_bindings /workspace/install_cpp_bindings
@@ -31,6 +42,7 @@ CMAKE_ARGS=(
     "-DCMAKE_INSTALL_PREFIX=/workspace/install_cpp_bindings"
     "-DCMAKE_INSTALL_LIBDIR=lib"
     "-DSVS_RUNTIME_ENABLE_LVQ_LEANVEC=${ENABLE_LVQ_LEANVEC:-ON}"
+    "-DSVS_RUNTIME_ENABLE_IVF=${ENABLE_IVF:-OFF}"
 )
 
 if [ -n "$SVS_URL" ]; then
@@ -45,7 +57,14 @@ cmake --install .
 # Build conda package for cpp runtime bindings
 source /opt/conda/etc/profile.d/conda.sh
 cd /workspace
-ENABLE_LVQ_LEANVEC=${ENABLE_LVQ_LEANVEC:-ON} SVS_URL="${SVS_URL}" SUFFIX="${SUFFIX}" conda build bindings/cpp/conda-recipe --output-folder /workspace/conda-bld
+# Free disk space by removing large build artifacts not needed for ctest or conda build.
+# Keep the test binary (tests/) so the workflow can run ctest in a subsequent step.
+find /workspace/bindings/cpp/build_cpp_bindings -name '*.o' -delete 2>/dev/null || true
+find /workspace/bindings/cpp/build_cpp_bindings -name '*.a' -delete 2>/dev/null || true
+find /workspace/bindings/cpp/build_cpp_bindings -name '*.so*' -not -path '*/tests/*' -not -name 'libsvs_runtime*' -delete 2>/dev/null || true
+# Use /workspace for temp files to avoid filling up /tmp during LTO compilation
+mkdir -p /workspace/tmp
+TMPDIR=/workspace/tmp ENABLE_LVQ_LEANVEC=${ENABLE_LVQ_LEANVEC:-ON} ENABLE_IVF=${ENABLE_IVF:-OFF} SVS_URL="${SVS_URL}" SUFFIX="${SUFFIX}" conda build bindings/cpp/conda-recipe --output-folder /workspace/conda-bld
 
 # Create tarball with symlink for compatibility
 cd /workspace/install_cpp_bindings && \

.github/scripts/test-cpp-runtime-bindings.sh

@@ -29,13 +29,20 @@ conda activate svsenv
 conda config --set solver libmamba
 conda install -y -c conda-forge cmake=3.30.4 make=4.2 swig=4.0 "numpy>=2.0,<3.0" scipy=1.16 pytest=7.4 gflags=2.2
 conda install -y -c conda-forge gxx_linux-64=14.2 sysroot_linux-64=2.17
-conda install -y mkl=2022.2.1 mkl-devel=2022.2.1
+conda install -y mkl=2025.3 mkl-devel=2025.3
 
 # Install libsvs-runtime from local conda package
 conda install -y /runtime_conda/libsvs-runtime-*.conda
 
 # Validate python and C++ tests against FAISS CI
-git clone https://github.com/facebookresearch/faiss.git
+ENABLE_IVF="${ENABLE_IVF:-OFF}"
+if [ "${ENABLE_IVF}" = "ON" ]; then
+  echo "IVF enabled: cloning forked FAISS with IVF support"
+  git clone --branch ib/svs_ivf https://github.com/ibhati/faiss.git
+else
+  echo "IVF disabled: cloning upstream FAISS"
+  git clone https://github.com/facebookresearch/faiss.git
+fi
 cd faiss
 
 echo "==============================================="

.github/workflows/build-cpp-runtime-bindings.yml

@@ -37,10 +37,16 @@ jobs:
         include:
           - name: "with static library"
             enable_lvq_leanvec: "ON"
+            enable_ivf: "OFF"
             suffix: ""
           - name: "public only"
             enable_lvq_leanvec: "OFF"
+            enable_ivf: "OFF"
             suffix: "-public-only"
+          - name: "IVF public only"
+            enable_lvq_leanvec: "OFF"
+            enable_ivf: "ON"
+            suffix: "-ivf"
       fail-fast: false
 
     steps:
@@ -56,6 +62,7 @@ jobs:
             -v ${{ github.workspace }}:/workspace \
             -w /workspace \
             -e ENABLE_LVQ_LEANVEC=${{ matrix.enable_lvq_leanvec }} \
+            -e ENABLE_IVF=${{ matrix.enable_ivf }} \
             -e SUFFIX=${{ matrix.suffix }} \
             svs-manylinux228:latest \
             /bin/bash .github/scripts/build-cpp-runtime-bindings.sh
@@ -80,8 +87,11 @@ jobs:
           docker run --rm \
             -v ${{ github.workspace }}:/workspace \
             -w /workspace \
+            -e ENABLE_IVF=${{ matrix.enable_ivf }} \
             svs-manylinux228:latest \
-            /bin/bash -c "source /etc/bashrc || true && ctest --test-dir bindings/cpp/build_cpp_bindings/tests --output-on-failure --no-tests=error --verbose"
+            /bin/bash -c "source /etc/bashrc || true && \
+              if [ \"\${ENABLE_IVF}\" = 'ON' ] && [ -f /opt/intel/oneapi/setvars.sh ]; then source /opt/intel/oneapi/setvars.sh 2>/dev/null || true; fi && \
+              ctest --test-dir bindings/cpp/build_cpp_bindings/tests --output-on-failure --no-tests=error --verbose"
 
   # Run full test script using the built artifacts
   test:
@@ -93,8 +103,13 @@ jobs:
         include:
           - name: "with static library"
             suffix: ""
+            enable_ivf: "OFF"
           - name: "public only"
             suffix: "-public-only"
+            enable_ivf: "OFF"
+          - name: "IVF public only"
+            suffix: "-ivf"
+            enable_ivf: "ON"
 
     steps:
       - uses: actions/checkout@v6
@@ -121,5 +136,6 @@ jobs:
             -v ${{ github.workspace }}/runtime_conda:/runtime_conda \
             -w /workspace \
             -e SUFFIX=${{ matrix.suffix }} \
+            -e ENABLE_IVF=${{ matrix.enable_ivf }} \
             svs-manylinux228:latest \
             /bin/bash .github/scripts/test-cpp-runtime-bindings.sh

bindings/cpp/CMakeLists.txt

@@ -16,6 +16,9 @@ cmake_minimum_required(VERSION 3.21)
 project(svs_runtime VERSION 0.2.0 LANGUAGES CXX)
 set(TARGET_NAME svs_runtime)
 
+# IVF requires MKL, so it's optional
+option(SVS_RUNTIME_ENABLE_IVF "Enable compilation of SVS runtime with IVF support (requires MKL)" OFF)
+
 set(SVS_RUNTIME_HEADERS
     include/svs/runtime/version.h
     include/svs/runtime/api_defs.h
@@ -38,6 +41,23 @@ set(SVS_RUNTIME_SOURCES
     src/flat_index.cpp
 )
 
+# Add IVF files if enabled
+if (SVS_RUNTIME_ENABLE_IVF)
+    message(STATUS "SVS runtime will be built with IVF support (requires MKL)")
+    list(APPEND SVS_RUNTIME_HEADERS
+        include/svs/runtime/ivf_index.h
+        include/svs/runtime/dynamic_ivf_index.h
+    )
+    list(APPEND SVS_RUNTIME_SOURCES
+        src/ivf_index_impl.h
+        src/dynamic_ivf_index_impl.h
+        src/ivf_index.cpp
+        src/dynamic_ivf_index.cpp
+    )
+else()
+    message(STATUS "SVS runtime will be built without IVF support")
+endif()
+
 option(SVS_RUNTIME_ENABLE_LVQ_LEANVEC "Enable compilation of SVS runtime with LVQ and LeanVec support" ON)
 if (SVS_RUNTIME_ENABLE_LVQ_LEANVEC)
     message(STATUS "SVS runtime will be built with LVQ support")
@@ -75,21 +95,24 @@ set_target_properties(${TARGET_NAME} PROPERTIES CXX_EXTENSIONS OFF)
 set_target_properties(${TARGET_NAME} PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR} )
 
 if (SVS_RUNTIME_ENABLE_LVQ_LEANVEC)
-    if(DEFINED SVS_LVQ_HEADER AND DEFINED SVS_LEANVEC_HEADER)
-        # expected that pre-defined headers are implementation headers
-        message(STATUS "Using pre-defined LVQ header: ${SVS_LVQ_HEADER}")
-        message(STATUS "Using pre-defined LeanVec header: ${SVS_LEANVEC_HEADER}")
-    else()
-        set(SVS_LVQ_HEADER "svs/extensions/vamana/lvq.h")
-        set(SVS_LEANVEC_HEADER "svs/extensions/vamana/leanvec.h")
-    endif()
 
     if(RUNTIME_BINDINGS_PRIVATE_SOURCE_BUILD)
         message(STATUS "Building directly from private sources")
         target_link_libraries(${TARGET_NAME} PRIVATE
             svs::svs
+            svs_compile_options
         )
         link_mkl_static(${TARGET_NAME})
+        if(SVS_LVQ_HEADER)
+            target_compile_definitions(${TARGET_NAME} PRIVATE
+                SVS_LVQ_HEADER="${SVS_LVQ_HEADER}"
+            )
+        endif()
+        if(SVS_LEANVEC_HEADER)
+            target_compile_definitions(${TARGET_NAME} PRIVATE
+                SVS_LEANVEC_HEADER="${SVS_LEANVEC_HEADER}"
+            )
+        endif()
     elseif(TARGET svs::svs)
         message(FATAL_ERROR
             "Pre-built LVQ/LeanVec SVS library cannot be used in SVS main build. "
@@ -123,13 +146,14 @@ if (SVS_RUNTIME_ENABLE_LVQ_LEANVEC)
             svs::svs_static_library
         )
     endif()
-    target_compile_definitions(${TARGET_NAME} PRIVATE
-        PUBLIC "SVS_LVQ_HEADER=\"${SVS_LVQ_HEADER}\""
-        PUBLIC "SVS_LEANVEC_HEADER=\"${SVS_LEANVEC_HEADER}\""
-    )
+    target_compile_definitions(${TARGET_NAME} PUBLIC SVS_RUNTIME_HAVE_LVQ_LEANVEC)
 else()
     # Include the SVS library directly if needed.
     if (NOT TARGET svs::svs)
+        # Pass IVF flag to parent build if IVF is enabled
+        if (SVS_RUNTIME_ENABLE_IVF)
+            set(SVS_EXPERIMENTAL_ENABLE_IVF ON CACHE BOOL "" FORCE)
+        endif()
         add_subdirectory("../.." "${CMAKE_CURRENT_BINARY_DIR}/svs")
     endif()
     target_link_libraries(${TARGET_NAME} PRIVATE
@@ -139,6 +163,11 @@ else()
     )
 endif()
 
+# IVF requires Intel(R) MKL support
+if (SVS_RUNTIME_ENABLE_IVF)
+    target_compile_definitions(${TARGET_NAME} PRIVATE SVS_RUNTIME_ENABLE_IVF SVS_HAVE_MKL=1)
+endif()
+
 # installing
 include(GNUInstallDirs)
 

bindings/cpp/conda-recipe/build.sh

@@ -25,11 +25,23 @@ else
     echo "WARNING: gcc-toolset-11 not found, proceeding without sourcing it"
 fi
 
+# Source MKL environment if IVF is enabled (IVF requires MKL)
+if [ "${ENABLE_IVF:-OFF}" = "ON" ]; then
+    if [ -f /opt/intel/oneapi/setvars.sh ]; then
+        source /opt/intel/oneapi/setvars.sh --include-intel-llvm 2>/dev/null || true
+        echo "MKL sourced for IVF build: MKLROOT=${MKLROOT}"
+    else
+        echo "ERROR: IVF enabled but MKL setvars.sh not found"
+        exit 1
+    fi
+fi
+
 # build runtime tests flag?
 CMAKE_ARGS=(
     "-DCMAKE_INSTALL_PREFIX=${PREFIX}"
     "-DSVS_BUILD_RUNTIME_TESTS=OFF"
     "-DSVS_RUNTIME_ENABLE_LVQ_LEANVEC=${ENABLE_LVQ_LEANVEC:-ON}"
+    "-DSVS_RUNTIME_ENABLE_IVF=${ENABLE_IVF:-OFF}"
 )
 
 # Add SVS_URL if specified (for fetching static library)

bindings/cpp/conda-recipe/meta.yaml

@@ -29,6 +29,11 @@ build:
   string: {{ GIT_DESCRIBE_HASH }}_{{ buildnumber }}{{ variant_suffix }}
   skip: true  # [not linux]
   include_recipe: False
+  script_env:
+    - ENABLE_IVF
+    - ENABLE_LVQ_LEANVEC
+    - SUFFIX
+    - SVS_URL
 
 requirements:
   build: