Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for working with binary indexes. #80

Open
wants to merge 15 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions faiss-sys/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,10 @@ fn static_link_faiss() {
cfg.define("FAISS_ENABLE_C_API", "ON")
.define("BUILD_SHARED_LIBS", "OFF")
.define("CMAKE_BUILD_TYPE", "Release")
.define("FAISS_ENABLE_GPU", if cfg!(feature = "gpu") {
"ON"
} else {
"OFF"
})
.define(
"FAISS_ENABLE_GPU",
if cfg!(feature = "gpu") { "ON" } else { "OFF" },
)
.define("FAISS_ENABLE_PYTHON", "OFF")
.define("BUILD_TESTING", "OFF")
.very_verbose(true);
Expand Down
2 changes: 1 addition & 1 deletion faiss-sys/faiss
Submodule faiss updated 44 files
+4 −4 .circleci/config.yml
+20 −17 INSTALL.md
+2 −135 benchs/link_and_code/README.md
+0 −300 benchs/link_and_code/bench_link_and_code.py
+0 −236 benchs/link_and_code/datasets.py
+0 −241 benchs/link_and_code/neighbor_codec.py
+12 −0 c_api/clone_index_c.cpp
+4 −0 c_api/clone_index_c.h
+15 −1 c_api/index_factory_c.cpp
+10 −1 c_api/index_factory_c.h
+3 −3 conda/faiss-gpu-raft/meta.yaml
+5 −5 contrib/torch_utils.py
+1 −0 demos/offline_ivf/config_ssnpp.yaml
+20 −68 demos/offline_ivf/offline_ivf.py
+4 −1 faiss/CMakeLists.txt
+6 −10 faiss/IndexHNSW.cpp
+8 −0 faiss/IndexIVF.h
+21 −2 faiss/IndexIVFPQFastScan.cpp
+2 −0 faiss/gpu/GpuCloner.cpp
+2 −8 faiss/gpu/GpuIndex.h
+2 −1 faiss/gpu/impl/RaftIVFFlat.cu
+29 −0 faiss/gpu/test/test_index_cpu_to_gpu.py
+7 −0 faiss/impl/FaissAssert.h
+2 −2 faiss/impl/HNSW.cpp
+5 −15 faiss/impl/HNSW.h
+8 −8 faiss/impl/LocalSearchQuantizer.cpp
+34 −0 faiss/impl/LookupTableScaler.h
+20 −3 faiss/impl/ResultHandler.h
+1 −1 faiss/impl/index_read.cpp
+12 −4 faiss/impl/io.cpp
+4 −4 faiss/impl/io.h
+447 −0 faiss/impl/pq4_fast_scan_search_qbs.cpp
+1 −1 faiss/impl/simd_result_handlers.h
+6 −0 faiss/index_io.h
+2 −2 faiss/python/swigfaiss.swig
+6 −1 faiss/utils/simdlib.h
+296 −0 faiss/utils/simdlib_avx512.h
+26 −3 tests/CMakeLists.txt
+61 −0 tests/test_disable_pq_sdc_tables.cpp
+48 −15 tests/test_fast_scan_ivf.py
+10 −0 tests/test_graph_based.py
+9 −0 tests/test_index_binary.py
+5 −30 tests/test_merge.cpp
+39 −0 tests/test_util.h
1,015 changes: 515 additions & 500 deletions faiss-sys/src/bindings.rs

Large diffs are not rendered by default.

1,015 changes: 515 additions & 500 deletions faiss-sys/src/bindings_gpu.rs

Large diffs are not rendered by default.

14 changes: 14 additions & 0 deletions faiss-sys/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,20 @@ mod tests {
assert!(!last_error.is_null());
}
}
#[test]
fn flat_index_binary() {
const D: usize = 8;
unsafe {
let description = CString::new::<&str>("BFlat".as_ref()).unwrap();
let mut index_ptr = ::std::ptr::null_mut();
let code = faiss_index_binary_factory(
&mut index_ptr,
(D & 0x7FFF_FFFF) as i32,
description.as_ptr(),
);
assert_eq!(code, 0);
}
}

#[test]
fn flat_index() {
Expand Down
77 changes: 76 additions & 1 deletion src/index/io.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@

use crate::error::{Error, Result};
use crate::faiss_try;
use crate::index::{CpuIndex, FromInnerPtr, IndexImpl, NativeIndex};
use crate::index::{
BinaryIndexImpl, CpuIndex, CpuIndexBinary, FromInnerPtr, FromInnerPtrBinary, IndexImpl,
NativeIndex, NativeIndexBinary,
};
use faiss_sys::*;
use std::ffi::CString;
use std::os::raw::c_int;
Expand Down Expand Up @@ -31,6 +34,30 @@ where
}
}

/// Write a binary index to a file.
///
/// # Error
///
/// This function returns an error if the description contains any byte with the value `\0` (since
/// it cannot be converted to a C string), or if the internal index writing operation fails.
pub fn write_index_binary<I, P>(index: &I, file_name: P) -> Result<()>
where
I: NativeIndexBinary,
I: CpuIndexBinary,
P: AsRef<str>,
{
unsafe {
let f = file_name.as_ref();
let f = CString::new(f).map_err(|_| Error::BadFilePath)?;

faiss_try(faiss_write_index_binary_fname(
index.inner_ptr(),
f.as_ptr(),
))?;
Ok(())
}
}

/// Read an index from a file.
///
/// # Error
Expand All @@ -54,6 +81,29 @@ where
}
}

/// Read a binary index from a file.
///
/// # Error
///
/// This function returns an error if the description contains any byte with the value `\0` (since
/// it cannot be converted to a C string), or if the internal index reading operation fails.
pub fn read_index_binary<P>(file_name: P) -> Result<BinaryIndexImpl>
where
P: AsRef<str>,
{
unsafe {
let f = file_name.as_ref();
let f = CString::new(f).map_err(|_| Error::BadFilePath)?;
let mut inner = ptr::null_mut();
faiss_try(faiss_read_index_binary_fname(
f.as_ptr(),
IoFlags::MEM_RESIDENT.into(),
&mut inner,
))?;
Ok(BinaryIndexImpl::from_inner_ptr(inner))
}
}

/// Read an index from a file with I/O flags.
///
/// You can memory map some index types with this.
Expand All @@ -79,6 +129,31 @@ where
}
}

/// Read a binary index from a file with I/O flags.
///
/// You can memory map some index types with this.
///
/// # Error
///
/// This function returns an error if the description contains any byte with the value `\0` (since
/// it cannot be converted to a C string), or if the internal index reading operation fails.
pub fn read_index_binary_with_flags<P>(file_name: P, io_flags: IoFlags) -> Result<BinaryIndexImpl>
where
P: AsRef<str>,
{
unsafe {
let f = file_name.as_ref();
let f = CString::new(f).map_err(|_| Error::BadFilePath)?;
let mut inner = ptr::null_mut();
faiss_try(faiss_read_index_binary_fname(
f.as_ptr(),
io_flags.0 as c_int,
&mut inner,
))?;
Ok(BinaryIndexImpl::from_inner_ptr(inner))
}
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down
Loading
Loading