From a63d0e0247169d1cc1b861dec1c5c1bb571a31bb Mon Sep 17 00:00:00 2001 From: Debian Deep Learning Team Date: Sun, 6 Feb 2022 12:19:35 +0100 Subject: Use system GTest Gbp-Pq: Name 0001-Use-system-GTest.patch --- CMakeLists.txt | 1 + tests/CMakeLists.txt | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6cdc37c..7d0ad00 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,6 +56,7 @@ option(FAISS_ENABLE_GPU "Enable support for GPU indexes." ON) option(FAISS_ENABLE_RAFT "Enable RAFT for GPU indexes." OFF) option(FAISS_ENABLE_PYTHON "Build Python extension." ON) option(FAISS_ENABLE_C_API "Build C API." OFF) +option(FAISS_USE_SYSTEM_GTEST "Use system-provided gtest library." ON) if(FAISS_ENABLE_GPU) set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER}) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 10243b9..b716226 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -58,18 +58,24 @@ if(FAISS_OPT_LEVEL STREQUAL "avx512") target_link_libraries(faiss_test PRIVATE faiss_avx512) endif() +if(NOT FAISS_USE_SYSTEM_GTEST) include(FetchContent) FetchContent_Declare(googletest URL "https://github.com/google/googletest/archive/release-1.12.1.tar.gz") set(BUILD_GMOCK CACHE BOOL OFF) set(INSTALL_GTEST CACHE BOOL OFF) FetchContent_MakeAvailable(googletest) +else() + find_package(GTest REQUIRED) + include_directories(${GTEST_INCLUDE_DIRS}) +endif() + find_package(OpenMP REQUIRED) target_link_libraries(faiss_test PRIVATE OpenMP::OpenMP_CXX - gtest_main + GTest::gtest_main $<$:raft::raft> ) -- cgit v1.2.3 From 5b78093f7d7865645ecdcd0b5504324e982bb5b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Sun, 6 Feb 2022 12:23:54 +0100 Subject: Use SWIGWORDSIZE32 on 32 bit architectures Gbp-Pq: Name 0002-Use-SWIGWORDSIZE32-on-32-bit-architectures.patch --- faiss/python/CMakeLists.txt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/faiss/python/CMakeLists.txt b/faiss/python/CMakeLists.txt index 8bca710..26eaa54 100644 --- a/faiss/python/CMakeLists.txt +++ b/faiss/python/CMakeLists.txt @@ -25,9 +25,15 @@ macro(configure_swigfaiss source) USE_TARGET_INCLUDE_DIRECTORIES TRUE ) if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin" AND NOT WIN32) - set_source_files_properties(${source} PROPERTIES - SWIG_FLAGS -DSWIGWORDSIZE64 - ) + if(CMAKE_SIZEOF_VOID_P EQUAL 8) + set_source_files_properties(${source} PROPERTIES + SWIG_FLAGS -DSWIGWORDSIZE64 + ) + else() + set_source_files_properties(${source} PROPERTIES + SWIG_FLAGS -DSWIGWORDSIZE32 + ) + endif() endif() if(WIN32) set_source_files_properties(${source} PROPERTIES -- cgit v1.2.3 From c8e3a5133a0153c3c05d85723c8b9f258c87e790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Tue, 8 Feb 2022 11:53:12 +0100 Subject: Fix CMake package export Gbp-Pq: Name 0003-Fix-CMake-package-export.patch --- faiss/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt index a890a46..ee54600 100644 --- a/faiss/CMakeLists.txt +++ b/faiss/CMakeLists.txt @@ -358,9 +358,9 @@ configure_file(${PROJECT_SOURCE_DIR}/cmake/faiss-config.cmake.in ) install(FILES ${PROJECT_BINARY_DIR}/cmake/faiss-config.cmake ${PROJECT_BINARY_DIR}/cmake/faiss-config-version.cmake - DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/faiss + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/faiss ) install(EXPORT faiss-targets - DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/faiss + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/faiss ) -- cgit v1.2.3 From d59c074a9791e18abcca8ef4b271300ee5aa7a10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Wed, 9 Feb 2022 01:28:47 +0100 Subject: Fix implementation for 32 bit size_t Gbp-Pq: Name 0004-Fix-implementation-for-32-bit-size_t.patch --- faiss/index_factory.cpp | 2 +- faiss/invlists/OnDiskInvertedLists.cpp | 5 +++-- faiss/utils/utils.cpp | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/faiss/index_factory.cpp b/faiss/index_factory.cpp index 0d61b73..883753e 100644 --- a/faiss/index_factory.cpp +++ b/faiss/index_factory.cpp @@ -731,7 +731,7 @@ std::unique_ptr index_factory_sub( } if (verbose) { - printf("after () normalization: %s %ld parenthesis indexes d=%d\n", + printf("after () normalization: %s %zu parenthesis indexes d=%d\n", description.c_str(), parenthesis_indexes.size(), d); diff --git a/faiss/invlists/OnDiskInvertedLists.cpp b/faiss/invlists/OnDiskInvertedLists.cpp index 3017d16..2c234d7 100644 --- a/faiss/invlists/OnDiskInvertedLists.cpp +++ b/faiss/invlists/OnDiskInvertedLists.cpp @@ -11,6 +11,7 @@ #include +#include #include #include @@ -524,7 +525,7 @@ void OnDiskInvertedLists::free_slot(size_t offset, size_t capacity) { it++; } - size_t inf = ((size_t)1) << 60; + size_t inf = (std::numeric_limits::max() - (std::numeric_limits::max() >> 1)) >> 1; size_t end_prev = inf; if (it != slots.begin()) { @@ -533,7 +534,7 @@ void OnDiskInvertedLists::free_slot(size_t offset, size_t capacity) { end_prev = prev->offset + prev->capacity; } - size_t begin_next = ((size_t)1) << 60; + size_t begin_next = inf; if (it != slots.end()) { begin_next = it->offset; } diff --git a/faiss/utils/utils.cpp b/faiss/utils/utils.cpp index dc6fadd..147afb5 100644 --- a/faiss/utils/utils.cpp +++ b/faiss/utils/utils.cpp @@ -169,7 +169,7 @@ size_t get_mem_usage_kb() { char buf[256]; if (!fgets(buf, 256, f)) break; - if (sscanf(buf, "VmRSS: %ld kB", &sz) == 1) + if (sscanf(buf, "VmRSS: %zu kB", &sz) == 1) break; } fclose(f); -- cgit v1.2.3 From 1347fffe00bb60ea13a749dda3551c140fcefcc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Wed, 9 Feb 2022 01:38:34 +0100 Subject: Fix floating point comparisons in unit tests Gbp-Pq: Name 0005-Fix-floating-point-comparisons-in-unit-tests.patch --- tests/test_lowlevel_ivf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_lowlevel_ivf.cpp b/tests/test_lowlevel_ivf.cpp index e28e2a9..a4d8410 100644 --- a/tests/test_lowlevel_ivf.cpp +++ b/tests/test_lowlevel_ivf.cpp @@ -359,7 +359,7 @@ void test_lowlevel_access_binary(const char* index_key) { float computed_D = scanner->distance_to_code( xb.data() + vno * il->code_size); - EXPECT_EQ(computed_D, D[jj]); + EXPECT_FLOAT_EQ(computed_D, D[jj]); } } } -- cgit v1.2.3 From 93806e87ad3cf3316b07bcd938c25ac6e3045ca2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Wed, 9 Feb 2022 08:08:13 +0100 Subject: Add GTest filter Gbp-Pq: Name 0006-Add-GTest-filter.patch --- tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b716226..c1510ec 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -81,4 +81,4 @@ target_link_libraries(faiss_test PRIVATE # Defines `gtest_discover_tests()`. include(GoogleTest) -gtest_discover_tests(faiss_test) +gtest_discover_tests(faiss_test TEST_FILTER "${FAISS_GTEST_FILTER}") -- cgit v1.2.3 From 309833c27f6b1d5f90bfe38f4ff43080afdbda42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Thu, 22 Jun 2023 17:51:36 +0200 Subject: Fix build with gcc-13 Gbp-Pq: Name 0007-Fix-build-with-gcc-13.patch --- faiss/Index.h | 1 + 1 file changed, 1 insertion(+) diff --git a/faiss/Index.h b/faiss/Index.h index 3d1bdb9..9fc9e19 100644 --- a/faiss/Index.h +++ b/faiss/Index.h @@ -11,6 +11,7 @@ #define FAISS_INDEX_H #include +#include #include #include #include -- cgit v1.2.3 From 2adffc324d294e27d535e365287b4fe4493832b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Sat, 24 Jun 2023 21:29:46 +0200 Subject: Install missing header file Gbp-Pq: Name 0008-Install-missing-header-file.patch --- faiss/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt index ee54600..aec195b 100644 --- a/faiss/CMakeLists.txt +++ b/faiss/CMakeLists.txt @@ -149,6 +149,7 @@ set(FAISS_HEADERS impl/AuxIndexStructures.h impl/CodePacker.h impl/IDSelector.h + impl/CodePacker.h impl/DistanceComputer.h impl/FaissAssert.h impl/FaissException.h -- cgit v1.2.3 From 0c01501e37145e9703b7083c83db66705e0202ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Wed, 17 Apr 2024 17:20:52 +0200 Subject: Fix FTBFS with swig >= 4.2.0 Origin: upstream, https://github.com/facebookresearch/faiss/pull/3315 Bug: https://github.com/facebookresearch/faiss/issues/3239 Bug-Debian: https://bugs.debian.org/1061741 Gbp-Pq: Name 0009-Fix-FTBFS-with-swig-4.2.0.patch --- faiss/python/swigfaiss.swig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/faiss/python/swigfaiss.swig b/faiss/python/swigfaiss.swig index fb7f50d..c536676 100644 --- a/faiss/python/swigfaiss.swig +++ b/faiss/python/swigfaiss.swig @@ -1022,14 +1022,14 @@ PyObject *swig_ptr (PyObject *a) return SWIG_NewPointerObj(data, SWIGTYPE_p_bool, 0); } if(PyArray_TYPE(ao) == NPY_UINT64) { -#ifdef SWIGWORDSIZE64 +#if (__SIZEOF_LONG__ == 8) return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long_long, 0); #endif } if(PyArray_TYPE(ao) == NPY_INT64) { -#ifdef SWIGWORDSIZE64 +#if (__SIZEOF_LONG__ == 8) return SWIG_NewPointerObj(data, SWIGTYPE_p_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_long_long, 0); -- cgit v1.2.3 From 2876319c93ac7510fd0a3a016b9f5657d3a9031f Mon Sep 17 00:00:00 2001 From: Debian Deep Learning Team Date: Sun, 6 Feb 2022 12:19:35 +0100 Subject: Use system GTest Gbp-Pq: Name 0001-Use-system-GTest.patch --- CMakeLists.txt | 1 + tests/CMakeLists.txt | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6cdc37c..7d0ad00 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,6 +56,7 @@ option(FAISS_ENABLE_GPU "Enable support for GPU indexes." ON) option(FAISS_ENABLE_RAFT "Enable RAFT for GPU indexes." OFF) option(FAISS_ENABLE_PYTHON "Build Python extension." ON) option(FAISS_ENABLE_C_API "Build C API." OFF) +option(FAISS_USE_SYSTEM_GTEST "Use system-provided gtest library." ON) if(FAISS_ENABLE_GPU) set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER}) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 10243b9..b716226 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -58,18 +58,24 @@ if(FAISS_OPT_LEVEL STREQUAL "avx512") target_link_libraries(faiss_test PRIVATE faiss_avx512) endif() +if(NOT FAISS_USE_SYSTEM_GTEST) include(FetchContent) FetchContent_Declare(googletest URL "https://github.com/google/googletest/archive/release-1.12.1.tar.gz") set(BUILD_GMOCK CACHE BOOL OFF) set(INSTALL_GTEST CACHE BOOL OFF) FetchContent_MakeAvailable(googletest) +else() + find_package(GTest REQUIRED) + include_directories(${GTEST_INCLUDE_DIRS}) +endif() + find_package(OpenMP REQUIRED) target_link_libraries(faiss_test PRIVATE OpenMP::OpenMP_CXX - gtest_main + GTest::gtest_main $<$:raft::raft> ) -- cgit v1.2.3 From a9ebd03e95e5b4d2bc1cd3c6ac7ba96b6d536cfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Sun, 6 Feb 2022 12:23:54 +0100 Subject: Use SWIGWORDSIZE32 on 32 bit architectures Gbp-Pq: Name 0002-Use-SWIGWORDSIZE32-on-32-bit-architectures.patch --- faiss/python/CMakeLists.txt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/faiss/python/CMakeLists.txt b/faiss/python/CMakeLists.txt index 8bca710..26eaa54 100644 --- a/faiss/python/CMakeLists.txt +++ b/faiss/python/CMakeLists.txt @@ -25,9 +25,15 @@ macro(configure_swigfaiss source) USE_TARGET_INCLUDE_DIRECTORIES TRUE ) if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin" AND NOT WIN32) - set_source_files_properties(${source} PROPERTIES - SWIG_FLAGS -DSWIGWORDSIZE64 - ) + if(CMAKE_SIZEOF_VOID_P EQUAL 8) + set_source_files_properties(${source} PROPERTIES + SWIG_FLAGS -DSWIGWORDSIZE64 + ) + else() + set_source_files_properties(${source} PROPERTIES + SWIG_FLAGS -DSWIGWORDSIZE32 + ) + endif() endif() if(WIN32) set_source_files_properties(${source} PROPERTIES -- cgit v1.2.3 From a7ee9b4c27a6401cbd989cbeb9a8e87e1cc5c0b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Tue, 8 Feb 2022 11:53:12 +0100 Subject: Fix CMake package export Gbp-Pq: Name 0003-Fix-CMake-package-export.patch --- faiss/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt index a890a46..ee54600 100644 --- a/faiss/CMakeLists.txt +++ b/faiss/CMakeLists.txt @@ -358,9 +358,9 @@ configure_file(${PROJECT_SOURCE_DIR}/cmake/faiss-config.cmake.in ) install(FILES ${PROJECT_BINARY_DIR}/cmake/faiss-config.cmake ${PROJECT_BINARY_DIR}/cmake/faiss-config-version.cmake - DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/faiss + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/faiss ) install(EXPORT faiss-targets - DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/faiss + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/faiss ) -- cgit v1.2.3 From 01e1da87d8b75cc949c96597387ac16980454d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Wed, 9 Feb 2022 01:28:47 +0100 Subject: Fix implementation for 32 bit size_t Gbp-Pq: Name 0004-Fix-implementation-for-32-bit-size_t.patch --- faiss/IndexIVF.cpp | 2 +- faiss/IndexIVFIndependentQuantizer.cpp | 2 +- faiss/index_factory.cpp | 2 +- faiss/invlists/OnDiskInvertedLists.cpp | 4 ++-- faiss/utils/utils.cpp | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/faiss/IndexIVF.cpp b/faiss/IndexIVF.cpp index 95d3bc9..ffaae4a 100644 --- a/faiss/IndexIVF.cpp +++ b/faiss/IndexIVF.cpp @@ -1150,7 +1150,7 @@ void IndexIVF::train(idx_t n, const float* x) { // optional subsampling idx_t max_nt = train_encoder_num_vectors(); if (max_nt <= 0) { - max_nt = (size_t)1 << 35; + max_nt = (size_t)1 << 30; } TransformedVectors tv( diff --git a/faiss/IndexIVFIndependentQuantizer.cpp b/faiss/IndexIVFIndependentQuantizer.cpp index 2073dd2..d5be6e3 100644 --- a/faiss/IndexIVFIndependentQuantizer.cpp +++ b/faiss/IndexIVFIndependentQuantizer.cpp @@ -140,7 +140,7 @@ void IndexIVFIndependentQuantizer::train(idx_t n, const float* x) { // optional subsampling idx_t max_nt = index_ivf->train_encoder_num_vectors(); if (max_nt <= 0) { - max_nt = (size_t)1 << 35; + max_nt = (size_t)1 << 30; } SubsampledVectors sv(index_ivf->d, &n, max_nt, x); diff --git a/faiss/index_factory.cpp b/faiss/index_factory.cpp index 0d61b73..883753e 100644 --- a/faiss/index_factory.cpp +++ b/faiss/index_factory.cpp @@ -731,7 +731,7 @@ std::unique_ptr index_factory_sub( } if (verbose) { - printf("after () normalization: %s %ld parenthesis indexes d=%d\n", + printf("after () normalization: %s %zu parenthesis indexes d=%d\n", description.c_str(), parenthesis_indexes.size(), d); diff --git a/faiss/invlists/OnDiskInvertedLists.cpp b/faiss/invlists/OnDiskInvertedLists.cpp index 3017d16..b363747 100644 --- a/faiss/invlists/OnDiskInvertedLists.cpp +++ b/faiss/invlists/OnDiskInvertedLists.cpp @@ -524,7 +524,7 @@ void OnDiskInvertedLists::free_slot(size_t offset, size_t capacity) { it++; } - size_t inf = ((size_t)1) << 60; + size_t inf = size_t(1) << (8 * __SIZEOF_SIZE_T__ - 2); size_t end_prev = inf; if (it != slots.begin()) { @@ -533,7 +533,7 @@ void OnDiskInvertedLists::free_slot(size_t offset, size_t capacity) { end_prev = prev->offset + prev->capacity; } - size_t begin_next = ((size_t)1) << 60; + size_t begin_next = inf; if (it != slots.end()) { begin_next = it->offset; } diff --git a/faiss/utils/utils.cpp b/faiss/utils/utils.cpp index dc6fadd..147afb5 100644 --- a/faiss/utils/utils.cpp +++ b/faiss/utils/utils.cpp @@ -169,7 +169,7 @@ size_t get_mem_usage_kb() { char buf[256]; if (!fgets(buf, 256, f)) break; - if (sscanf(buf, "VmRSS: %ld kB", &sz) == 1) + if (sscanf(buf, "VmRSS: %zu kB", &sz) == 1) break; } fclose(f); -- cgit v1.2.3 From 79ff5a4c453ae2879469e040699fd393b2bb2fe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Wed, 9 Feb 2022 01:38:34 +0100 Subject: Fix floating point comparisons in unit tests Gbp-Pq: Name 0005-Fix-floating-point-comparisons-in-unit-tests.patch --- tests/test_lowlevel_ivf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_lowlevel_ivf.cpp b/tests/test_lowlevel_ivf.cpp index e28e2a9..a4d8410 100644 --- a/tests/test_lowlevel_ivf.cpp +++ b/tests/test_lowlevel_ivf.cpp @@ -359,7 +359,7 @@ void test_lowlevel_access_binary(const char* index_key) { float computed_D = scanner->distance_to_code( xb.data() + vno * il->code_size); - EXPECT_EQ(computed_D, D[jj]); + EXPECT_FLOAT_EQ(computed_D, D[jj]); } } } -- cgit v1.2.3 From a73ba6f1593e467e27326a1a5f2e3b06ebab363b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Wed, 9 Feb 2022 08:08:13 +0100 Subject: Add GTest filter Gbp-Pq: Name 0006-Add-GTest-filter.patch --- tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b716226..c1510ec 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -81,4 +81,4 @@ target_link_libraries(faiss_test PRIVATE # Defines `gtest_discover_tests()`. include(GoogleTest) -gtest_discover_tests(faiss_test) +gtest_discover_tests(faiss_test TEST_FILTER "${FAISS_GTEST_FILTER}") -- cgit v1.2.3 From a5ff4313129006343840327ec2a181492f7a3089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Thu, 22 Jun 2023 17:51:36 +0200 Subject: Fix build with gcc-13 Gbp-Pq: Name 0007-Fix-build-with-gcc-13.patch --- faiss/Index.h | 1 + 1 file changed, 1 insertion(+) diff --git a/faiss/Index.h b/faiss/Index.h index 3d1bdb9..9fc9e19 100644 --- a/faiss/Index.h +++ b/faiss/Index.h @@ -11,6 +11,7 @@ #define FAISS_INDEX_H #include +#include #include #include #include -- cgit v1.2.3 From ba8ab3dc94eb3c48df9a78e70f60af84e365d41e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Sat, 24 Jun 2023 21:29:46 +0200 Subject: Install missing header file Gbp-Pq: Name 0008-Install-missing-header-file.patch --- faiss/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt index ee54600..aec195b 100644 --- a/faiss/CMakeLists.txt +++ b/faiss/CMakeLists.txt @@ -149,6 +149,7 @@ set(FAISS_HEADERS impl/AuxIndexStructures.h impl/CodePacker.h impl/IDSelector.h + impl/CodePacker.h impl/DistanceComputer.h impl/FaissAssert.h impl/FaissException.h -- cgit v1.2.3 From 3a31715d7d33d7cb46312d711ac040f92a9d62d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Wed, 17 Apr 2024 17:20:52 +0200 Subject: Fix FTBFS with swig >= 4.2.0 Origin: upstream, https://github.com/facebookresearch/faiss/pull/3315 Bug: https://github.com/facebookresearch/faiss/issues/3239 Bug-Debian: https://bugs.debian.org/1061741 Gbp-Pq: Name 0009-Fix-FTBFS-with-swig-4.2.0.patch --- faiss/python/swigfaiss.swig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/faiss/python/swigfaiss.swig b/faiss/python/swigfaiss.swig index fb7f50d..c536676 100644 --- a/faiss/python/swigfaiss.swig +++ b/faiss/python/swigfaiss.swig @@ -1022,14 +1022,14 @@ PyObject *swig_ptr (PyObject *a) return SWIG_NewPointerObj(data, SWIGTYPE_p_bool, 0); } if(PyArray_TYPE(ao) == NPY_UINT64) { -#ifdef SWIGWORDSIZE64 +#if (__SIZEOF_LONG__ == 8) return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long_long, 0); #endif } if(PyArray_TYPE(ao) == NPY_INT64) { -#ifdef SWIGWORDSIZE64 +#if (__SIZEOF_LONG__ == 8) return SWIG_NewPointerObj(data, SWIGTYPE_p_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_long_long, 0); -- cgit v1.2.3 From bc6656fbb7fbc45e3500238353563beec585781d Mon Sep 17 00:00:00 2001 From: Matthijs Douze Date: Mon, 22 Apr 2024 00:31:45 +0200 Subject: Big endian support Forwarded: https://github.com/facebookresearch/faiss/pull/3361 Gbp-Pq: Name 0010-Big-endian-support.patch --- contrib/vecs_io.py | 12 +++- faiss/cppcontrib/detail/UintReader.h | 121 +++++++++++++++++++++++++++----- faiss/cppcontrib/sa_decode/Level2-inl.h | 114 ++++++++++++++++++++++-------- faiss/impl/platform_macros.h | 8 +++ faiss/impl/pq4_fast_scan.cpp | 11 +++ faiss/python/CMakeLists.txt | 13 ++++ 6 files changed, 231 insertions(+), 48 deletions(-) diff --git a/contrib/vecs_io.py b/contrib/vecs_io.py index ea75d5f..5d18c0b 100644 --- a/contrib/vecs_io.py +++ b/contrib/vecs_io.py @@ -3,6 +3,7 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. +import sys import numpy as np """ @@ -13,6 +14,8 @@ definition of the formats here: http://corpus-texmex.irisa.fr/ def ivecs_read(fname): a = np.fromfile(fname, dtype='int32') + if sys.big_endian: + a.byteswap(inplace=True) d = a[0] return a.reshape(-1, d + 1)[:, 1:].copy() @@ -22,6 +25,7 @@ def fvecs_read(fname): def ivecs_mmap(fname): + assert not sys.big_endian a = np.memmap(fname, dtype='int32', mode='r') d = a[0] return a.reshape(-1, d + 1)[:, 1:] @@ -33,7 +37,11 @@ def fvecs_mmap(fname): def bvecs_mmap(fname): x = np.memmap(fname, dtype='uint8', mode='r') - d = x[:4].view('int32')[0] + if sys.big_endian: + da = x[:4][::-1].copy() + d = da.view('int32')[0] + else: + d = x[:4].view('int32')[0] return x.reshape(-1, d + 4)[:, 4:] @@ -42,6 +50,8 @@ def ivecs_write(fname, m): m1 = np.empty((n, d + 1), dtype='int32') m1[:, 0] = d m1[:, 1:] = m + if sys.big_endian: + m1.byteswap(inplace=True) m1.tofile(fname) diff --git a/faiss/cppcontrib/detail/UintReader.h b/faiss/cppcontrib/detail/UintReader.h index 81e600f..d0b5836 100644 --- a/faiss/cppcontrib/detail/UintReader.h +++ b/faiss/cppcontrib/detail/UintReader.h @@ -7,8 +7,18 @@ #pragma once +#include #include +#ifdef FAISS_BIG_ENDIAN +#define Swap2Bytes(val) ((((val) >> 8) & 0x00FF) | (((val) << 8) & 0xFF00)) + +#define Swap4Bytes(val) \ + ((((val) >> 24) & 0x000000FF) | (((val) >> 8) & 0x0000FF00) | \ + (((val) << 8) & 0x00FF0000) | (((val) << 24) & 0xFF000000)) + +#endif + namespace faiss { namespace cppcontrib { namespace detail { @@ -31,7 +41,11 @@ struct Uint8Reader { if (N_ELEMENTS > CPOS + 3) { const uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 4); +#ifdef FAISS_BIG_ENDIAN + return (code32) >> 24; +#else return (code32 & 0x000000FF); +#endif } else { return codes[CPOS]; } @@ -40,7 +54,11 @@ struct Uint8Reader { if (N_ELEMENTS > CPOS + 2) { const uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 4); +#ifdef FAISS_BIG_ENDIAN + return (code32 & 0x00FF0000) >> 16; +#else return (code32 & 0x0000FF00) >> 8; +#endif } else { return codes[CPOS]; } @@ -49,7 +67,11 @@ struct Uint8Reader { if (N_ELEMENTS > CPOS + 1) { const uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 4); +#ifdef FAISS_BIG_ENDIAN + return (code32 & 0x0000FF00) >> 8; +#else return (code32 & 0x00FF0000) >> 16; +#endif } else { return codes[CPOS]; } @@ -58,7 +80,11 @@ struct Uint8Reader { if (N_ELEMENTS > CPOS) { const uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 4); +#ifdef FAISS_BIG_ENDIAN + return (code32 & 0x000000FF); +#else return (code32) >> 24; +#endif } else { return codes[CPOS]; } @@ -87,40 +113,61 @@ struct Uint10Reader { switch (SUB_ELEMENT) { case 0: { if (N_ELEMENTS > CPOS + 2) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b0000001111111111); } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5 + 0); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b0000001111111111); } } case 1: { if (N_ELEMENTS > CPOS + 1) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b000011111111110000000000) >> 10; } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5 + 1); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b0000111111111100) >> 2; } } case 2: { if (N_ELEMENTS > CPOS) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b00111111111100000000000000000000) >> 20; } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5 + 2); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b0011111111110000) >> 4; } } case 3: { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5 + 3); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b1111111111000000) >> 6; } } @@ -147,45 +194,69 @@ struct Uint12Reader { switch (SUB_ELEMENT) { case 0: { if (N_ELEMENTS > CPOS + 2) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b0000111111111111); } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6 + 0); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b0000111111111111); } } case 1: { if (N_ELEMENTS > CPOS + 1) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b111111111111000000000000) >> 12; } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6 + 1); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b1111111111110000) >> 4; } } case 2: { if (N_ELEMENTS > CPOS + 1) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6 + 2); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b000011111111111100000000) >> 8; } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6 + 3); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b0000111111111111); } } case 3: { if (N_ELEMENTS > CPOS) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6 + 2); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b11111111111100000000000000000000) >> 20; } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6 + 4); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b1111111111110000) >> 4; } } @@ -208,23 +279,39 @@ struct Uint16Reader { switch (SUB_ELEMENT) { case 0: { if (N_ELEMENTS > CPOS + 1) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 4); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0x0000FFFF); } else { const uint16_t* const __restrict codesFp16 = reinterpret_cast(codes); +#ifdef FAISS_BIG_ENDIAN + uint16_t rt = codesFp16[CPOS]; + rt = Swap2Bytes(rt); + return rt; +#endif return codesFp16[CPOS]; } } case 1: { if (N_ELEMENTS > CPOS) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 4); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return code32 >> 16; } else { const uint16_t* const __restrict codesFp16 = reinterpret_cast(codes); +#ifdef FAISS_BIG_ENDIAN + uint16_t rt = codesFp16[CPOS]; + rt = Swap2Bytes(rt); + return rt; +#endif return codesFp16[CPOS]; } } diff --git a/faiss/cppcontrib/sa_decode/Level2-inl.h b/faiss/cppcontrib/sa_decode/Level2-inl.h index 36355af..4f9bdc4 100644 --- a/faiss/cppcontrib/sa_decode/Level2-inl.h +++ b/faiss/cppcontrib/sa_decode/Level2-inl.h @@ -12,6 +12,16 @@ #include #include +#include + +#ifdef FAISS_BIG_ENDIAN +#define Swap2Bytes(val) ((((val) >> 8) & 0x00FF) | (((val) << 8) & 0xFF00)) +#endif + +#ifndef FAISS_BIG_ENDIAN +#define FAISS_BIG_ENDIAN 0 +#define Swap2Bytes(val) val +#endif namespace faiss { namespace cppcontrib { @@ -72,9 +82,14 @@ struct Index2LevelDecoder { const intptr_t coarseCentroidOffset = i % COARSE_SIZE; const intptr_t fineCentroidIdx = i / FINE_SIZE; const intptr_t fineCentroidOffset = i % FINE_SIZE; - - const intptr_t coarseCode = coarse[coarseCentroidIdx]; - const intptr_t fineCode = fine[fineCentroidIdx]; + intptr_t coarseCode, fineCode; + if (FAISS_BIG_ENDIAN && sizeof(coarse_storage_type) == 2) { + coarseCode = Swap2Bytes(coarse[coarseCentroidIdx]); + fineCode = Swap2Bytes(fine[fineCentroidIdx]); + } else { + coarseCode = coarse[coarseCentroidIdx]; + fineCode = fine[fineCentroidIdx]; + } const float* const __restrict coarsePtr = pqCoarseCentroids + (coarseCentroidIdx * COARSE_TABLE_BYTES + coarseCode) * @@ -112,9 +127,14 @@ struct Index2LevelDecoder { const intptr_t fineCentroidIdx = i / FINE_SIZE; const intptr_t fineCentroidOffset = i % FINE_SIZE; - const intptr_t coarseCode = coarse[coarseCentroidIdx]; - const intptr_t fineCode = fine[fineCentroidIdx]; - + intptr_t coarseCode, fineCode; + if (FAISS_BIG_ENDIAN && sizeof(coarse_storage_type) == 2) { + coarseCode = Swap2Bytes(coarse[coarseCentroidIdx]); + fineCode = Swap2Bytes(fine[fineCentroidIdx]); + } else { + coarseCode = coarse[coarseCentroidIdx]; + fineCode = fine[fineCentroidIdx]; + } const float* const __restrict coarsePtr = pqCoarseCentroids + (coarseCentroidIdx * COARSE_TABLE_BYTES + coarseCode) * COARSE_SIZE + @@ -162,11 +182,18 @@ struct Index2LevelDecoder { const intptr_t coarseCentroidOffset = i % COARSE_SIZE; const intptr_t fineCentroidIdx = i / FINE_SIZE; const intptr_t fineCentroidOffset = i % FINE_SIZE; - - const intptr_t coarseCode0 = coarse0[coarseCentroidIdx]; - const intptr_t fineCode0 = fine0[fineCentroidIdx]; - const intptr_t coarseCode1 = coarse1[coarseCentroidIdx]; - const intptr_t fineCode1 = fine1[fineCentroidIdx]; + intptr_t coarseCode0, coarseCode1, fineCode0, fineCode1; + if (FAISS_BIG_ENDIAN && sizeof(coarse_storage_type) == 2) { + coarseCode0 = Swap2Bytes(coarse0[coarseCentroidIdx]); + fineCode0 = Swap2Bytes(fine0[fineCentroidIdx]); + coarseCode1 = Swap2Bytes(coarse1[coarseCentroidIdx]); + fineCode1 = Swap2Bytes(fine1[fineCentroidIdx]); + } else { + coarseCode0 = coarse0[coarseCentroidIdx]; + fineCode0 = fine0[fineCentroidIdx]; + coarseCode1 = coarse1[coarseCentroidIdx]; + fineCode1 = fine1[fineCentroidIdx]; + } const float* const __restrict coarsePtr0 = pqCoarseCentroids0 + (coarseCentroidIdx * COARSE_TABLE_BYTES + coarseCode0) * @@ -222,11 +249,18 @@ struct Index2LevelDecoder { const intptr_t coarseCentroidOffset = i % COARSE_SIZE; const intptr_t fineCentroidIdx = i / FINE_SIZE; const intptr_t fineCentroidOffset = i % FINE_SIZE; - - const intptr_t coarseCode0 = coarse0[coarseCentroidIdx]; - const intptr_t fineCode0 = fine0[fineCentroidIdx]; - const intptr_t coarseCode1 = coarse1[coarseCentroidIdx]; - const intptr_t fineCode1 = fine1[fineCentroidIdx]; + intptr_t coarseCode0, coarseCode1, fineCode0, fineCode1; + if (FAISS_BIG_ENDIAN && sizeof(coarse_storage_type) == 2) { + coarseCode0 = Swap2Bytes(coarse0[coarseCentroidIdx]); + fineCode0 = Swap2Bytes(fine0[fineCentroidIdx]); + coarseCode1 = Swap2Bytes(coarse1[coarseCentroidIdx]); + fineCode1 = Swap2Bytes(fine1[fineCentroidIdx]); + } else { + coarseCode0 = coarse0[coarseCentroidIdx]; + fineCode0 = fine0[fineCentroidIdx]; + coarseCode1 = coarse1[coarseCentroidIdx]; + fineCode1 = fine1[fineCentroidIdx]; + } const float* const __restrict coarsePtr0 = pqCoarseCentroids + (coarseCentroidIdx * COARSE_TABLE_BYTES + coarseCode0) * @@ -292,13 +326,23 @@ struct Index2LevelDecoder { const intptr_t coarseCentroidOffset = i % COARSE_SIZE; const intptr_t fineCentroidIdx = i / FINE_SIZE; const intptr_t fineCentroidOffset = i % FINE_SIZE; - - const intptr_t coarseCode0 = coarse0[coarseCentroidIdx]; - const intptr_t fineCode0 = fine0[fineCentroidIdx]; - const intptr_t coarseCode1 = coarse1[coarseCentroidIdx]; - const intptr_t fineCode1 = fine1[fineCentroidIdx]; - const intptr_t coarseCode2 = coarse2[coarseCentroidIdx]; - const intptr_t fineCode2 = fine2[fineCentroidIdx]; + intptr_t coarseCode0, coarseCode1, fineCode0, fineCode1; + intptr_t coarseCode2, fineCode2; + if (FAISS_BIG_ENDIAN && sizeof(coarse_storage_type) == 2) { + coarseCode0 = Swap2Bytes(coarse0[coarseCentroidIdx]); + fineCode0 = Swap2Bytes(fine0[fineCentroidIdx]); + coarseCode1 = Swap2Bytes(coarse1[coarseCentroidIdx]); + fineCode1 = Swap2Bytes(fine1[fineCentroidIdx]); + coarseCode2 = Swap2Bytes(coarse2[coarseCentroidIdx]); + fineCode2 = Swap2Bytes(fine2[fineCentroidIdx]); + } else { + coarseCode0 = coarse0[coarseCentroidIdx]; + fineCode0 = fine0[fineCentroidIdx]; + coarseCode1 = coarse1[coarseCentroidIdx]; + fineCode1 = fine1[fineCentroidIdx]; + coarseCode2 = coarse2[coarseCentroidIdx]; + fineCode2 = fine2[fineCentroidIdx]; + } const float* const __restrict coarsePtr0 = pqCoarseCentroids0 + (coarseCentroidIdx * COARSE_TABLE_BYTES + coarseCode0) * @@ -369,13 +413,23 @@ struct Index2LevelDecoder { const intptr_t coarseCentroidOffset = i % COARSE_SIZE; const intptr_t fineCentroidIdx = i / FINE_SIZE; const intptr_t fineCentroidOffset = i % FINE_SIZE; - - const intptr_t coarseCode0 = coarse0[coarseCentroidIdx]; - const intptr_t fineCode0 = fine0[fineCentroidIdx]; - const intptr_t coarseCode1 = coarse1[coarseCentroidIdx]; - const intptr_t fineCode1 = fine1[fineCentroidIdx]; - const intptr_t coarseCode2 = coarse2[coarseCentroidIdx]; - const intptr_t fineCode2 = fine2[fineCentroidIdx]; + intptr_t coarseCode0, fineCode0, coarseCode1, fineCode1; + intptr_t coarseCode2, fineCode2; + if (FAISS_BIG_ENDIAN && sizeof(coarse_storage_type) == 2) { + coarseCode0 = Swap2Bytes(coarse0[coarseCentroidIdx]); + fineCode0 = Swap2Bytes(fine0[fineCentroidIdx]); + coarseCode1 = Swap2Bytes(coarse1[coarseCentroidIdx]); + fineCode1 = Swap2Bytes(fine1[fineCentroidIdx]); + coarseCode2 = Swap2Bytes(coarse2[coarseCentroidIdx]); + fineCode2 = Swap2Bytes(fine2[fineCentroidIdx]); + } else { + coarseCode0 = coarse0[coarseCentroidIdx]; + fineCode0 = fine0[fineCentroidIdx]; + coarseCode1 = coarse1[coarseCentroidIdx]; + fineCode1 = fine1[fineCentroidIdx]; + coarseCode2 = coarse2[coarseCentroidIdx]; + fineCode2 = fine2[fineCentroidIdx]; + } const float* const __restrict coarsePtr0 = pqCoarseCentroids + (coarseCentroidIdx * COARSE_TABLE_BYTES + coarseCode0) * diff --git a/faiss/impl/platform_macros.h b/faiss/impl/platform_macros.h index 2aecc51..57e2343 100644 --- a/faiss/impl/platform_macros.h +++ b/faiss/impl/platform_macros.h @@ -165,3 +165,11 @@ inline int __builtin_clzll(uint64_t x) { #endif // clang-format on + +/******************************************************* + * BIGENDIAN specific macros + *******************************************************/ +#if !defined(_MSC_VER) && \ + (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) +#define FAISS_BIG_ENDIAN 1 +#endif diff --git a/faiss/impl/pq4_fast_scan.cpp b/faiss/impl/pq4_fast_scan.cpp index 6173ece..127646e 100644 --- a/faiss/impl/pq4_fast_scan.cpp +++ b/faiss/impl/pq4_fast_scan.cpp @@ -6,6 +6,7 @@ */ #include +#include #include #include @@ -58,8 +59,13 @@ void pq4_pack_codes( return; } memset(blocks, 0, nb * nsq / 2); +#ifdef FAISS_BIG_ENDIAN + const uint8_t perm0[16] = { + 8, 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7}; +#else const uint8_t perm0[16] = { 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15}; +#endif uint8_t* codes2 = blocks; for (size_t i0 = 0; i0 < nb; i0 += bbs) { @@ -93,8 +99,13 @@ void pq4_pack_codes_range( size_t bbs, size_t nsq, uint8_t* blocks) { +#ifdef FAISS_BIG_ENDIAN + const uint8_t perm0[16] = { + 8, 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7}; +#else const uint8_t perm0[16] = { 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15}; +#endif // range of affected blocks size_t block0 = i0 / bbs; diff --git a/faiss/python/CMakeLists.txt b/faiss/python/CMakeLists.txt index 26eaa54..f1a24dc 100644 --- a/faiss/python/CMakeLists.txt +++ b/faiss/python/CMakeLists.txt @@ -73,11 +73,20 @@ else() find_package(faiss REQUIRED) endif() +if (${CMAKE_SYSTEM_NAME} MATCHES "AIX") +swig_add_library(swigfaiss + TYPE MODULE + LANGUAGE python + SOURCES swigfaiss.swig +) +else () swig_add_library(swigfaiss TYPE SHARED LANGUAGE python SOURCES swigfaiss.swig ) +endif() + set_property(TARGET swigfaiss PROPERTY SWIG_COMPILE_OPTIONS -doxygen) set_property(SOURCE swigfaiss_avx2.swig @@ -166,6 +175,10 @@ set_property(TARGET faiss_python_callbacks PROPERTY POSITION_INDEPENDENT_CODE ON ) +if (${CMAKE_SYSTEM_NAME} MATCHES "AIX") +target_link_libraries(faiss_python_callbacks PRIVATE faiss) +endif() + # Hack so that python_callbacks.h can be included as # `#include `. target_include_directories(faiss_python_callbacks PRIVATE ${PROJECT_SOURCE_DIR}/../..) -- cgit v1.2.3 From 79bfbb66d0c552bf90da493b2e705139ff9b8ea6 Mon Sep 17 00:00:00 2001 From: Debian Deep Learning Team Date: Sun, 6 Feb 2022 12:19:35 +0100 Subject: Use system GTest Gbp-Pq: Name 0001-Use-system-GTest.patch --- CMakeLists.txt | 1 + tests/CMakeLists.txt | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6cdc37c..7d0ad00 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,6 +56,7 @@ option(FAISS_ENABLE_GPU "Enable support for GPU indexes." ON) option(FAISS_ENABLE_RAFT "Enable RAFT for GPU indexes." OFF) option(FAISS_ENABLE_PYTHON "Build Python extension." ON) option(FAISS_ENABLE_C_API "Build C API." OFF) +option(FAISS_USE_SYSTEM_GTEST "Use system-provided gtest library." ON) if(FAISS_ENABLE_GPU) set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER}) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 10243b9..b716226 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -58,18 +58,24 @@ if(FAISS_OPT_LEVEL STREQUAL "avx512") target_link_libraries(faiss_test PRIVATE faiss_avx512) endif() +if(NOT FAISS_USE_SYSTEM_GTEST) include(FetchContent) FetchContent_Declare(googletest URL "https://github.com/google/googletest/archive/release-1.12.1.tar.gz") set(BUILD_GMOCK CACHE BOOL OFF) set(INSTALL_GTEST CACHE BOOL OFF) FetchContent_MakeAvailable(googletest) +else() + find_package(GTest REQUIRED) + include_directories(${GTEST_INCLUDE_DIRS}) +endif() + find_package(OpenMP REQUIRED) target_link_libraries(faiss_test PRIVATE OpenMP::OpenMP_CXX - gtest_main + GTest::gtest_main $<$:raft::raft> ) -- cgit v1.2.3 From e482528fc54ac96363dc441a3630d282f79bc623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Sun, 6 Feb 2022 12:23:54 +0100 Subject: Use SWIGWORDSIZE32 on 32 bit architectures Gbp-Pq: Name 0002-Use-SWIGWORDSIZE32-on-32-bit-architectures.patch --- faiss/python/CMakeLists.txt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/faiss/python/CMakeLists.txt b/faiss/python/CMakeLists.txt index 8bca710..26eaa54 100644 --- a/faiss/python/CMakeLists.txt +++ b/faiss/python/CMakeLists.txt @@ -25,9 +25,15 @@ macro(configure_swigfaiss source) USE_TARGET_INCLUDE_DIRECTORIES TRUE ) if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin" AND NOT WIN32) - set_source_files_properties(${source} PROPERTIES - SWIG_FLAGS -DSWIGWORDSIZE64 - ) + if(CMAKE_SIZEOF_VOID_P EQUAL 8) + set_source_files_properties(${source} PROPERTIES + SWIG_FLAGS -DSWIGWORDSIZE64 + ) + else() + set_source_files_properties(${source} PROPERTIES + SWIG_FLAGS -DSWIGWORDSIZE32 + ) + endif() endif() if(WIN32) set_source_files_properties(${source} PROPERTIES -- cgit v1.2.3 From e478589d7d34e741f20e8682bc8ddac76ccd0e05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Tue, 8 Feb 2022 11:53:12 +0100 Subject: Fix CMake package export Gbp-Pq: Name 0003-Fix-CMake-package-export.patch --- faiss/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt index a890a46..ee54600 100644 --- a/faiss/CMakeLists.txt +++ b/faiss/CMakeLists.txt @@ -358,9 +358,9 @@ configure_file(${PROJECT_SOURCE_DIR}/cmake/faiss-config.cmake.in ) install(FILES ${PROJECT_BINARY_DIR}/cmake/faiss-config.cmake ${PROJECT_BINARY_DIR}/cmake/faiss-config-version.cmake - DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/faiss + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/faiss ) install(EXPORT faiss-targets - DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/faiss + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/faiss ) -- cgit v1.2.3 From fb70e7c33472a690fc64c63d5feded4818a59fc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Wed, 9 Feb 2022 01:28:47 +0100 Subject: Fix implementation for 32 bit size_t Gbp-Pq: Name 0004-Fix-implementation-for-32-bit-size_t.patch --- faiss/IndexIVF.cpp | 2 +- faiss/IndexIVFIndependentQuantizer.cpp | 2 +- faiss/index_factory.cpp | 2 +- faiss/invlists/OnDiskInvertedLists.cpp | 4 ++-- faiss/utils/utils.cpp | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/faiss/IndexIVF.cpp b/faiss/IndexIVF.cpp index 95d3bc9..ffaae4a 100644 --- a/faiss/IndexIVF.cpp +++ b/faiss/IndexIVF.cpp @@ -1150,7 +1150,7 @@ void IndexIVF::train(idx_t n, const float* x) { // optional subsampling idx_t max_nt = train_encoder_num_vectors(); if (max_nt <= 0) { - max_nt = (size_t)1 << 35; + max_nt = (size_t)1 << 30; } TransformedVectors tv( diff --git a/faiss/IndexIVFIndependentQuantizer.cpp b/faiss/IndexIVFIndependentQuantizer.cpp index 2073dd2..d5be6e3 100644 --- a/faiss/IndexIVFIndependentQuantizer.cpp +++ b/faiss/IndexIVFIndependentQuantizer.cpp @@ -140,7 +140,7 @@ void IndexIVFIndependentQuantizer::train(idx_t n, const float* x) { // optional subsampling idx_t max_nt = index_ivf->train_encoder_num_vectors(); if (max_nt <= 0) { - max_nt = (size_t)1 << 35; + max_nt = (size_t)1 << 30; } SubsampledVectors sv(index_ivf->d, &n, max_nt, x); diff --git a/faiss/index_factory.cpp b/faiss/index_factory.cpp index 0d61b73..883753e 100644 --- a/faiss/index_factory.cpp +++ b/faiss/index_factory.cpp @@ -731,7 +731,7 @@ std::unique_ptr index_factory_sub( } if (verbose) { - printf("after () normalization: %s %ld parenthesis indexes d=%d\n", + printf("after () normalization: %s %zu parenthesis indexes d=%d\n", description.c_str(), parenthesis_indexes.size(), d); diff --git a/faiss/invlists/OnDiskInvertedLists.cpp b/faiss/invlists/OnDiskInvertedLists.cpp index 3017d16..b363747 100644 --- a/faiss/invlists/OnDiskInvertedLists.cpp +++ b/faiss/invlists/OnDiskInvertedLists.cpp @@ -524,7 +524,7 @@ void OnDiskInvertedLists::free_slot(size_t offset, size_t capacity) { it++; } - size_t inf = ((size_t)1) << 60; + size_t inf = size_t(1) << (8 * __SIZEOF_SIZE_T__ - 2); size_t end_prev = inf; if (it != slots.begin()) { @@ -533,7 +533,7 @@ void OnDiskInvertedLists::free_slot(size_t offset, size_t capacity) { end_prev = prev->offset + prev->capacity; } - size_t begin_next = ((size_t)1) << 60; + size_t begin_next = inf; if (it != slots.end()) { begin_next = it->offset; } diff --git a/faiss/utils/utils.cpp b/faiss/utils/utils.cpp index dc6fadd..147afb5 100644 --- a/faiss/utils/utils.cpp +++ b/faiss/utils/utils.cpp @@ -169,7 +169,7 @@ size_t get_mem_usage_kb() { char buf[256]; if (!fgets(buf, 256, f)) break; - if (sscanf(buf, "VmRSS: %ld kB", &sz) == 1) + if (sscanf(buf, "VmRSS: %zu kB", &sz) == 1) break; } fclose(f); -- cgit v1.2.3 From fd3d95373ba1dc0a8237e5c4e9a0efec8851ce15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Wed, 9 Feb 2022 01:38:34 +0100 Subject: Fix floating point comparisons in unit tests Gbp-Pq: Name 0005-Fix-floating-point-comparisons-in-unit-tests.patch --- tests/test_lowlevel_ivf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_lowlevel_ivf.cpp b/tests/test_lowlevel_ivf.cpp index e28e2a9..a4d8410 100644 --- a/tests/test_lowlevel_ivf.cpp +++ b/tests/test_lowlevel_ivf.cpp @@ -359,7 +359,7 @@ void test_lowlevel_access_binary(const char* index_key) { float computed_D = scanner->distance_to_code( xb.data() + vno * il->code_size); - EXPECT_EQ(computed_D, D[jj]); + EXPECT_FLOAT_EQ(computed_D, D[jj]); } } } -- cgit v1.2.3 From d585f791357b8814b4905a4d98be8fc783b82715 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Wed, 9 Feb 2022 08:08:13 +0100 Subject: Add GTest filter Gbp-Pq: Name 0006-Add-GTest-filter.patch --- tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b716226..c1510ec 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -81,4 +81,4 @@ target_link_libraries(faiss_test PRIVATE # Defines `gtest_discover_tests()`. include(GoogleTest) -gtest_discover_tests(faiss_test) +gtest_discover_tests(faiss_test TEST_FILTER "${FAISS_GTEST_FILTER}") -- cgit v1.2.3 From 95ca01b75e5e8275b9d5c5209777486a6cf64910 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Thu, 22 Jun 2023 17:51:36 +0200 Subject: Fix build with gcc-13 Gbp-Pq: Name 0007-Fix-build-with-gcc-13.patch --- faiss/Index.h | 1 + 1 file changed, 1 insertion(+) diff --git a/faiss/Index.h b/faiss/Index.h index 3d1bdb9..9fc9e19 100644 --- a/faiss/Index.h +++ b/faiss/Index.h @@ -11,6 +11,7 @@ #define FAISS_INDEX_H #include +#include #include #include #include -- cgit v1.2.3 From f0e4f9a5dc5f56f8b63301fe3a528c5164a2c136 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Sat, 24 Jun 2023 21:29:46 +0200 Subject: Install missing header file Gbp-Pq: Name 0008-Install-missing-header-file.patch --- faiss/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt index ee54600..aec195b 100644 --- a/faiss/CMakeLists.txt +++ b/faiss/CMakeLists.txt @@ -149,6 +149,7 @@ set(FAISS_HEADERS impl/AuxIndexStructures.h impl/CodePacker.h impl/IDSelector.h + impl/CodePacker.h impl/DistanceComputer.h impl/FaissAssert.h impl/FaissException.h -- cgit v1.2.3 From 3b352a1a2698a37835ef052d41e89f7590f7f9cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Wed, 17 Apr 2024 17:20:52 +0200 Subject: Fix FTBFS with swig >= 4.2.0 Origin: upstream, https://github.com/facebookresearch/faiss/pull/3315 Bug: https://github.com/facebookresearch/faiss/issues/3239 Bug-Debian: https://bugs.debian.org/1061741 Gbp-Pq: Name 0009-Fix-FTBFS-with-swig-4.2.0.patch --- faiss/python/swigfaiss.swig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/faiss/python/swigfaiss.swig b/faiss/python/swigfaiss.swig index fb7f50d..c536676 100644 --- a/faiss/python/swigfaiss.swig +++ b/faiss/python/swigfaiss.swig @@ -1022,14 +1022,14 @@ PyObject *swig_ptr (PyObject *a) return SWIG_NewPointerObj(data, SWIGTYPE_p_bool, 0); } if(PyArray_TYPE(ao) == NPY_UINT64) { -#ifdef SWIGWORDSIZE64 +#if (__SIZEOF_LONG__ == 8) return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long_long, 0); #endif } if(PyArray_TYPE(ao) == NPY_INT64) { -#ifdef SWIGWORDSIZE64 +#if (__SIZEOF_LONG__ == 8) return SWIG_NewPointerObj(data, SWIGTYPE_p_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_long_long, 0); -- cgit v1.2.3 From 2b88f15b826ee461dee060b8984bdd2173eaf942 Mon Sep 17 00:00:00 2001 From: Matthijs Douze Date: Mon, 22 Apr 2024 00:31:45 +0200 Subject: Big endian support Forwarded: https://github.com/facebookresearch/faiss/pull/3361 Gbp-Pq: Name 0010-Big-endian-support.patch --- contrib/vecs_io.py | 12 +++- faiss/cppcontrib/detail/UintReader.h | 121 +++++++++++++++++++++++++++----- faiss/cppcontrib/sa_decode/Level2-inl.h | 114 ++++++++++++++++++++++-------- faiss/impl/platform_macros.h | 8 +++ faiss/impl/pq4_fast_scan.cpp | 11 +++ faiss/python/CMakeLists.txt | 13 ++++ 6 files changed, 231 insertions(+), 48 deletions(-) diff --git a/contrib/vecs_io.py b/contrib/vecs_io.py index ea75d5f..5d18c0b 100644 --- a/contrib/vecs_io.py +++ b/contrib/vecs_io.py @@ -3,6 +3,7 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. +import sys import numpy as np """ @@ -13,6 +14,8 @@ definition of the formats here: http://corpus-texmex.irisa.fr/ def ivecs_read(fname): a = np.fromfile(fname, dtype='int32') + if sys.big_endian: + a.byteswap(inplace=True) d = a[0] return a.reshape(-1, d + 1)[:, 1:].copy() @@ -22,6 +25,7 @@ def fvecs_read(fname): def ivecs_mmap(fname): + assert not sys.big_endian a = np.memmap(fname, dtype='int32', mode='r') d = a[0] return a.reshape(-1, d + 1)[:, 1:] @@ -33,7 +37,11 @@ def fvecs_mmap(fname): def bvecs_mmap(fname): x = np.memmap(fname, dtype='uint8', mode='r') - d = x[:4].view('int32')[0] + if sys.big_endian: + da = x[:4][::-1].copy() + d = da.view('int32')[0] + else: + d = x[:4].view('int32')[0] return x.reshape(-1, d + 4)[:, 4:] @@ -42,6 +50,8 @@ def ivecs_write(fname, m): m1 = np.empty((n, d + 1), dtype='int32') m1[:, 0] = d m1[:, 1:] = m + if sys.big_endian: + m1.byteswap(inplace=True) m1.tofile(fname) diff --git a/faiss/cppcontrib/detail/UintReader.h b/faiss/cppcontrib/detail/UintReader.h index 81e600f..d0b5836 100644 --- a/faiss/cppcontrib/detail/UintReader.h +++ b/faiss/cppcontrib/detail/UintReader.h @@ -7,8 +7,18 @@ #pragma once +#include #include +#ifdef FAISS_BIG_ENDIAN +#define Swap2Bytes(val) ((((val) >> 8) & 0x00FF) | (((val) << 8) & 0xFF00)) + +#define Swap4Bytes(val) \ + ((((val) >> 24) & 0x000000FF) | (((val) >> 8) & 0x0000FF00) | \ + (((val) << 8) & 0x00FF0000) | (((val) << 24) & 0xFF000000)) + +#endif + namespace faiss { namespace cppcontrib { namespace detail { @@ -31,7 +41,11 @@ struct Uint8Reader { if (N_ELEMENTS > CPOS + 3) { const uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 4); +#ifdef FAISS_BIG_ENDIAN + return (code32) >> 24; +#else return (code32 & 0x000000FF); +#endif } else { return codes[CPOS]; } @@ -40,7 +54,11 @@ struct Uint8Reader { if (N_ELEMENTS > CPOS + 2) { const uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 4); +#ifdef FAISS_BIG_ENDIAN + return (code32 & 0x00FF0000) >> 16; +#else return (code32 & 0x0000FF00) >> 8; +#endif } else { return codes[CPOS]; } @@ -49,7 +67,11 @@ struct Uint8Reader { if (N_ELEMENTS > CPOS + 1) { const uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 4); +#ifdef FAISS_BIG_ENDIAN + return (code32 & 0x0000FF00) >> 8; +#else return (code32 & 0x00FF0000) >> 16; +#endif } else { return codes[CPOS]; } @@ -58,7 +80,11 @@ struct Uint8Reader { if (N_ELEMENTS > CPOS) { const uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 4); +#ifdef FAISS_BIG_ENDIAN + return (code32 & 0x000000FF); +#else return (code32) >> 24; +#endif } else { return codes[CPOS]; } @@ -87,40 +113,61 @@ struct Uint10Reader { switch (SUB_ELEMENT) { case 0: { if (N_ELEMENTS > CPOS + 2) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b0000001111111111); } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5 + 0); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b0000001111111111); } } case 1: { if (N_ELEMENTS > CPOS + 1) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b000011111111110000000000) >> 10; } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5 + 1); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b0000111111111100) >> 2; } } case 2: { if (N_ELEMENTS > CPOS) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b00111111111100000000000000000000) >> 20; } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5 + 2); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b0011111111110000) >> 4; } } case 3: { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5 + 3); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b1111111111000000) >> 6; } } @@ -147,45 +194,69 @@ struct Uint12Reader { switch (SUB_ELEMENT) { case 0: { if (N_ELEMENTS > CPOS + 2) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b0000111111111111); } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6 + 0); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b0000111111111111); } } case 1: { if (N_ELEMENTS > CPOS + 1) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b111111111111000000000000) >> 12; } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6 + 1); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b1111111111110000) >> 4; } } case 2: { if (N_ELEMENTS > CPOS + 1) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6 + 2); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b000011111111111100000000) >> 8; } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6 + 3); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b0000111111111111); } } case 3: { if (N_ELEMENTS > CPOS) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6 + 2); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b11111111111100000000000000000000) >> 20; } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6 + 4); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b1111111111110000) >> 4; } } @@ -208,23 +279,39 @@ struct Uint16Reader { switch (SUB_ELEMENT) { case 0: { if (N_ELEMENTS > CPOS + 1) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 4); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0x0000FFFF); } else { const uint16_t* const __restrict codesFp16 = reinterpret_cast(codes); +#ifdef FAISS_BIG_ENDIAN + uint16_t rt = codesFp16[CPOS]; + rt = Swap2Bytes(rt); + return rt; +#endif return codesFp16[CPOS]; } } case 1: { if (N_ELEMENTS > CPOS) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 4); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return code32 >> 16; } else { const uint16_t* const __restrict codesFp16 = reinterpret_cast(codes); +#ifdef FAISS_BIG_ENDIAN + uint16_t rt = codesFp16[CPOS]; + rt = Swap2Bytes(rt); + return rt; +#endif return codesFp16[CPOS]; } } diff --git a/faiss/cppcontrib/sa_decode/Level2-inl.h b/faiss/cppcontrib/sa_decode/Level2-inl.h index 36355af..4f9bdc4 100644 --- a/faiss/cppcontrib/sa_decode/Level2-inl.h +++ b/faiss/cppcontrib/sa_decode/Level2-inl.h @@ -12,6 +12,16 @@ #include #include +#include + +#ifdef FAISS_BIG_ENDIAN +#define Swap2Bytes(val) ((((val) >> 8) & 0x00FF) | (((val) << 8) & 0xFF00)) +#endif + +#ifndef FAISS_BIG_ENDIAN +#define FAISS_BIG_ENDIAN 0 +#define Swap2Bytes(val) val +#endif namespace faiss { namespace cppcontrib { @@ -72,9 +82,14 @@ struct Index2LevelDecoder { const intptr_t coarseCentroidOffset = i % COARSE_SIZE; const intptr_t fineCentroidIdx = i / FINE_SIZE; const intptr_t fineCentroidOffset = i % FINE_SIZE; - - const intptr_t coarseCode = coarse[coarseCentroidIdx]; - const intptr_t fineCode = fine[fineCentroidIdx]; + intptr_t coarseCode, fineCode; + if (FAISS_BIG_ENDIAN && sizeof(coarse_storage_type) == 2) { + coarseCode = Swap2Bytes(coarse[coarseCentroidIdx]); + fineCode = Swap2Bytes(fine[fineCentroidIdx]); + } else { + coarseCode = coarse[coarseCentroidIdx]; + fineCode = fine[fineCentroidIdx]; + } const float* const __restrict coarsePtr = pqCoarseCentroids + (coarseCentroidIdx * COARSE_TABLE_BYTES + coarseCode) * @@ -112,9 +127,14 @@ struct Index2LevelDecoder { const intptr_t fineCentroidIdx = i / FINE_SIZE; const intptr_t fineCentroidOffset = i % FINE_SIZE; - const intptr_t coarseCode = coarse[coarseCentroidIdx]; - const intptr_t fineCode = fine[fineCentroidIdx]; - + intptr_t coarseCode, fineCode; + if (FAISS_BIG_ENDIAN && sizeof(coarse_storage_type) == 2) { + coarseCode = Swap2Bytes(coarse[coarseCentroidIdx]); + fineCode = Swap2Bytes(fine[fineCentroidIdx]); + } else { + coarseCode = coarse[coarseCentroidIdx]; + fineCode = fine[fineCentroidIdx]; + } const float* const __restrict coarsePtr = pqCoarseCentroids + (coarseCentroidIdx * COARSE_TABLE_BYTES + coarseCode) * COARSE_SIZE + @@ -162,11 +182,18 @@ struct Index2LevelDecoder { const intptr_t coarseCentroidOffset = i % COARSE_SIZE; const intptr_t fineCentroidIdx = i / FINE_SIZE; const intptr_t fineCentroidOffset = i % FINE_SIZE; - - const intptr_t coarseCode0 = coarse0[coarseCentroidIdx]; - const intptr_t fineCode0 = fine0[fineCentroidIdx]; - const intptr_t coarseCode1 = coarse1[coarseCentroidIdx]; - const intptr_t fineCode1 = fine1[fineCentroidIdx]; + intptr_t coarseCode0, coarseCode1, fineCode0, fineCode1; + if (FAISS_BIG_ENDIAN && sizeof(coarse_storage_type) == 2) { + coarseCode0 = Swap2Bytes(coarse0[coarseCentroidIdx]); + fineCode0 = Swap2Bytes(fine0[fineCentroidIdx]); + coarseCode1 = Swap2Bytes(coarse1[coarseCentroidIdx]); + fineCode1 = Swap2Bytes(fine1[fineCentroidIdx]); + } else { + coarseCode0 = coarse0[coarseCentroidIdx]; + fineCode0 = fine0[fineCentroidIdx]; + coarseCode1 = coarse1[coarseCentroidIdx]; + fineCode1 = fine1[fineCentroidIdx]; + } const float* const __restrict coarsePtr0 = pqCoarseCentroids0 + (coarseCentroidIdx * COARSE_TABLE_BYTES + coarseCode0) * @@ -222,11 +249,18 @@ struct Index2LevelDecoder { const intptr_t coarseCentroidOffset = i % COARSE_SIZE; const intptr_t fineCentroidIdx = i / FINE_SIZE; const intptr_t fineCentroidOffset = i % FINE_SIZE; - - const intptr_t coarseCode0 = coarse0[coarseCentroidIdx]; - const intptr_t fineCode0 = fine0[fineCentroidIdx]; - const intptr_t coarseCode1 = coarse1[coarseCentroidIdx]; - const intptr_t fineCode1 = fine1[fineCentroidIdx]; + intptr_t coarseCode0, coarseCode1, fineCode0, fineCode1; + if (FAISS_BIG_ENDIAN && sizeof(coarse_storage_type) == 2) { + coarseCode0 = Swap2Bytes(coarse0[coarseCentroidIdx]); + fineCode0 = Swap2Bytes(fine0[fineCentroidIdx]); + coarseCode1 = Swap2Bytes(coarse1[coarseCentroidIdx]); + fineCode1 = Swap2Bytes(fine1[fineCentroidIdx]); + } else { + coarseCode0 = coarse0[coarseCentroidIdx]; + fineCode0 = fine0[fineCentroidIdx]; + coarseCode1 = coarse1[coarseCentroidIdx]; + fineCode1 = fine1[fineCentroidIdx]; + } const float* const __restrict coarsePtr0 = pqCoarseCentroids + (coarseCentroidIdx * COARSE_TABLE_BYTES + coarseCode0) * @@ -292,13 +326,23 @@ struct Index2LevelDecoder { const intptr_t coarseCentroidOffset = i % COARSE_SIZE; const intptr_t fineCentroidIdx = i / FINE_SIZE; const intptr_t fineCentroidOffset = i % FINE_SIZE; - - const intptr_t coarseCode0 = coarse0[coarseCentroidIdx]; - const intptr_t fineCode0 = fine0[fineCentroidIdx]; - const intptr_t coarseCode1 = coarse1[coarseCentroidIdx]; - const intptr_t fineCode1 = fine1[fineCentroidIdx]; - const intptr_t coarseCode2 = coarse2[coarseCentroidIdx]; - const intptr_t fineCode2 = fine2[fineCentroidIdx]; + intptr_t coarseCode0, coarseCode1, fineCode0, fineCode1; + intptr_t coarseCode2, fineCode2; + if (FAISS_BIG_ENDIAN && sizeof(coarse_storage_type) == 2) { + coarseCode0 = Swap2Bytes(coarse0[coarseCentroidIdx]); + fineCode0 = Swap2Bytes(fine0[fineCentroidIdx]); + coarseCode1 = Swap2Bytes(coarse1[coarseCentroidIdx]); + fineCode1 = Swap2Bytes(fine1[fineCentroidIdx]); + coarseCode2 = Swap2Bytes(coarse2[coarseCentroidIdx]); + fineCode2 = Swap2Bytes(fine2[fineCentroidIdx]); + } else { + coarseCode0 = coarse0[coarseCentroidIdx]; + fineCode0 = fine0[fineCentroidIdx]; + coarseCode1 = coarse1[coarseCentroidIdx]; + fineCode1 = fine1[fineCentroidIdx]; + coarseCode2 = coarse2[coarseCentroidIdx]; + fineCode2 = fine2[fineCentroidIdx]; + } const float* const __restrict coarsePtr0 = pqCoarseCentroids0 + (coarseCentroidIdx * COARSE_TABLE_BYTES + coarseCode0) * @@ -369,13 +413,23 @@ struct Index2LevelDecoder { const intptr_t coarseCentroidOffset = i % COARSE_SIZE; const intptr_t fineCentroidIdx = i / FINE_SIZE; const intptr_t fineCentroidOffset = i % FINE_SIZE; - - const intptr_t coarseCode0 = coarse0[coarseCentroidIdx]; - const intptr_t fineCode0 = fine0[fineCentroidIdx]; - const intptr_t coarseCode1 = coarse1[coarseCentroidIdx]; - const intptr_t fineCode1 = fine1[fineCentroidIdx]; - const intptr_t coarseCode2 = coarse2[coarseCentroidIdx]; - const intptr_t fineCode2 = fine2[fineCentroidIdx]; + intptr_t coarseCode0, fineCode0, coarseCode1, fineCode1; + intptr_t coarseCode2, fineCode2; + if (FAISS_BIG_ENDIAN && sizeof(coarse_storage_type) == 2) { + coarseCode0 = Swap2Bytes(coarse0[coarseCentroidIdx]); + fineCode0 = Swap2Bytes(fine0[fineCentroidIdx]); + coarseCode1 = Swap2Bytes(coarse1[coarseCentroidIdx]); + fineCode1 = Swap2Bytes(fine1[fineCentroidIdx]); + coarseCode2 = Swap2Bytes(coarse2[coarseCentroidIdx]); + fineCode2 = Swap2Bytes(fine2[fineCentroidIdx]); + } else { + coarseCode0 = coarse0[coarseCentroidIdx]; + fineCode0 = fine0[fineCentroidIdx]; + coarseCode1 = coarse1[coarseCentroidIdx]; + fineCode1 = fine1[fineCentroidIdx]; + coarseCode2 = coarse2[coarseCentroidIdx]; + fineCode2 = fine2[fineCentroidIdx]; + } const float* const __restrict coarsePtr0 = pqCoarseCentroids + (coarseCentroidIdx * COARSE_TABLE_BYTES + coarseCode0) * diff --git a/faiss/impl/platform_macros.h b/faiss/impl/platform_macros.h index 2aecc51..57e2343 100644 --- a/faiss/impl/platform_macros.h +++ b/faiss/impl/platform_macros.h @@ -165,3 +165,11 @@ inline int __builtin_clzll(uint64_t x) { #endif // clang-format on + +/******************************************************* + * BIGENDIAN specific macros + *******************************************************/ +#if !defined(_MSC_VER) && \ + (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) +#define FAISS_BIG_ENDIAN 1 +#endif diff --git a/faiss/impl/pq4_fast_scan.cpp b/faiss/impl/pq4_fast_scan.cpp index 6173ece..127646e 100644 --- a/faiss/impl/pq4_fast_scan.cpp +++ b/faiss/impl/pq4_fast_scan.cpp @@ -6,6 +6,7 @@ */ #include +#include #include #include @@ -58,8 +59,13 @@ void pq4_pack_codes( return; } memset(blocks, 0, nb * nsq / 2); +#ifdef FAISS_BIG_ENDIAN + const uint8_t perm0[16] = { + 8, 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7}; +#else const uint8_t perm0[16] = { 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15}; +#endif uint8_t* codes2 = blocks; for (size_t i0 = 0; i0 < nb; i0 += bbs) { @@ -93,8 +99,13 @@ void pq4_pack_codes_range( size_t bbs, size_t nsq, uint8_t* blocks) { +#ifdef FAISS_BIG_ENDIAN + const uint8_t perm0[16] = { + 8, 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7}; +#else const uint8_t perm0[16] = { 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15}; +#endif // range of affected blocks size_t block0 = i0 / bbs; diff --git a/faiss/python/CMakeLists.txt b/faiss/python/CMakeLists.txt index 26eaa54..f1a24dc 100644 --- a/faiss/python/CMakeLists.txt +++ b/faiss/python/CMakeLists.txt @@ -73,11 +73,20 @@ else() find_package(faiss REQUIRED) endif() +if (${CMAKE_SYSTEM_NAME} MATCHES "AIX") +swig_add_library(swigfaiss + TYPE MODULE + LANGUAGE python + SOURCES swigfaiss.swig +) +else () swig_add_library(swigfaiss TYPE SHARED LANGUAGE python SOURCES swigfaiss.swig ) +endif() + set_property(TARGET swigfaiss PROPERTY SWIG_COMPILE_OPTIONS -doxygen) set_property(SOURCE swigfaiss_avx2.swig @@ -166,6 +175,10 @@ set_property(TARGET faiss_python_callbacks PROPERTY POSITION_INDEPENDENT_CODE ON ) +if (${CMAKE_SYSTEM_NAME} MATCHES "AIX") +target_link_libraries(faiss_python_callbacks PRIVATE faiss) +endif() + # Hack so that python_callbacks.h can be included as # `#include `. target_include_directories(faiss_python_callbacks PRIVATE ${PROJECT_SOURCE_DIR}/../..) -- cgit v1.2.3 From 4e6085a8b3e14e0aff1ba784b6862152ab531a18 Mon Sep 17 00:00:00 2001 From: Debian Deep Learning Team Date: Sun, 6 Feb 2022 12:19:35 +0100 Subject: Use system GTest Gbp-Pq: Name 0001-Use-system-GTest.patch --- CMakeLists.txt | 1 + tests/CMakeLists.txt | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6cdc37c..7d0ad00 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,6 +56,7 @@ option(FAISS_ENABLE_GPU "Enable support for GPU indexes." ON) option(FAISS_ENABLE_RAFT "Enable RAFT for GPU indexes." OFF) option(FAISS_ENABLE_PYTHON "Build Python extension." ON) option(FAISS_ENABLE_C_API "Build C API." OFF) +option(FAISS_USE_SYSTEM_GTEST "Use system-provided gtest library." ON) if(FAISS_ENABLE_GPU) set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER}) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 10243b9..b716226 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -58,18 +58,24 @@ if(FAISS_OPT_LEVEL STREQUAL "avx512") target_link_libraries(faiss_test PRIVATE faiss_avx512) endif() +if(NOT FAISS_USE_SYSTEM_GTEST) include(FetchContent) FetchContent_Declare(googletest URL "https://github.com/google/googletest/archive/release-1.12.1.tar.gz") set(BUILD_GMOCK CACHE BOOL OFF) set(INSTALL_GTEST CACHE BOOL OFF) FetchContent_MakeAvailable(googletest) +else() + find_package(GTest REQUIRED) + include_directories(${GTEST_INCLUDE_DIRS}) +endif() + find_package(OpenMP REQUIRED) target_link_libraries(faiss_test PRIVATE OpenMP::OpenMP_CXX - gtest_main + GTest::gtest_main $<$:raft::raft> ) -- cgit v1.2.3 From 62337f8247f9598322b2b27a902e545b1ad5e172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Sun, 6 Feb 2022 12:23:54 +0100 Subject: Use SWIGWORDSIZE32 on 32 bit architectures Gbp-Pq: Name 0002-Use-SWIGWORDSIZE32-on-32-bit-architectures.patch --- faiss/python/CMakeLists.txt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/faiss/python/CMakeLists.txt b/faiss/python/CMakeLists.txt index 8bca710..26eaa54 100644 --- a/faiss/python/CMakeLists.txt +++ b/faiss/python/CMakeLists.txt @@ -25,9 +25,15 @@ macro(configure_swigfaiss source) USE_TARGET_INCLUDE_DIRECTORIES TRUE ) if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin" AND NOT WIN32) - set_source_files_properties(${source} PROPERTIES - SWIG_FLAGS -DSWIGWORDSIZE64 - ) + if(CMAKE_SIZEOF_VOID_P EQUAL 8) + set_source_files_properties(${source} PROPERTIES + SWIG_FLAGS -DSWIGWORDSIZE64 + ) + else() + set_source_files_properties(${source} PROPERTIES + SWIG_FLAGS -DSWIGWORDSIZE32 + ) + endif() endif() if(WIN32) set_source_files_properties(${source} PROPERTIES -- cgit v1.2.3 From f0d89cfa2426d83bcd001d7ae6e3320a4e742f3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Tue, 8 Feb 2022 11:53:12 +0100 Subject: Fix CMake package export Gbp-Pq: Name 0003-Fix-CMake-package-export.patch --- faiss/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt index a890a46..ee54600 100644 --- a/faiss/CMakeLists.txt +++ b/faiss/CMakeLists.txt @@ -358,9 +358,9 @@ configure_file(${PROJECT_SOURCE_DIR}/cmake/faiss-config.cmake.in ) install(FILES ${PROJECT_BINARY_DIR}/cmake/faiss-config.cmake ${PROJECT_BINARY_DIR}/cmake/faiss-config-version.cmake - DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/faiss + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/faiss ) install(EXPORT faiss-targets - DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/faiss + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/faiss ) -- cgit v1.2.3 From 1d07fe41bee9f08c6287a3bb19b7cf881e3bf4fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Wed, 9 Feb 2022 01:28:47 +0100 Subject: Fix implementation for 32 bit size_t Gbp-Pq: Name 0004-Fix-implementation-for-32-bit-size_t.patch --- faiss/IndexIVF.cpp | 2 +- faiss/IndexIVFIndependentQuantizer.cpp | 2 +- faiss/index_factory.cpp | 2 +- faiss/invlists/OnDiskInvertedLists.cpp | 4 ++-- faiss/utils/utils.cpp | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/faiss/IndexIVF.cpp b/faiss/IndexIVF.cpp index 95d3bc9..ffaae4a 100644 --- a/faiss/IndexIVF.cpp +++ b/faiss/IndexIVF.cpp @@ -1150,7 +1150,7 @@ void IndexIVF::train(idx_t n, const float* x) { // optional subsampling idx_t max_nt = train_encoder_num_vectors(); if (max_nt <= 0) { - max_nt = (size_t)1 << 35; + max_nt = (size_t)1 << 30; } TransformedVectors tv( diff --git a/faiss/IndexIVFIndependentQuantizer.cpp b/faiss/IndexIVFIndependentQuantizer.cpp index 2073dd2..d5be6e3 100644 --- a/faiss/IndexIVFIndependentQuantizer.cpp +++ b/faiss/IndexIVFIndependentQuantizer.cpp @@ -140,7 +140,7 @@ void IndexIVFIndependentQuantizer::train(idx_t n, const float* x) { // optional subsampling idx_t max_nt = index_ivf->train_encoder_num_vectors(); if (max_nt <= 0) { - max_nt = (size_t)1 << 35; + max_nt = (size_t)1 << 30; } SubsampledVectors sv(index_ivf->d, &n, max_nt, x); diff --git a/faiss/index_factory.cpp b/faiss/index_factory.cpp index 0d61b73..883753e 100644 --- a/faiss/index_factory.cpp +++ b/faiss/index_factory.cpp @@ -731,7 +731,7 @@ std::unique_ptr index_factory_sub( } if (verbose) { - printf("after () normalization: %s %ld parenthesis indexes d=%d\n", + printf("after () normalization: %s %zu parenthesis indexes d=%d\n", description.c_str(), parenthesis_indexes.size(), d); diff --git a/faiss/invlists/OnDiskInvertedLists.cpp b/faiss/invlists/OnDiskInvertedLists.cpp index 3017d16..b363747 100644 --- a/faiss/invlists/OnDiskInvertedLists.cpp +++ b/faiss/invlists/OnDiskInvertedLists.cpp @@ -524,7 +524,7 @@ void OnDiskInvertedLists::free_slot(size_t offset, size_t capacity) { it++; } - size_t inf = ((size_t)1) << 60; + size_t inf = size_t(1) << (8 * __SIZEOF_SIZE_T__ - 2); size_t end_prev = inf; if (it != slots.begin()) { @@ -533,7 +533,7 @@ void OnDiskInvertedLists::free_slot(size_t offset, size_t capacity) { end_prev = prev->offset + prev->capacity; } - size_t begin_next = ((size_t)1) << 60; + size_t begin_next = inf; if (it != slots.end()) { begin_next = it->offset; } diff --git a/faiss/utils/utils.cpp b/faiss/utils/utils.cpp index dc6fadd..147afb5 100644 --- a/faiss/utils/utils.cpp +++ b/faiss/utils/utils.cpp @@ -169,7 +169,7 @@ size_t get_mem_usage_kb() { char buf[256]; if (!fgets(buf, 256, f)) break; - if (sscanf(buf, "VmRSS: %ld kB", &sz) == 1) + if (sscanf(buf, "VmRSS: %zu kB", &sz) == 1) break; } fclose(f); -- cgit v1.2.3 From 8fb85f7ee2e10750ff2a2ac3ac13b35af72cf29b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Wed, 9 Feb 2022 01:38:34 +0100 Subject: Fix floating point comparisons in unit tests Gbp-Pq: Name 0005-Fix-floating-point-comparisons-in-unit-tests.patch --- tests/test_lowlevel_ivf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_lowlevel_ivf.cpp b/tests/test_lowlevel_ivf.cpp index e28e2a9..a4d8410 100644 --- a/tests/test_lowlevel_ivf.cpp +++ b/tests/test_lowlevel_ivf.cpp @@ -359,7 +359,7 @@ void test_lowlevel_access_binary(const char* index_key) { float computed_D = scanner->distance_to_code( xb.data() + vno * il->code_size); - EXPECT_EQ(computed_D, D[jj]); + EXPECT_FLOAT_EQ(computed_D, D[jj]); } } } -- cgit v1.2.3 From 45d8f5bac6868163ba6b270013f850d91fefe2a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Wed, 9 Feb 2022 08:08:13 +0100 Subject: Add GTest filter Gbp-Pq: Name 0006-Add-GTest-filter.patch --- tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b716226..c1510ec 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -81,4 +81,4 @@ target_link_libraries(faiss_test PRIVATE # Defines `gtest_discover_tests()`. include(GoogleTest) -gtest_discover_tests(faiss_test) +gtest_discover_tests(faiss_test TEST_FILTER "${FAISS_GTEST_FILTER}") -- cgit v1.2.3 From 5f295cb40b08c1b3916be403de9dfe423bcc13b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Thu, 22 Jun 2023 17:51:36 +0200 Subject: Fix build with gcc-13 Gbp-Pq: Name 0007-Fix-build-with-gcc-13.patch --- faiss/Index.h | 1 + 1 file changed, 1 insertion(+) diff --git a/faiss/Index.h b/faiss/Index.h index 3d1bdb9..9fc9e19 100644 --- a/faiss/Index.h +++ b/faiss/Index.h @@ -11,6 +11,7 @@ #define FAISS_INDEX_H #include +#include #include #include #include -- cgit v1.2.3 From 12fbaff48ee48f5b317cc2ab911882c8d724cbf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Sat, 24 Jun 2023 21:29:46 +0200 Subject: Install missing header file Gbp-Pq: Name 0008-Install-missing-header-file.patch --- faiss/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt index ee54600..aec195b 100644 --- a/faiss/CMakeLists.txt +++ b/faiss/CMakeLists.txt @@ -149,6 +149,7 @@ set(FAISS_HEADERS impl/AuxIndexStructures.h impl/CodePacker.h impl/IDSelector.h + impl/CodePacker.h impl/DistanceComputer.h impl/FaissAssert.h impl/FaissException.h -- cgit v1.2.3 From a871400be43c97b3a91fbdf849752a9cb39ed6d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Wed, 17 Apr 2024 17:20:52 +0200 Subject: Fix FTBFS with swig >= 4.2.0 Origin: upstream, https://github.com/facebookresearch/faiss/pull/3315 Bug: https://github.com/facebookresearch/faiss/issues/3239 Bug-Debian: https://bugs.debian.org/1061741 Gbp-Pq: Name 0009-Fix-FTBFS-with-swig-4.2.0.patch --- faiss/python/swigfaiss.swig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/faiss/python/swigfaiss.swig b/faiss/python/swigfaiss.swig index fb7f50d..c536676 100644 --- a/faiss/python/swigfaiss.swig +++ b/faiss/python/swigfaiss.swig @@ -1022,14 +1022,14 @@ PyObject *swig_ptr (PyObject *a) return SWIG_NewPointerObj(data, SWIGTYPE_p_bool, 0); } if(PyArray_TYPE(ao) == NPY_UINT64) { -#ifdef SWIGWORDSIZE64 +#if (__SIZEOF_LONG__ == 8) return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long_long, 0); #endif } if(PyArray_TYPE(ao) == NPY_INT64) { -#ifdef SWIGWORDSIZE64 +#if (__SIZEOF_LONG__ == 8) return SWIG_NewPointerObj(data, SWIGTYPE_p_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_long_long, 0); -- cgit v1.2.3 From 81f0d37d1840e618ca9477364336e4695018225a Mon Sep 17 00:00:00 2001 From: Matthijs Douze Date: Mon, 22 Apr 2024 00:31:45 +0200 Subject: Big endian support Forwarded: https://github.com/facebookresearch/faiss/pull/3361 Gbp-Pq: Name 0010-Big-endian-support.patch --- contrib/vecs_io.py | 12 +++- faiss/cppcontrib/detail/UintReader.h | 121 +++++++++++++++++++++++++++----- faiss/cppcontrib/sa_decode/Level2-inl.h | 114 ++++++++++++++++++++++-------- faiss/impl/platform_macros.h | 8 +++ faiss/impl/pq4_fast_scan.cpp | 11 +++ faiss/python/CMakeLists.txt | 13 ++++ 6 files changed, 231 insertions(+), 48 deletions(-) diff --git a/contrib/vecs_io.py b/contrib/vecs_io.py index ea75d5f..5d18c0b 100644 --- a/contrib/vecs_io.py +++ b/contrib/vecs_io.py @@ -3,6 +3,7 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. +import sys import numpy as np """ @@ -13,6 +14,8 @@ definition of the formats here: http://corpus-texmex.irisa.fr/ def ivecs_read(fname): a = np.fromfile(fname, dtype='int32') + if sys.big_endian: + a.byteswap(inplace=True) d = a[0] return a.reshape(-1, d + 1)[:, 1:].copy() @@ -22,6 +25,7 @@ def fvecs_read(fname): def ivecs_mmap(fname): + assert not sys.big_endian a = np.memmap(fname, dtype='int32', mode='r') d = a[0] return a.reshape(-1, d + 1)[:, 1:] @@ -33,7 +37,11 @@ def fvecs_mmap(fname): def bvecs_mmap(fname): x = np.memmap(fname, dtype='uint8', mode='r') - d = x[:4].view('int32')[0] + if sys.big_endian: + da = x[:4][::-1].copy() + d = da.view('int32')[0] + else: + d = x[:4].view('int32')[0] return x.reshape(-1, d + 4)[:, 4:] @@ -42,6 +50,8 @@ def ivecs_write(fname, m): m1 = np.empty((n, d + 1), dtype='int32') m1[:, 0] = d m1[:, 1:] = m + if sys.big_endian: + m1.byteswap(inplace=True) m1.tofile(fname) diff --git a/faiss/cppcontrib/detail/UintReader.h b/faiss/cppcontrib/detail/UintReader.h index 81e600f..d0b5836 100644 --- a/faiss/cppcontrib/detail/UintReader.h +++ b/faiss/cppcontrib/detail/UintReader.h @@ -7,8 +7,18 @@ #pragma once +#include #include +#ifdef FAISS_BIG_ENDIAN +#define Swap2Bytes(val) ((((val) >> 8) & 0x00FF) | (((val) << 8) & 0xFF00)) + +#define Swap4Bytes(val) \ + ((((val) >> 24) & 0x000000FF) | (((val) >> 8) & 0x0000FF00) | \ + (((val) << 8) & 0x00FF0000) | (((val) << 24) & 0xFF000000)) + +#endif + namespace faiss { namespace cppcontrib { namespace detail { @@ -31,7 +41,11 @@ struct Uint8Reader { if (N_ELEMENTS > CPOS + 3) { const uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 4); +#ifdef FAISS_BIG_ENDIAN + return (code32) >> 24; +#else return (code32 & 0x000000FF); +#endif } else { return codes[CPOS]; } @@ -40,7 +54,11 @@ struct Uint8Reader { if (N_ELEMENTS > CPOS + 2) { const uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 4); +#ifdef FAISS_BIG_ENDIAN + return (code32 & 0x00FF0000) >> 16; +#else return (code32 & 0x0000FF00) >> 8; +#endif } else { return codes[CPOS]; } @@ -49,7 +67,11 @@ struct Uint8Reader { if (N_ELEMENTS > CPOS + 1) { const uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 4); +#ifdef FAISS_BIG_ENDIAN + return (code32 & 0x0000FF00) >> 8; +#else return (code32 & 0x00FF0000) >> 16; +#endif } else { return codes[CPOS]; } @@ -58,7 +80,11 @@ struct Uint8Reader { if (N_ELEMENTS > CPOS) { const uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 4); +#ifdef FAISS_BIG_ENDIAN + return (code32 & 0x000000FF); +#else return (code32) >> 24; +#endif } else { return codes[CPOS]; } @@ -87,40 +113,61 @@ struct Uint10Reader { switch (SUB_ELEMENT) { case 0: { if (N_ELEMENTS > CPOS + 2) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b0000001111111111); } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5 + 0); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b0000001111111111); } } case 1: { if (N_ELEMENTS > CPOS + 1) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b000011111111110000000000) >> 10; } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5 + 1); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b0000111111111100) >> 2; } } case 2: { if (N_ELEMENTS > CPOS) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b00111111111100000000000000000000) >> 20; } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5 + 2); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b0011111111110000) >> 4; } } case 3: { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 5 + 3); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b1111111111000000) >> 6; } } @@ -147,45 +194,69 @@ struct Uint12Reader { switch (SUB_ELEMENT) { case 0: { if (N_ELEMENTS > CPOS + 2) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b0000111111111111); } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6 + 0); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b0000111111111111); } } case 1: { if (N_ELEMENTS > CPOS + 1) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b111111111111000000000000) >> 12; } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6 + 1); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b1111111111110000) >> 4; } } case 2: { if (N_ELEMENTS > CPOS + 1) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6 + 2); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b000011111111111100000000) >> 8; } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6 + 3); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b0000111111111111); } } case 3: { if (N_ELEMENTS > CPOS) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6 + 2); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0b11111111111100000000000000000000) >> 20; } else { - const uint16_t code16 = *reinterpret_cast( + uint16_t code16 = *reinterpret_cast( codes + ELEMENT_TO_READ * 6 + 4); +#ifdef FAISS_BIG_ENDIAN + code16 = Swap2Bytes(code16); +#endif return (code16 & 0b1111111111110000) >> 4; } } @@ -208,23 +279,39 @@ struct Uint16Reader { switch (SUB_ELEMENT) { case 0: { if (N_ELEMENTS > CPOS + 1) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 4); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return (code32 & 0x0000FFFF); } else { const uint16_t* const __restrict codesFp16 = reinterpret_cast(codes); +#ifdef FAISS_BIG_ENDIAN + uint16_t rt = codesFp16[CPOS]; + rt = Swap2Bytes(rt); + return rt; +#endif return codesFp16[CPOS]; } } case 1: { if (N_ELEMENTS > CPOS) { - const uint32_t code32 = *reinterpret_cast( + uint32_t code32 = *reinterpret_cast( codes + ELEMENT_TO_READ * 4); +#ifdef FAISS_BIG_ENDIAN + code32 = Swap4Bytes(code32); +#endif return code32 >> 16; } else { const uint16_t* const __restrict codesFp16 = reinterpret_cast(codes); +#ifdef FAISS_BIG_ENDIAN + uint16_t rt = codesFp16[CPOS]; + rt = Swap2Bytes(rt); + return rt; +#endif return codesFp16[CPOS]; } } diff --git a/faiss/cppcontrib/sa_decode/Level2-inl.h b/faiss/cppcontrib/sa_decode/Level2-inl.h index 36355af..4f9bdc4 100644 --- a/faiss/cppcontrib/sa_decode/Level2-inl.h +++ b/faiss/cppcontrib/sa_decode/Level2-inl.h @@ -12,6 +12,16 @@ #include #include +#include + +#ifdef FAISS_BIG_ENDIAN +#define Swap2Bytes(val) ((((val) >> 8) & 0x00FF) | (((val) << 8) & 0xFF00)) +#endif + +#ifndef FAISS_BIG_ENDIAN +#define FAISS_BIG_ENDIAN 0 +#define Swap2Bytes(val) val +#endif namespace faiss { namespace cppcontrib { @@ -72,9 +82,14 @@ struct Index2LevelDecoder { const intptr_t coarseCentroidOffset = i % COARSE_SIZE; const intptr_t fineCentroidIdx = i / FINE_SIZE; const intptr_t fineCentroidOffset = i % FINE_SIZE; - - const intptr_t coarseCode = coarse[coarseCentroidIdx]; - const intptr_t fineCode = fine[fineCentroidIdx]; + intptr_t coarseCode, fineCode; + if (FAISS_BIG_ENDIAN && sizeof(coarse_storage_type) == 2) { + coarseCode = Swap2Bytes(coarse[coarseCentroidIdx]); + fineCode = Swap2Bytes(fine[fineCentroidIdx]); + } else { + coarseCode = coarse[coarseCentroidIdx]; + fineCode = fine[fineCentroidIdx]; + } const float* const __restrict coarsePtr = pqCoarseCentroids + (coarseCentroidIdx * COARSE_TABLE_BYTES + coarseCode) * @@ -112,9 +127,14 @@ struct Index2LevelDecoder { const intptr_t fineCentroidIdx = i / FINE_SIZE; const intptr_t fineCentroidOffset = i % FINE_SIZE; - const intptr_t coarseCode = coarse[coarseCentroidIdx]; - const intptr_t fineCode = fine[fineCentroidIdx]; - + intptr_t coarseCode, fineCode; + if (FAISS_BIG_ENDIAN && sizeof(coarse_storage_type) == 2) { + coarseCode = Swap2Bytes(coarse[coarseCentroidIdx]); + fineCode = Swap2Bytes(fine[fineCentroidIdx]); + } else { + coarseCode = coarse[coarseCentroidIdx]; + fineCode = fine[fineCentroidIdx]; + } const float* const __restrict coarsePtr = pqCoarseCentroids + (coarseCentroidIdx * COARSE_TABLE_BYTES + coarseCode) * COARSE_SIZE + @@ -162,11 +182,18 @@ struct Index2LevelDecoder { const intptr_t coarseCentroidOffset = i % COARSE_SIZE; const intptr_t fineCentroidIdx = i / FINE_SIZE; const intptr_t fineCentroidOffset = i % FINE_SIZE; - - const intptr_t coarseCode0 = coarse0[coarseCentroidIdx]; - const intptr_t fineCode0 = fine0[fineCentroidIdx]; - const intptr_t coarseCode1 = coarse1[coarseCentroidIdx]; - const intptr_t fineCode1 = fine1[fineCentroidIdx]; + intptr_t coarseCode0, coarseCode1, fineCode0, fineCode1; + if (FAISS_BIG_ENDIAN && sizeof(coarse_storage_type) == 2) { + coarseCode0 = Swap2Bytes(coarse0[coarseCentroidIdx]); + fineCode0 = Swap2Bytes(fine0[fineCentroidIdx]); + coarseCode1 = Swap2Bytes(coarse1[coarseCentroidIdx]); + fineCode1 = Swap2Bytes(fine1[fineCentroidIdx]); + } else { + coarseCode0 = coarse0[coarseCentroidIdx]; + fineCode0 = fine0[fineCentroidIdx]; + coarseCode1 = coarse1[coarseCentroidIdx]; + fineCode1 = fine1[fineCentroidIdx]; + } const float* const __restrict coarsePtr0 = pqCoarseCentroids0 + (coarseCentroidIdx * COARSE_TABLE_BYTES + coarseCode0) * @@ -222,11 +249,18 @@ struct Index2LevelDecoder { const intptr_t coarseCentroidOffset = i % COARSE_SIZE; const intptr_t fineCentroidIdx = i / FINE_SIZE; const intptr_t fineCentroidOffset = i % FINE_SIZE; - - const intptr_t coarseCode0 = coarse0[coarseCentroidIdx]; - const intptr_t fineCode0 = fine0[fineCentroidIdx]; - const intptr_t coarseCode1 = coarse1[coarseCentroidIdx]; - const intptr_t fineCode1 = fine1[fineCentroidIdx]; + intptr_t coarseCode0, coarseCode1, fineCode0, fineCode1; + if (FAISS_BIG_ENDIAN && sizeof(coarse_storage_type) == 2) { + coarseCode0 = Swap2Bytes(coarse0[coarseCentroidIdx]); + fineCode0 = Swap2Bytes(fine0[fineCentroidIdx]); + coarseCode1 = Swap2Bytes(coarse1[coarseCentroidIdx]); + fineCode1 = Swap2Bytes(fine1[fineCentroidIdx]); + } else { + coarseCode0 = coarse0[coarseCentroidIdx]; + fineCode0 = fine0[fineCentroidIdx]; + coarseCode1 = coarse1[coarseCentroidIdx]; + fineCode1 = fine1[fineCentroidIdx]; + } const float* const __restrict coarsePtr0 = pqCoarseCentroids + (coarseCentroidIdx * COARSE_TABLE_BYTES + coarseCode0) * @@ -292,13 +326,23 @@ struct Index2LevelDecoder { const intptr_t coarseCentroidOffset = i % COARSE_SIZE; const intptr_t fineCentroidIdx = i / FINE_SIZE; const intptr_t fineCentroidOffset = i % FINE_SIZE; - - const intptr_t coarseCode0 = coarse0[coarseCentroidIdx]; - const intptr_t fineCode0 = fine0[fineCentroidIdx]; - const intptr_t coarseCode1 = coarse1[coarseCentroidIdx]; - const intptr_t fineCode1 = fine1[fineCentroidIdx]; - const intptr_t coarseCode2 = coarse2[coarseCentroidIdx]; - const intptr_t fineCode2 = fine2[fineCentroidIdx]; + intptr_t coarseCode0, coarseCode1, fineCode0, fineCode1; + intptr_t coarseCode2, fineCode2; + if (FAISS_BIG_ENDIAN && sizeof(coarse_storage_type) == 2) { + coarseCode0 = Swap2Bytes(coarse0[coarseCentroidIdx]); + fineCode0 = Swap2Bytes(fine0[fineCentroidIdx]); + coarseCode1 = Swap2Bytes(coarse1[coarseCentroidIdx]); + fineCode1 = Swap2Bytes(fine1[fineCentroidIdx]); + coarseCode2 = Swap2Bytes(coarse2[coarseCentroidIdx]); + fineCode2 = Swap2Bytes(fine2[fineCentroidIdx]); + } else { + coarseCode0 = coarse0[coarseCentroidIdx]; + fineCode0 = fine0[fineCentroidIdx]; + coarseCode1 = coarse1[coarseCentroidIdx]; + fineCode1 = fine1[fineCentroidIdx]; + coarseCode2 = coarse2[coarseCentroidIdx]; + fineCode2 = fine2[fineCentroidIdx]; + } const float* const __restrict coarsePtr0 = pqCoarseCentroids0 + (coarseCentroidIdx * COARSE_TABLE_BYTES + coarseCode0) * @@ -369,13 +413,23 @@ struct Index2LevelDecoder { const intptr_t coarseCentroidOffset = i % COARSE_SIZE; const intptr_t fineCentroidIdx = i / FINE_SIZE; const intptr_t fineCentroidOffset = i % FINE_SIZE; - - const intptr_t coarseCode0 = coarse0[coarseCentroidIdx]; - const intptr_t fineCode0 = fine0[fineCentroidIdx]; - const intptr_t coarseCode1 = coarse1[coarseCentroidIdx]; - const intptr_t fineCode1 = fine1[fineCentroidIdx]; - const intptr_t coarseCode2 = coarse2[coarseCentroidIdx]; - const intptr_t fineCode2 = fine2[fineCentroidIdx]; + intptr_t coarseCode0, fineCode0, coarseCode1, fineCode1; + intptr_t coarseCode2, fineCode2; + if (FAISS_BIG_ENDIAN && sizeof(coarse_storage_type) == 2) { + coarseCode0 = Swap2Bytes(coarse0[coarseCentroidIdx]); + fineCode0 = Swap2Bytes(fine0[fineCentroidIdx]); + coarseCode1 = Swap2Bytes(coarse1[coarseCentroidIdx]); + fineCode1 = Swap2Bytes(fine1[fineCentroidIdx]); + coarseCode2 = Swap2Bytes(coarse2[coarseCentroidIdx]); + fineCode2 = Swap2Bytes(fine2[fineCentroidIdx]); + } else { + coarseCode0 = coarse0[coarseCentroidIdx]; + fineCode0 = fine0[fineCentroidIdx]; + coarseCode1 = coarse1[coarseCentroidIdx]; + fineCode1 = fine1[fineCentroidIdx]; + coarseCode2 = coarse2[coarseCentroidIdx]; + fineCode2 = fine2[fineCentroidIdx]; + } const float* const __restrict coarsePtr0 = pqCoarseCentroids + (coarseCentroidIdx * COARSE_TABLE_BYTES + coarseCode0) * diff --git a/faiss/impl/platform_macros.h b/faiss/impl/platform_macros.h index 2aecc51..57e2343 100644 --- a/faiss/impl/platform_macros.h +++ b/faiss/impl/platform_macros.h @@ -165,3 +165,11 @@ inline int __builtin_clzll(uint64_t x) { #endif // clang-format on + +/******************************************************* + * BIGENDIAN specific macros + *******************************************************/ +#if !defined(_MSC_VER) && \ + (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) +#define FAISS_BIG_ENDIAN 1 +#endif diff --git a/faiss/impl/pq4_fast_scan.cpp b/faiss/impl/pq4_fast_scan.cpp index 6173ece..127646e 100644 --- a/faiss/impl/pq4_fast_scan.cpp +++ b/faiss/impl/pq4_fast_scan.cpp @@ -6,6 +6,7 @@ */ #include +#include #include #include @@ -58,8 +59,13 @@ void pq4_pack_codes( return; } memset(blocks, 0, nb * nsq / 2); +#ifdef FAISS_BIG_ENDIAN + const uint8_t perm0[16] = { + 8, 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7}; +#else const uint8_t perm0[16] = { 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15}; +#endif uint8_t* codes2 = blocks; for (size_t i0 = 0; i0 < nb; i0 += bbs) { @@ -93,8 +99,13 @@ void pq4_pack_codes_range( size_t bbs, size_t nsq, uint8_t* blocks) { +#ifdef FAISS_BIG_ENDIAN + const uint8_t perm0[16] = { + 8, 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7}; +#else const uint8_t perm0[16] = { 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15}; +#endif // range of affected blocks size_t block0 = i0 / bbs; diff --git a/faiss/python/CMakeLists.txt b/faiss/python/CMakeLists.txt index 26eaa54..f1a24dc 100644 --- a/faiss/python/CMakeLists.txt +++ b/faiss/python/CMakeLists.txt @@ -73,11 +73,20 @@ else() find_package(faiss REQUIRED) endif() +if (${CMAKE_SYSTEM_NAME} MATCHES "AIX") +swig_add_library(swigfaiss + TYPE MODULE + LANGUAGE python + SOURCES swigfaiss.swig +) +else () swig_add_library(swigfaiss TYPE SHARED LANGUAGE python SOURCES swigfaiss.swig ) +endif() + set_property(TARGET swigfaiss PROPERTY SWIG_COMPILE_OPTIONS -doxygen) set_property(SOURCE swigfaiss_avx2.swig @@ -166,6 +175,10 @@ set_property(TARGET faiss_python_callbacks PROPERTY POSITION_INDEPENDENT_CODE ON ) +if (${CMAKE_SYSTEM_NAME} MATCHES "AIX") +target_link_libraries(faiss_python_callbacks PRIVATE faiss) +endif() + # Hack so that python_callbacks.h can be included as # `#include `. target_include_directories(faiss_python_callbacks PRIVATE ${PROJECT_SOURCE_DIR}/../..) -- cgit v1.2.3