I am trying to compile this project in Ubuntu(wsl). Intel-mkl libraries should be installed (sudo apt install intel-mkl) and faiss has been built and installed without seemingly any issues. However, when I try to build the project using make command, I am met with errors.
CMakeLists
project(faiss_clustering)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti -Wall -pedantic -Wextra -fopenmp")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -fsanitize=address")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -g -flto -ftree-vectorize -march=native")
add_executable(faiss_clustering main.cpp)
target_link_libraries(faiss_clustering PRIVATE faiss blas)
main.cpp
#include <iostream>
#include <vector>
#include <string>
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <faiss/Clustering.h>
#include <faiss/IndexFlat.h>
#include <faiss/utils/utils.h>
using namespace std;
struct sfile_info {
size_t dim;
size_t count;
char file_name[256];
sfile_info(char* p_file_name) {
strcpy(file_name, p_file_name);
}
};
float* fvecs_read(sfile_info& fi)
{
FILE* f = fopen(fi.file_name, "r");
if (!f) {
fprintf(stderr, "could not open %s\n", fi.file_name);
perror("");
abort();
}
int d;
fread(&d, 1, sizeof(int), f);
assert((d > 0 && d < 1000000) || !"unreasonable dimension");
fseek(f, 0, SEEK_SET);
struct stat st;
fstat(fileno(f), &st);
size_t sz = st.st_size;
assert(sz % ((d + 1) * 4) == 0 || !"weird file size");
size_t n = sz / ((d + 1) * 4);
fi.dim = d;
fi.count = n;
float* x = new float[n * (d + 1)];
size_t nr = fread(x, sizeof(float), n * (d + 1), f);
assert(nr == n * (d + 1) || !"could not read whole file");
// shift array to remove row headers
for (size_t i = 0; i < n; i++)
memmove(x + i * d, x + 1 + i * (d + 1), d * sizeof(*x));
fclose(f);
return x;
}
// not very clean, but works as long as sizeof(int) == sizeof(float)
int* ivecs_read(sfile_info& fi) {
return (int*)fvecs_read(fi);
}
double distance(float* data, float* query, size_t dim)
{
double dist = 0;
for (int k = 0; k < dim; k++)
{
double r = data[k] - query[k];
dist += r * r;
}
return dist;
}
int main()
{
int numberOfEMIterations = 20;
size_t numberOfClusters = 1000;
/////////////////////////////////////////////////////// READ DATA
sfile_info fi_base("../../sift/sift_base.fvecs");
sfile_info fi_learn("../../sift/sift_learn.fvecs");
sfile_info fi_query("../../sift/sift_query.fvecs");
sfile_info fi_groundtruth("../../sift/sift_groundtruth.ivecs");
float* mass = fvecs_read(fi_base);
float* massL = fvecs_read(fi_learn);
float* massQ = fvecs_read(fi_query);
int* massQA = ivecs_read(fi_groundtruth);
assert(fi_base.dim == fi_query.dim || !"query does not have same dimension as base set");
assert(fi_base.dim == fi_learn.dim || !"learn does not have same dimension as base set");
assert(fi_query.count == fi_groundtruth.count || !"incorrect number of ground truth entries");
/////////////////////////////////////////////////////// PROCESS CLUSTERING
faiss::ClusteringParameters cp;
cp.niter = numberOfEMIterations;
cp.verbose = true; // print out per-iteration stats
faiss::IndexFlatL2 index(fi_learn.dim);
faiss::Clustering kMeans(fi_learn.dim, numberOfClusters, cp);
kMeans.train(fi_learn.count, massL, index);
// Print first ten centroids
int print_center_count = 10;
for (int c = 0; c < numberOfClusters; c++)
{
if (print_center_count-- == 0) break;
for (int d = 0; d < fi_learn.dim; d++) {
std::cout << kMeans.centroids[c * fi_learn.dim + d] << " ";
}
std::cout << c << "\n";
}
}
Errors:
kudlic@LAPTOP-FUCA8DQ5:/mnt/d/codes/AVD/AVD/faiss_clustering/build$ make
[ 50%] Linking CXX executable faiss_clustering
/usr/bin/ld: /usr/local/lib/libfaiss.a(VectorTransform.cpp.o): in function `(anonymous namespace)::eig(unsigned long, double*, double*, int)':
VectorTransform.cpp:(.text+0x176a): undefined reference to `dsyev_'
/usr/bin/ld: VectorTransform.cpp:(.text+0x17e4): undefined reference to `dsyev_'
/usr/bin/ld: /usr/local/lib/libfaiss.a(VectorTransform.cpp.o): in function `faiss::ITQMatrix::train(long, float const*)':
VectorTransform.cpp:(.text+0x43d4): undefined reference to `dgesvd_'
/usr/bin/ld: VectorTransform.cpp:(.text+0x4606): undefined reference to `dgesvd_'
/usr/bin/ld: /usr/local/lib/libfaiss.a(VectorTransform.cpp.o): in function `faiss::OPQMatrix::train(long, float const*)':
VectorTransform.cpp:(.text+0x6a7c): undefined reference to `sgesvd_'
/usr/bin/ld: VectorTransform.cpp:(.text+0x6b63): undefined reference to `sgesvd_'
/usr/bin/ld: /usr/local/lib/libfaiss.a(utils.cpp.o): in function `faiss::matrix_qr(int, int, float*)':
utils.cpp:(.text+0x86b): undefined reference to `sgeqrf_'
/usr/bin/ld: utils.cpp:(.text+0x93d): undefined reference to `sgeqrf_'
/usr/bin/ld: utils.cpp:(.text+0x9a5): undefined reference to `sorgqr_'
collect2: error: ld returned 1 exit status
make[2]: *** [CMakeFiles/faiss_clustering.dir/build.make:97: faiss_clustering] Error 1
make[1]: *** [CMakeFiles/Makefile2:83: CMakeFiles/faiss_clustering.dir/all] Error 2
make: *** [Makefile:91: all] Error 2
If I try to go to VectorTransform.cpp in vscode and hover over one of the problematic functions, there seems to be only declaration, as these seem to be intel-mkl functions.
I first tried building and installing faiss library in windows directories, where the project is. I then tried to build and install it again in linux user directory, in hopes it might fix it, alas to no avail.
Do you have any ideas what steps I should take to resolve the issue? If I forgot any info, tell me and I will add it in.
As specified on: https://cmake.org/cmake/help/latest/module/FindLAPACK.html Adding following snippet to CMakeLists solves the issue
set(BLA_VENDOR Intel10_64lp)
find_package(LAPACK)