c++linuxintel-mklfaiss

Faiss project not compiling, undefined refferences


I am trying to compile this project in Ubuntu(wsl). Intel-mkl libraries should be installed (sudo apt install intel-mkl) and faiss has been built and installed without seemingly any issues. However, when I try to build the project using make command, I am met with errors.

CMakeLists

project(faiss_clustering)


set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti -Wall -pedantic -Wextra -fopenmp")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -fsanitize=address")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -g -flto -ftree-vectorize -march=native")


add_executable(faiss_clustering main.cpp)
target_link_libraries(faiss_clustering PRIVATE faiss blas)

main.cpp

#include <iostream>
#include <vector>
#include <string>
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>

#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

#include <faiss/Clustering.h>
#include <faiss/IndexFlat.h>
#include <faiss/utils/utils.h>


using namespace std;

struct sfile_info {
    size_t dim;
    size_t count;
    char file_name[256];

    sfile_info(char* p_file_name) {
        strcpy(file_name, p_file_name);
    }
};

float* fvecs_read(sfile_info& fi)
{
    FILE* f = fopen(fi.file_name, "r");
    if (!f) {
        fprintf(stderr, "could not open %s\n", fi.file_name);
        perror("");
        abort();
    }
    int d;
    fread(&d, 1, sizeof(int), f);
    assert((d > 0 && d < 1000000) || !"unreasonable dimension");
    fseek(f, 0, SEEK_SET);
    struct stat st;
    fstat(fileno(f), &st);
    size_t sz = st.st_size;
    assert(sz % ((d + 1) * 4) == 0 || !"weird file size");
    size_t n = sz / ((d + 1) * 4);

    fi.dim = d;
    fi.count = n;
    float* x = new float[n * (d + 1)];
    size_t nr = fread(x, sizeof(float), n * (d + 1), f);
    assert(nr == n * (d + 1) || !"could not read whole file");

    // shift array to remove row headers
    for (size_t i = 0; i < n; i++)
        memmove(x + i * d, x + 1 + i * (d + 1), d * sizeof(*x));

    fclose(f);
    return x;
}

// not very clean, but works as long as sizeof(int) == sizeof(float)
int* ivecs_read(sfile_info& fi) {
    return (int*)fvecs_read(fi);
}

double distance(float* data, float* query, size_t dim)
{
    double dist = 0;
    for (int k = 0; k < dim; k++)
    {
        double r = data[k] - query[k];
        dist += r * r;
    }
    return dist;
}


int main()
{
    int numberOfEMIterations = 20;
    size_t numberOfClusters = 1000;

    /////////////////////////////////////////////////////// READ DATA
    sfile_info fi_base("../../sift/sift_base.fvecs");
    sfile_info fi_learn("../../sift/sift_learn.fvecs");
    sfile_info fi_query("../../sift/sift_query.fvecs");
    sfile_info fi_groundtruth("../../sift/sift_groundtruth.ivecs");

    float* mass = fvecs_read(fi_base);
    float* massL = fvecs_read(fi_learn);
    float* massQ = fvecs_read(fi_query);
    int* massQA = ivecs_read(fi_groundtruth);

    assert(fi_base.dim == fi_query.dim || !"query does not have same dimension as base set");
    assert(fi_base.dim == fi_learn.dim || !"learn does not have same dimension as base set");
    assert(fi_query.count == fi_groundtruth.count || !"incorrect number of ground truth entries");


    /////////////////////////////////////////////////////// PROCESS CLUSTERING

    faiss::ClusteringParameters cp;
    cp.niter = numberOfEMIterations;
    cp.verbose = true; // print out per-iteration stats

    faiss::IndexFlatL2 index(fi_learn.dim);
    faiss::Clustering kMeans(fi_learn.dim, numberOfClusters, cp);
    kMeans.train(fi_learn.count, massL, index);

    // Print first ten centroids
    int print_center_count = 10;
    for (int c = 0; c < numberOfClusters; c++)
    {
        if (print_center_count-- == 0) break;
        for (int d = 0; d < fi_learn.dim; d++) {
            std::cout << kMeans.centroids[c * fi_learn.dim + d] << " ";    
        }
        std::cout << c << "\n";
    }
  
}

Errors:

kudlic@LAPTOP-FUCA8DQ5:/mnt/d/codes/AVD/AVD/faiss_clustering/build$ make
[ 50%] Linking CXX executable faiss_clustering
/usr/bin/ld: /usr/local/lib/libfaiss.a(VectorTransform.cpp.o): in function `(anonymous namespace)::eig(unsigned long, double*, double*, int)':
VectorTransform.cpp:(.text+0x176a): undefined reference to `dsyev_'
/usr/bin/ld: VectorTransform.cpp:(.text+0x17e4): undefined reference to `dsyev_'
/usr/bin/ld: /usr/local/lib/libfaiss.a(VectorTransform.cpp.o): in function `faiss::ITQMatrix::train(long, float const*)':
VectorTransform.cpp:(.text+0x43d4): undefined reference to `dgesvd_'
/usr/bin/ld: VectorTransform.cpp:(.text+0x4606): undefined reference to `dgesvd_'
/usr/bin/ld: /usr/local/lib/libfaiss.a(VectorTransform.cpp.o): in function `faiss::OPQMatrix::train(long, float const*)':
VectorTransform.cpp:(.text+0x6a7c): undefined reference to `sgesvd_'
/usr/bin/ld: VectorTransform.cpp:(.text+0x6b63): undefined reference to `sgesvd_'
/usr/bin/ld: /usr/local/lib/libfaiss.a(utils.cpp.o): in function `faiss::matrix_qr(int, int, float*)':
utils.cpp:(.text+0x86b): undefined reference to `sgeqrf_'
/usr/bin/ld: utils.cpp:(.text+0x93d): undefined reference to `sgeqrf_'
/usr/bin/ld: utils.cpp:(.text+0x9a5): undefined reference to `sorgqr_'
collect2: error: ld returned 1 exit status
make[2]: *** [CMakeFiles/faiss_clustering.dir/build.make:97: faiss_clustering] Error 1
make[1]: *** [CMakeFiles/Makefile2:83: CMakeFiles/faiss_clustering.dir/all] Error 2
make: *** [Makefile:91: all] Error 2

If I try to go to VectorTransform.cpp in vscode and hover over one of the problematic functions, there seems to be only declaration, as these seem to be intel-mkl functions. enter image description here

I first tried building and installing faiss library in windows directories, where the project is. I then tried to build and install it again in linux user directory, in hopes it might fix it, alas to no avail.

Do you have any ideas what steps I should take to resolve the issue? If I forgot any info, tell me and I will add it in.


Solution

  • As specified on: https://cmake.org/cmake/help/latest/module/FindLAPACK.html Adding following snippet to CMakeLists solves the issue

    set(BLA_VENDOR Intel10_64lp) 
    find_package(LAPACK)