I am trying to write some c++ code that calls ScaLAPACK and I encountered this problem. I want to do a column reordering for a distributed matrix. Following the explanation together with the source code in ScaLAPACK, I still cannot figure out what is wrong with my parameters. Here is a minimum 4 processes 7x7 matrix, 6x6 block cyclic example:
#include <cstdlib>
#include <cassert>
#include <string>
#include <iostream>
#include <iomanip>
#include <vector>
#include "mpi.h"
// build with:
// mpicxx --std=c++17 test.cpp -L/opt/scalapack-2.2.0 -L/opt/LAPACK/3.10.0/ -L/opt/OpenBLAS/0.3.19/lib64 -lscalapack -llapack -lopenblas -lgfortran -o test.x
// run with:
// OMP_NUM_THREADS=1 LD_LIBRARY_PATH=/opt/LAPACK/3.10.0:/opt/OpenBLAS/0.3.19/lib64:/opt/MPICH/4.0.2/lib:$LD_LIBRARY_PATH mpirun -np 4 ./test.x
int myid;
extern "C" {
void Cblacs_get(const int ictxt, const int what, int *val);
void Cblacs_pinfo(int *myrank, int *nprocs);
void Cblacs_gridinit(int *ictxt, const char *order, const int nprow, const int npcol);
void Cblacs_gridinfo(const int ictxt, int *nprow, int *npcol, int *myrow, int *mycol);
void Cblacs_gridexit(const int ictxt);
void descinit_(int *desc,
const int *m, const int *n, const int *mb, const int *nb, const int *irsrc, const int *icsrc, const int *ictxt, const int *lld, const int *info);
void pdlapiv_(char *direc, char *rowcol, char *pivroc, int *m, int *n, double *A, int *IA, int *JA, int *DESCA, int *IPIV, int *ip, int *jp, int *descip, int *iwork);
void pdlapv2_(char *direc, char *rowcol, int *m, int *n, double *A, int *IA, int *JA, int *DESCA, int *IPIV, int *ip, int *jp, int *descip);
// compute LOCr or LOCc (local size of data for distributed array)
int numroc_(int *n, int *nb, int *iproc, int *isrcproc, int *nprocs);
}
void random_fill_matrix(double *A, int locr, int locc) {
std::srand(42+myid);
for (int i = 0; i < locr; i++)
for (int j = 0; j < locc; j++)
A[i + j * locr] = (double)rand() / RAND_MAX;
}
void validate_pdlapiv() {
int numprocs, info, ictxt;
int myrow, mycol;
int nprow = 2, npcol = 2;
int zero = 0, one = 1, two = 2;
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
// Initialize BLACS context
Cblacs_pinfo(&myid, &numprocs);
Cblacs_get(0, 0, &ictxt);
Cblacs_gridinit(&ictxt, "Row", nprow, npcol);
Cblacs_gridinfo(ictxt, &nprow, &npcol, &myrow, &mycol);
// Define matrix dimensions and block size
int m = 7, n = 7, mb = 6, nb = 6;
double *A, *B;
// Initialize matrix descriptor for A
int locr = numroc_(&m, &mb, &myrow, &zero, &nprow); // Local number of rows of A
int locc = numroc_(&n, &nb, &mycol, &zero, &npcol); // Local number of columns of A
int lldA = std::max(1, locr); // Local leading dimension of A
int descA[9], descB[9], descip[9];
descinit_(descA, &m, &n, &mb, &nb, &zero, &zero, &ictxt, &lldA, &info);
descinit_(descB, &m, &n, &mb, &nb, &zero, &zero, &ictxt, &lldA, &info);
descinit_(descip, &two, &n, &one, &nb, &zero, &zero, &ictxt, &lldA, &info);
std::vector<double> vA(locr * locc, 0);
std::vector<double> vB(locr * locc, 0);
A = vA.data();
B = vB.data();
// Initialize matrix A with random values and make a copy because PSGEQPF will overwrite it
random_fill_matrix(A, locr, locc);
std::copy(A, A + locr * locc, B);
std::vector<int> ipiv;
if (myid % 2 == 0) {
ipiv = std::vector<int> {3, 4, 6, 7, 2, 1};
} else {
ipiv = std::vector<int> {5};
}
char direc = 'F', rowcol = 'C';
// append several 0 to ipiv
ipiv.insert(ipiv.end(), 6, 0);
pdlapv2_(&direc, &rowcol, &m, &n, A, &one, &one, descA, ipiv.data(), &one, &one, descip);
Cblacs_gridexit(ictxt);
}
int main(int argc, char **argv) {
MPI_Init(&argc, &argv);
validate_pdlapiv();
MPI_Finalize();
return 0;
}
The program runs but ends with seg fault, and the permuted result is incorrect. (I wanted to move the first column to the third, and the second to the fourth, see defs of ipiv.)
What might be the correct way to fix this?
EDIT. I included the full cpp file runable. The seg fault is:
*** Error in `./test.x': free(): invalid next size (normal): 0x00000000015b6680 ***
*** Error in `./test.x': free(): invalid next size (normal): 0x00000000023fc680 ***
======= Backtrace: =========
/lib64/libc.so.6(+0x81329)[0x7ffb628c4329]
======= Backtrace: =========
/lib64/libc.so.6(+0x81329)[0x7f25f0d6f329]
/opt/MPICH/4.0.2/lib/libmpi.so.12(+0x340c20)[0x7f25f1cf0c20]
/opt/MPICH/4.0.2/lib/libmpi.so.12(+0x45a674)[0x7f25f1e0a674]
/opt/MPICH/4.0.2/lib/libmpi.so.12(+0x3a9952)[0x7f25f1d59952]
/opt/MPICH/4.0.2/lib/libmpi.so.12(+0x340c20)[0x7ffb63845c20]
/opt/MPICH/4.0.2/lib/libmpi.so.12(+0x45a674)[0x7ffb6395f674]
/opt/MPICH/4.0.2/lib/libmpi.so.12(+0x3a9952)[0x7ffb638ae952]
/opt/MPICH/4.0.2/lib/libmpi.so.12(+0x3a9bfd)[0x7ffb638aebfd]
/opt/MPICH/4.0.2/lib/libmpi.so.12(PMPI_Comm_free+0xe7)[0x7ffb635c3cf7]
/opt/MPICH/4.0.2/lib/libmpi.so.12(+0x3a9bfd)[0x7f25f1d59bfd]
/opt/MPICH/4.0.2/lib/libmpi.so.12(PMPI_Comm_free+0xe7)[0x7f25f1a6ecf7]
./test.x[0x40565f]
./test.x[0x4027d6]
./test.x[0x40565f]
./test.x[0x4027d6]
./test.x[0x4028d4]
./test.x[0x4028d4]
/lib64/libc.so.6(__libc_start_main+0xf5)[0x7ffb62865555]
./test.x[0x4020c9]
======= Memory map: ========
/lib64/libc.so.6(__libc_start_main+0xf5)[0x7f25f0d10555]
./test.x[0x4020c9]
======= Memory map: ========
00400000-00424000 r-xp 00000000 fd:00 6443312077 /home/user01/test/unittest/test.x
00623000-00624000 r--p 00023000 fd:00 6443312077 /home/user01/test/unittest/test.x
00624000-00625000 rw-p 00024000 fd:00 6443312077 /home/user01/test/unittest/test.x
0147d000-015e1000 rw-p 00000000 00:00 0 [heap]
200000000-400200000 ---p 00000000 00:00 0
7ffb30000000-7ffb40000000 ---p 00000000 00:00 0
7ffb40000000-7ffb40021000 rw-p 00000000 00:00 0
7ffb40021000-7ffb44000000 ---p 00000000 00:00 0
7ffb48000000-7ffb48021000 rw-p 00000000 00:00 0
7ffb48021000-7ffb4c000000 ---p 00000000 00:00 0
7ffb4eecd000-7ffb4f0e5000 rw-p 00000000 00:00 0
7ffb4f0e5000-7ffb550e5000 ---p 00000000 00:00 0
7ffb550e5000-7ffb5d0e5000 rw-p 00000000 00:00 0
7ffb5d1eb000-7ffb5e2ee000 rw-s 00000000 00:13 48103092 /dev/shm/mpic00400000-00424000 r-xp 00000000 fd:00 6443312077 /home/user01/test/unittest/test.x
00623000-00624000 r--p 00023000 fd:00 6443312077 /home/user01/test/unittest/test.x
00624000-00625000 rw-p 00024000 fd:00 6443312077 /home/user01/test/unittest/test.x
022c3000-02427000 rw-p 00000000 00:00 0 [heap]
200000000-400200000 ---p 00000000 00:00 0
7f25c0000000-7f25d0000000 ---p 00000000 00:00 0
7f25d0000000-7f25d0021000 rw-p 00000000 00:00 0
7f25d0021000-7f25d4000000 ---p 00000000 00:00 0
7f25d8000000-7f25d8021000 rw-p 00000000 00:00 0
7f25d8021000-7f25dc000000 ---p 00000000 00:00 0
7f25dd449000-7f25dd661000 rw-p 00000000 00:00 0
7f25dd661000-7f25de764000 rw-s 00000000 00:13 48103092 /dev/shm/mpich_shar_tmpExWL1x (deleted)
7f25de764000-7f25deb68000 rw-p 00000000 00:00 0
7f25deb68000-7f25deb69h_shar_tmpExWL1x (deleted)
7ffb5e2ee000-7ffb5eaf6000 rw-p 00000000 00:00 0
7ffb5eaf6000-7ffb5eaf7000 ---p 00000000 00:00 0
7ffb5eaf7000-7ffb5f2f7000 rw-p 00000000 00:00 0
7ffb5f2f7000-7ffb5f2f8000 ---p 00000000 00:00 0
7ffb5f2f8000-7ffb5faf8000 rw-p 00000000 00:00 0
7ffb5faf8000-7ffb5faf9000 ---p 00000000 00:00 0
7ffb5faf9000-7ffb602f9000 rw-p 00000000 00:00 0
7ffb602f9000-7ffb60300000 r-xp 00000000 08:02 1611635199 /usr/lib64/librt-2.17.so
7ffb60300000-7ffb604ff000 ---p 00007000 08:02 1611635199 /usr/lib64/librt-2.17.so
7ffb604ff000-7ffb60500000 r--p 00006000 08:02 1611635199 /usr/lib64/librt-2.17.so
7ffb60500000-7ffb60501000 rw-p 00007000 08:02 1611635199 /usr/lib64/librt-2.17.so
7ffb60501000-7ffb618cc000 r-xp 00000000 08:02 1714832245 /usr/lib64/libcuda.so.510.47.03
7ffb618cc000-7ffb61acb000 ---p 013cb000 08:02 1714832245 /usr/lib64/libcuda.so.510.47.03
7ffb61acb000-7ffb61bc1000 r--p 013ca000 08:02 171483000 ---p 00000000 00:00 0
7f25deb69000-7f25df369000 rw-p 00000000 00:00 0
7f25df369000-7f25e7369000 rw-p 00000000 00:00 0
7f25e739e000-7f25e77a2000 rw-p 00000000 00:00 0
7f25e77a2000-7f25e77a3000 ---p 00000000 00:00 0
7f25e77a3000-7f25e7fa3000 rw-p 00000000 00:00 0
7f25e7fa3000-7f25e7fa4000 ---p 00000000 00:00 0
7f25e7fa4000-7f25e87a4000 rw-p 00000000 00:00 0
7f25e87a4000-7f25ee7a4000 ---p 00000000 00:00 0
7f25ee7a4000-7f25ee7ab000 r-xp 00000000 08:02 1611635199 /usr/lib64/librt-2.17.so
7f25ee7ab000-7f25ee9aa000 ---p 00007000 08:02 1611635199 /usr/lib64/librt-2.17.so
7f25ee9aa000-7f25ee9ab000 r--p 00006000 08:02 1611635199 /usr/lib64/librt-2.17.so
7f25ee9ab000-7f25ee9ac000 rw-p 00007000 08:02 1611635199 /usr/lib64/librt-2.17.so
7f25ee9ac000-7f25efd77000 r-xp 00000000 08:02 1714832245 /usr/lib64/libcuda.so.510.47.03
7f25efd77000-7f25eff76000 ---p 013cb000 08:02 1714832245 /usr/lib64/libcuda.so.510.47.03
7f22245 /usr/lib64/libcuda.so.510.47.03
7ffb61bc1000-7ffb61ccc000 rw-p 014c0000 08:02 1714832245 /usr/lib64/libcuda.so.510.47.03
7ffb61ccc000-7ffb61d31000 rw-p 00000000 00:00 0
7ffb61d31000-7ffb61dce000 r-xp 00000000 08:02 966309 /opt/CUDA/11.6/targets/x86_64-linux/lib/libcudart.so.11.6.55
7ffb61dce000-7ffb61fce000 ---p 0009d000 08:02 966309 /opt/CUDA/11.6/targets/x86_64-linux/lib/libcudart.so.11.6.55
7ffb61fce000-7ffb61fd3000 r--p 0009d000 08:02 966309 /opt/CUDA/11.6/targets/x86_64-linux/lib/libcudart.so.11.6.55
7ffb61fd3000-7ffb61fd4000 rw-p 000a2000 08:02 966309 /opt/CUDA/11.6/targets/x86_64-linux/lib/libcudart.so.11.6.55
7ffb61fd4000-7ffb61fd5000 rw-p 00000000 00:00 0
7ffb61fd5000-7ffb61fd7000 r-xp 00000000 08:02 1611635192 /usr/lib64/libdl-2.17.so
7ffb61fd7000-7ffb621d7000 ---p 00002000 08:02 1611635192 /usr/lib64/libdl-2.17.so
7ffb621d7000-7ffb621d8000 r--p 000025eff76000-7f25f006c000 r--p 013ca000 08:02 1714832245 /usr/lib64/libcuda.so.510.47.03
7f25f006c000-7f25f0177000 rw-p 014c0000 08:02 1714832245 /usr/lib64/libcuda.so.510.47.03
7f25f0177000-7f25f01dc000 rw-p 00000000 00:00 0
7f25f01dc000-7f25f0279000 r-xp 00000000 08:02 966309 /opt/CUDA/11.6/targets/x86_64-linux/lib/libcudart.so.11.6.55
7f25f0279000-7f25f0479000 ---p 0009d000 08:02 966309 /opt/CUDA/11.6/targets/x86_64-linux/lib/libcudart.so.11.6.55
7f25f0479000-7f25f047e000 r--p 0009d000 08:02 966309 /opt/CUDA/11.6/targets/x86_64-linux/lib/libcudart.so.11.6.55
7f25f047e000-7f25f047f000 rw-p 000a2000 08:02 966309 /opt/CUDA/11.6/targets/x86_64-linux/lib/libcudart.so.11.6.55
7f25f047f000-7f25f0480000 rw-p 00000000 00:00 0
7f25f0480000-7f25f0482000 r-xp 00000000 08:02 1611635192 /usr/lib64/libdl-2.17.so
7f25f0482000-7f25f0682000 ---p 00002000 08:02 1611635192 /usr/lib64/l000 08:02 1611635192 /usr/lib64/libdl-2.17.so
7ffb621d8000-7ffb621d9000 rw-p 00003000 08:02 1611635192 /usr/lib64/libdl-2.17.so
7ffb621d9000-7ffb621df000 r-xp 00000000 08:02 1079834231 /opt/GCC/9.4.0/lib64/libatomic.so.1.2.0
7ffb621df000-7ffb623de000 ---p 00006000 08:02 1079834231 /opt/GCC/9.4.0/lib64/libatomic.so.1.2.0
7ffb623de000-7ffb623df000 r--p 00005000 08:02 1079834231 /opt/GCC/9.4.0/lib64/libatomic.so.1.2.0
7ffb623df000-7ffb623e0000 rw-p 00006000 08:02 1079834231 /opt/GCC/9.4.0/lib64/libatomic.so.1.2.0
7ffb623e0000-7ffb623e1000 rw-p 00000000 00:00 0
7ffb623e1000-7ffb623f8000 r-xp 00000000 08:02 1610773693 /usr/lib64/libpthread-2.17.so
7ffb623f8000-7ffb625f7000 ---p 00017000 08:02 1610773693 /usr/lib64/libpthread-2.17.so
7ffb625f7000-7ffb625f8000 r--p 00016000 08:02 1610773693 /usr/lib64/libpthread-2.17.so
7ffb625f8000-7ffb625f9000 rw-p 00017000 08:02 16107736ibdl-2.17.so
7f25f0682000-7f25f0683000 r--p 00002000 08:02 1611635192 /usr/lib64/libdl-2.17.so
7f25f0683000-7f25f0684000 rw-p 00003000 08:02 1611635192 /usr/lib64/libdl-2.17.so
7f25f0684000-7f25f068a000 r-xp 00000000 08:02 1079834231 /opt/GCC/9.4.0/lib64/libatomic.so.1.2.0
7f25f068a000-7f25f0889000 ---p 00006000 08:02 1079834231 /opt/GCC/9.4.0/lib64/libatomic.so.1.2.0
7f25f0889000-7f25f088a000 r--p 00005000 08:02 1079834231 /opt/GCC/9.4.0/lib64/libatomic.so.1.2.0
7f25f088a000-7f25f088b000 rw-p 00006000 08:02 1079834231 /opt/GCC/9.4.0/lib64/libatomic.so.1.2.0
7f25f088b000-7f25f088c000 rw-p 00000000 00:00 0
7f25f088c000-7f25f08a3000 r-xp 00000000 08:02 1610773693 /usr/lib64/libpthread-2.17.so
7f25f08a3000-7f25f0aa2000 ---p 00017000 08:02 1610773693 /usr/lib64/libpthread-2.17.so
7f25f0aa2000-7f25f0aa3000 r--p 00016000 08:02 1610773693 /usr/lib64/libpthread-2.17.so
7f25f93 /usr/lib64/libpthread-2.17.so
7ffb625f9000-7ffb625fd000 rw-p 00000000 00:00 0
7ffb625fd000-7ffb62642000 r-xp 00000000 08:02 1079834189 /opt/GCC/9.4.0/lib64/libquadmath.so.0.0.0
7ffb62642000-7ffb62841000 ---p 00045000 08:02 1079834189 /opt/GCC/9.4.0/lib64/libquadmath.so.0.0.0
7ffb62841000-7ffb62842000 r--p 00044000 08:02 1079834189 /opt/GCC/9.4.0/lib64/libquadmath.so.0.0.0
7ffb62842000-7ffb62843000 rw-p 00045000 08:02 1079834189 /opt/GCC/9.4.0/lib64/libquadmath.so.0.0.0
7ffb62843000-7ffb62a07000 r-xp 00000000 08:02 1611635190 /usr/lib64/libc-2.17.so
7ffb62a07000-7ffb62c06000 ---p 001c4000 08:02 1611635190 /usr/lib64/libc-2.17.so
7ffb62c06000-7ffb62c0a000 r--p 001c3000 08:02 1611635190 /usr/lib64/libc-2.17.so
7ffb62c0a000-7ffb62c0c000 rw-p 001c7000 08:02 1611635190 /usr/lib64/libc-2.17.so
7ffb62c0c000-7ffb62c11000 rw-p 00000000 00:00 0
7ffb62c11000-7ffb62c28000 r-x0aa3000-7f25f0aa4000 rw-p 00017000 08:02 1610773693 /usr/lib64/libpthread-2.17.so
7f25f0aa4000-7f25f0aa8000 rw-p 00000000 00:00 0
7f25f0aa8000-7f25f0aed000 r-xp 00000000 08:02 1079834189 /opt/GCC/9.4.0/lib64/libquadmath.so.0.0.0
7f25f0aed000-7f25f0cec000 ---p 00045000 08:02 1079834189 /opt/GCC/9.4.0/lib64/libquadmath.so.0.0.0
7f25f0cec000-7f25f0ced000 r--p 00044000 08:02 1079834189 /opt/GCC/9.4.0/lib64/libquadmath.so.0.0.0
7f25f0ced000-7f25f0cee000 rw-p 00045000 08:02 1079834189 /opt/GCC/9.4.0/lib64/libquadmath.so.0.0.0
7f25f0cee000-7f25f0eb2000 r-xp 00000000 08:02 1611635190 /usr/lib64/libc-2.17.so
7f25f0eb2000-7f25f10b1000 ---p 001c4000 08:02 1611635190 /usr/lib64/libc-2.17.so
7f25f10b1000-7f25f10b5000 r--p 001c3000 08:02 1611635190 /usr/lib64/libc-2.17.so
7f25f10b5000-7f25f10b7000 rw-p 001c7000 08:02 1611635190 /usr/lib64/libc-2.17.so
7f25f10b7000-7f25f10bc000 rw-p 00000000 00:00 0
7f25f10bc000-7f25f10d3000 r-xp 00000000 08:02 1241751835 /opt/GCC/9.4.0/lib64/libgcc_s.so.1
7f25f10d3000-7f25f12d2000 ---p 00017000 08:02 1241751835 /opt/GCC/9.4.0/lib64/libgcc_s.so.1
7f25f12d2000-7f25f12d3000 r--p 00016000 08:02 1241751835 /opt/GCC/9.4.0/lib64/libgcc_s.so.1
7f25f12d3000-7f25f12d4000 rw-p 00017000 08:02 1241751835 /opt/GCC/9.4.0/lib64/libgcc_s.so.1
7f25f12d4000-7f25f13d5000 r-xp 00000000 08:02 1611635193 /usr/lib64/libm-2.17.so
7f25f13d5000-7f25f15d4000 ---p 00101000 08:02 1611635193 /usr/lib64/libm-2.17.so
7f25f15d4000-7f25f15d5000 r--p 00100000 08:02 1611635193 /usr/lib64/libm-2.17.so
7f25f15d5000-7f25f15d6000 rw-p 00101000 08:02 1611635193 /usr/lib64/libm-2.17.so
7f25f15d6000-7f25f17a0000 r-xp 00000000 08:02 1079834147 /opt/GCC/9.4.0/lib64/libstdc++.so.6.0.28
7f25f17a0000-7f25f199f000 ---p 001ca000 08:02 1079834147 p 00000000 08:02 1241751835 /opt/GCC/9.4.0/lib64/libgcc_s.so.1
7ffb62c28000-7ffb62e27000 ---p 00017000 08:02 1241751835 /opt/GCC/9.4.0/lib64/libgcc_s.so.1
7ffb62e27000-7ffb62e28000 r--p 00016000 08:02 1241751835 /opt/GCC/9.4.0/lib64/libgcc_s.so.1
7ffb62e28000-7ffb62e29000 rw-p 00017000 08:02 1241751835 /opt/GCC/9.4.0/lib64/libgcc_s.so.1
7ffb62e29000-7ffb62f2a000 r-xp 00000000 08:02 1611635193 /usr/lib64/libm-2.17.so
7ffb62f2a000-7ffb63129000 ---p 00101000 08:02 1611635193 /usr/lib64/libm-2.17.so
7ffb63129000-7ffb6312a000 r--p 00100000 08:02 1611635193 /usr/lib64/libm-2.17.so
7ffb6312a000-7ffb6312b000 rw-p 00101000 08:02 1611635193 /usr/lib64/libm-2.17.so
7ffb6312b000-7ffb632f5000 r-xp 00000000 08:02 1079834147 /opt/GCC/9.4.0/lib64/libstdc++.so.6.0.28
7ffb632f5000-7ffb634f4000 ---p 001ca000 08:02 1079834147 /opt/GCC/9.4.0/lib64/libstdc++.so.6.0.28
7ffb634f4000-7ffb634ff000 r--p 001c9000 08:02 1079834147 /opt/GCC/9.4.0/lib64/libstdc++.so.6.0.28
7ffb634ff000-7ffb63502000 rw-p 001d4000 08:02 1079834147 /opt/GCC/9.4.0/lib64/libstdc++.so.6.0.28
7ffb63502000-7ffb63505000 rw-p 00000000 00:00 0
7ffb63505000-7ffb65dbc000 r-xp 00000000 08:02 6095621 /opt/MPICH/4.0.2/lib/libmpi.so.12.2.2
7ffb65dbc000-7ffb65fbc000 ---p 028b7000 08:02 6095621 /opt/MPICH/4.0.2/lib/libmpi.so.12.2.2
7ffb65fbc000-7ffb65fcb000 r--p 028b7000 08:02 6095621 /opt/MPICH/4.0.2/lib/libmpi.so.12.2.2
7ffb65fcb000-7ffb65fd6000 rw-p 028c6000 08:02 6095621 /opt/MPICH/4.0.2/lib/libmpi.so.12.2.2
7ffb65fd6000-7ffb660fa000 rw-p 00000000 00:00 0
7ffb660fa000-7ffb66118000 r-xp 00000000 08:02 6095630 /opt/MPICH/4.0.2/lib/libmpicxx.so.12.2.2
7ffb66118000-7ffb66318000 ---p 0001e000 08:02 6095630 /opt/MPICH/4.0.2/lib/libmpicxx.so.12.2.2
7ffb66318000-7ffb6631a000 r--p /opt/GCC/9.4.0/lib64/libstdc++.so.6.0.28
7f25f199f000-7f25f19aa000 r--p 001c9000 08:02 1079834147 /opt/GCC/9.4.0/lib64/libstdc++.so.6.0.28
7f25f19aa000-7f25f19ad000 rw-p 001d4000 08:02 1079834147 /opt/GCC/9.4.0/lib64/libstdc++.so.6.0.28
7f25f19ad000-7f25f19b0000 rw-p 00000000 00:00 0
7f25f19b0000-7f25f4267000 r-xp 00000000 08:02 6095621 /opt/MPICH/4.0.2/lib/libmpi.so.12.2.2
7f25f4267000-7f25f4467000 ---p 028b7000 08:02 6095621 /opt/MPICH/4.0.2/lib/libmpi.so.12.2.2
7f25f4467000-7f25f4476000 r--p 028b7000 08:02 6095621 /opt/MPICH/4.0.2/lib/libmpi.so.12.2.2
7f25f4476000-7f25f4481000 rw-p 028c6000 08:02 6095621 /opt/MPICH/4.0.2/lib/libmpi.so.12.2.2
7f25f4481000-7f25f45a5000 rw-p 00000000 00:00 0
7f25f45a5000-7f25f45c3000 r-xp 00000000 08:02 6095630 /opt/MPICH/4.0.2/lib/libmpicxx.so.12.2.2
7f25f45c3000-7f25f47c3000 ---p 0001e000 08:02 6095630 /opt/MPICH/4.0.2/lib/li0001e000 08:02 6095630 /opt/MPICH/4.0.2/lib/libmpicxx.so.12.2.2
7ffb6631a000-7ffb6631b000 rw-p 00020000 08:02 6095630 /opt/MPICH/4.0.2/lib/libmpicxx.so.12.2.2
7ffb6631b000-7ffb665a9000 r-xp 00000000 08:02 1079769982 /opt/GCC/9.4.0/lib64/libgfortran.so.5.0.0
7ffb665a9000-7ffb667a9000 ---p 0028e000 08:02 1079769982 /opt/GCC/9.4.0/lib64/libgfortran.so.5.0.0
7ffb667a9000-7ffb667aa000 r--p 0028e000 08:02 1079769982 /opt/GCC/9.4.0/lib64/libgfortran.so.5.0.0
7ffb667aa000-7ffb667ac000 rw-p 0028f000 08:02 1079769982 /opt/GCC/9.4.0/lib64/libgfortran.so.5.0.0
7ffb667ac000-7ffb67804000 r-xp 00000000 08:02 171220237 /opt/OpenBLAS/0.3.19/lib64/libopenblas.so.0.3
7ffb67804000-7ffb67a04000 ---p 01058000 08:02 171220237 /opt/OpenBLAS/0.3.19/lib64/libopenblas.so.0.3
7ffb67a04000-7ffb67a05000 r--p 01058000 08:02 171220237 /opt/OpenBLAS/0.3.19/lib64/libopenblas.so.0.3
7ffb67a050bmpicxx.so.12.2.2
7f25f47c3000-7f25f47c5000 r--p 0001e000 08:02 6095630 /opt/MPICH/4.0.2/lib/libmpicxx.so.12.2.2
7f25f47c5000-7f25f47c6000 rw-p 00020000 08:02 6095630 /opt/MPICH/4.0.2/lib/libmpicxx.so.12.2.2
7f25f47c6000-7f25f4a54000 r-xp 00000000 08:02 1079769982 /opt/GCC/9.4.0/lib64/libgfortran.so.5.0.0
7f25f4a54000-7f25f4c54000 ---p 0028e000 08:02 1079769982 /opt/GCC/9.4.0/lib64/libgfortran.so.5.0.0
7f25f4c54000-7f25f4c55000 r--p 0028e000 08:02 1079769982 /opt/GCC/9.4.0/lib64/libgfortran.so.5.0.0
7f25f4c55000-7f25f4c57000 rw-p 0028f000 08:02 1079769982 /opt/GCC/9.4.0/lib64/libgfortran.so.5.0.0
7f25f4c57000-7f25f5caf000 r-xp 00000000 08:02 171220237 /opt/OpenBLAS/0.3.19/lib64/libopenblas.so.0.3
7f25f5caf000-7f25f5eaf000 ---p 01058000 08:02 171220237 /opt/OpenBLAS/0.3.19/lib64/libopenblas.so.0.3
7f25f5eaf000-7f25f5eb0000 r--p 01058000 08:02 171220237 /opt/Op00-7ffb67a15000 rw-p 01059000 08:02 171220237 /opt/OpenBLAS/0.3.19/lib64/libopenblas.so.0.3
7ffb67a15000-7ffb67a63000 rw-p 00000000 00:00 0
7ffb67a63000-7ffb68151000 r-xp 00000000 08:02 537059108 /opt/LAPACK/3.10.0/liblapack.so.3.10.0
7ffb68151000-7ffb68350000 ---p 006ee000 08:02 537059108 /opt/LAPACK/3.10.0/liblapack.so.3.10.0
7ffb68350000-7ffb68351000 r--p 006ed000 08:02 537059108 /opt/LAPACK/3.10.0/liblapack.so.3.10.0
7ffb68351000-7ffb68355000 rw-p 006ee000 08:02 537059108 /opt/LAPACK/3.10.0/liblapack.so.3.10.0
7ffb68355000-7ffb68377000 r-xp 00000000 08:02 1611512475 /usr/lib64/ld-2.17.so
7ffb6837f000-7ffb68557000 rw-p 00000000 00:00 0
7ffb6856f000-7ffb68570000 rw-p 00000000 00:00 0
7ffb68570000-7ffb68571000 rw-s 00000000 00:13 48103094 /dev/shm/mpich_shar_tmpWnEFwo (deleted)
7ffb68571000-7ffb68572000 rw-s 00000000 00:13 48103093 /dev/shm/mpich_shar_tmpk0Zcht (deleted)enBLAS/0.3.19/lib64/libopenblas.so.0.3
7f25f5eb0000-7f25f5ec0000 rw-p 01059000 08:02 171220237 /opt/OpenBLAS/0.3.19/lib64/libopenblas.so.0.3
7f25f5ec0000-7f25f5f0e000 rw-p 00000000 00:00 0
7f25f5f0e000-7f25f65fc000 r-xp 00000000 08:02 537059108 /opt/LAPACK/3.10.0/liblapack.so.3.10.0
7f25f65fc000-7f25f67fb000 ---p 006ee000 08:02 537059108 /opt/LAPACK/3.10.0/liblapack.so.3.10.0
7f25f67fb000-7f25f67fc000 r--p 006ed000 08:02 537059108 /opt/LAPACK/3.10.0/liblapack.so.3.10.0
7f25f67fc000-7f25f6800000 rw-p 006ee000 08:02 537059108 /opt/LAPACK/3.10.0/liblapack.so.3.10.0
7f25f6800000-7f25f6822000 r-xp 00000000 08:02 1611512475 /usr/lib64/ld-2.17.so
7f25f682a000-7f25f6a02000 rw-p 00000000 00:00 0
7f25f6a1a000-7f25f6a1b000 rw-p 00000000 00:00 0
7f25f6a1b000-7f25f6a1c000 rw-s 00000000 00:13 48103094 /dev/shm/mpich_shar_tmpWnEFwo (deleted)
7f25f6a1c000-7f25f6a1d000 rw-s 00000000 00:13 48103093
7ffb68572000-7ffb68573000 rw-s 00000000 00:13 48103091 /dev/shm/mpich_shar_tmpcNToMC (deleted)
7ffb68573000-7ffb68576000 rw-p 00000000 00:00 0
7ffb68576000-7ffb68577000 r--p 00021000 08:02 1611512475 /usr/lib64/ld-2.17.so
7ffb68577000-7ffb68578000 rw-p 00022000 08:02 1611512475 /usr/lib64/ld-2.17.so
7ffb68578000-7ffb68579000 rw-p 00000000 00:00 0
7ffc1a41a000-7ffc1a43f000 rw-p 00000000 00:00 0 [stack]
7ffc1a476000-7ffc1a478000 r-xp 00000000 00:00 0 [vdso]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]
/dev/shm/mpich_shar_tmpk0Zcht (deleted)
7f25f6a1d000-7f25f6a1e000 rw-s 00000000 00:13 48103091 /dev/shm/mpich_shar_tmpcNToMC (deleted)
7f25f6a1e000-7f25f6a21000 rw-p 00000000 00:00 0
7f25f6a21000-7f25f6a22000 r--p 00021000 08:02 1611512475 /usr/lib64/ld-2.17.so
7f25f6a22000-7f25f6a23000 rw-p 00022000 08:02 1611512475 /usr/lib64/ld-2.17.so
7f25f6a23000-7f25f6a24000 rw-p 00000000 00:00 0
7ffc23f72000-7ffc23f96000 rw-p 00000000 00:00 0 [stack]
7ffc23fc3000-7ffc23fc5000 r-xp 00000000 00:00 0 [vdso]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]
===================================================================================
= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES
= PID 6751 RUNNING AT machinename01
= EXIT CODE: 9
= CLEANING UP REMAINING PROCESSES
= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES
===================================================================================
YOUR APPLICATION TERMINATED WITH THE EXIT STRING: Terminated (signal 15)
This typically refers to a problem with your application.
Please see the FAQ page for debugging suggestions
and the gdb runs forever and has to be forced to stop.
Second Edit:
By dumping core file I got 2 core dumpfile (I was running the program with 4 processes on the same machine).
Loading each of them into gdb and bt them I got
(gdb) bt
#0 0x00007fcb48901387 in raise () from /lib64/libc.so.6
#1 0x00007fcb48902a78 in abort () from /lib64/libc.so.6
#2 0x00007fcb48943f67 in __libc_message () from /lib64/libc.so.6
#3 0x00007fcb4894c329 in _int_free () from /lib64/libc.so.6
#4 0x00007fcb499369c7 in MPIR_Comm_delete_internal () from /opt/MPICH/4.0.2/lib/libmpi.so.12
#5 0x00007fcb4964bcf7 in PMPI_Comm_free () from /opt/MPICH/4.0.2/lib/libmpi.so.12
#6 0x0000000000405668 in Cblacs_gridexit ()
#7 0x00000000004027d6 in validate_pdlapiv () at test.cpp:91
#8 0x00000000004028d4 in main (argc=1, argv=0x7ffda5cc2258) at test.cpp:98
for both files.
So I finally get to the point that why would this becomes a question.
ipiv, as similar from LAPACK, refers to a sequence of indices, that the current index will exchange with. This means that [3,3,3] refers to (assume we start from 1)
which is different with the one I thought it would be.
The problem originated when I tried to permute the result of pdgeqpf, Column pivoting QR factorization, and the parameter ipiv confuses me.
And I went to LAPACK and found the correct name for it (IMHO) would be jpvt, where it exactly means, the k-th col stores the original jpvt[k]-th col.
So, for anyone coming and having this question, notice that ipiv is different from jpvt. (where ipiv usually comes from routines like Gaussian Elimination (solving linear algebra equations).