I am trying to allocate an array of structs, with each struct also containing dynamic arrays. They will later be communicated via MPI_Sendrecv:
struct cell {
double a, b, c, *aa, *bb;
} *Send_l, *Send_r;
I want Send_l and Send_r to have count number of elements, the arrays aa and bb should contain sAS number of elements. This is all done after MPI_Init.
void allocateForSendRecv(int count) {
int sAS = 5;
int iter = 0;
Send_l = (struct cell *)malloc(count * (sizeof(struct cell)));
for (iter = 0; iter < count; iter++) {
Send_l[iter].aa = (double *)malloc((sAS - 1) * sizeof(double));
Send_l[iter].bb = (double *)malloc((sAS - 1) * sizeof(double));
}
//sAS-1, as sizeof(struct cell) already contains a single (double) for aa and bb.
Send_r = (struct cell *)malloc(count * (sizeof(struct cell)));
for (iter = 0; iter < count; iter++) {
Send_r[iter].aa = (double *)malloc((sAS - 1) * sizeof(double));
Send_r[iter].bb = (double *)malloc((sAS - 1) * sizeof(double));
}
}
With this, I can freely allocate, fill and deallocate, however when I call the following, my results diverge from my reference (using all stack arrays).
MPI_Sendrecv(&(Send_r[0]), count, ..., &(Send_l[0]), count, ...)
I haven't found the exact reason, but posts about similar issues made me assume its due to my non-contiguous memory allocation. Ive tried to solve the problem by using a single malloc call, only to get a segmentation fault when I fill my arrays aa and bb:
Send_l = malloc(count * (sizeof(*Send_l)) + count *(sizeof(*Send_l) + 2 * (sAS - 1) * sizeof(double)));
Send_r = malloc(count * (sizeof(*Send_r)) + count *(sizeof(*Send_r) + 2 * (sAS - 1) * sizeof(double)));
I have reused some code to allocate 2D arrays and applied it to this struct problem, but haven't been able to make it work. Am I right in assuming that, with a functioning single malloc call and therefore contiguous memory allocation, my MPI_Sendrecv would work fine? Alternatively, would using MPI_Type_create_struct solve my non-contiguous memory problem?
Minimal example (without MPI) of segmentation fault. Using allocateSendRecv, everything is fine. But the single alloc in allocateInOneSendRecv gives me issues.
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
struct cell {
double a, b, c, *aa, *bb;
} *Send_l, *Send_r;
void allocateSendRecv(int count, int sAS);
void fillSendRecv(int count, int sAS);
void freeSendRecv(int count);
void printSendRecv(int count, int sAS);
void allocateInOneSendRecv(int count, int sAS);
int main(int argc, char *argv[])
{
const int count = 2;
const int sAS = 9;
allocateSendRecv(count, sAS);
//allocateInOneSendRecv(count, sAS);
fillSendRecv(count, sAS);
printSendRecv(count, sAS);
freeSendRecv(count);
return 0;
}
void allocateSendRecv(int count, int sAS) {
int iter = 0;
printf("Allocating!\n");
Send_r = (struct cell *)malloc(count * (sizeof(struct cell)));
for (iter = 0; iter < count; iter++) {
Send_r[iter].aa = (double *)malloc((sAS - 1) * sizeof(double));
Send_r[iter].bb = (double *)malloc((sAS - 1) * sizeof(double));
}
Send_l = (struct cell *)malloc(count * (sizeof(struct cell)));
for (iter = 0; iter < count; iter++) {
Send_l[iter].aa = (double *)malloc((sAS - 1) * sizeof(double));
Send_l[iter].bb = (double *)malloc((sAS - 1) * sizeof(double));
}
}
void allocateInOneSendRecv(int count, int sAS) {
printf("Allocating!\n");
Send_l = malloc(count * (sizeof(*Send_l)) + count *(sizeof(*Send_l) + 2 * (sAS - 1) * sizeof(double)));
Send_r = malloc(count * (sizeof(*Send_r)) + count *(sizeof(*Send_r) + 2 * (sAS - 1) * sizeof(double)));
}
void freeSendRecv(int count) {
int iter = 0;
printf("Deallocating!\n");
free(Send_r);
free(Send_l);
}
void fillSendRecv(int count, int sAS) {
int iter = 0;
int iter2= 0;
double dummyDouble = 5.0;
printf("Filling!\n");
for (iter = 0; iter < count; iter++) {
Send_l[iter].a = dummyDouble;
Send_l[iter].b = dummyDouble;
Send_l[iter].c = dummyDouble;
for (iter2 = 0; iter2 < sAS; iter2++) {
Send_l[iter].aa[iter2] = dummyDouble;
Send_l[iter].bb[iter2] = dummyDouble;
}
dummyDouble++;
Send_r[iter].a = dummyDouble;
Send_r[iter].b = dummyDouble;
Send_r[iter].c = dummyDouble;
for (iter2 = 0; iter2 < sAS; iter2++) {
Send_r[iter].aa[iter2] = dummyDouble;
Send_r[iter].bb[iter2] = dummyDouble;
}
dummyDouble++;
}
}
void printSendRecv(int count, int sAS) {
int iter = 0;
printf("Printing!\n");
for (iter = 0; iter < count; iter++) {
printf("%f \n", Send_l[iter].a);
printf("%f \n", Send_l[iter].b);
printf("%f \n", Send_l[iter].c);
printf("%f \n", Send_l[iter].aa[sAS - 1]);
printf("%f \n\n", Send_l[iter].bb[sAS - 1]);
printf("%f \n", Send_r[iter].a);
printf("%f \n", Send_r[iter].b);
printf("%f \n", Send_r[iter].c);
printf("%f \n", Send_r[iter].aa[sAS - 1]);
printf("%f \n\n", Send_r[iter].bb[sAS - 1]);
}
}
Your current problem is that you can only pass the start address of Send_l (resp. Send_r). From that point, all memory has to be contiguous and you must know its total size and give it later to MPI_SendRecv.
But after allocation, you must ensure that aa and bb members are correctly initialized to point inside the allocated bloc of memory.
A possible code could be:
void allocateSendRecv(int count, int subCount) {
int iter;
// total size of each struct
size_t sz = sizeof(struct cell) + 2 * subCount * sizeof(double);
// one single contiguous allocation
Send_r = malloc(count * sz); // nota: never cast malloc in C language!
// per each cell make aa and bb point into the allocated memory
for (iter = 0; iter < count; iter++) {
Send_r[iter].aa = ((double*)(Send_r + count)) + 2 * subCount * iter;
Send_r[iter].bb = Send_r[iter].aa + subCount;
}
// id. for Send_l
Send_l = malloc(count * sz);
for (iter = 0; iter < count; iter++) {
Send_l[iter].aa = ((double*)(Send_l + count)) + 2 * subCount * iter;
Send_l[iter].bb = Send_l[iter].aa + subCount;
}
}
Here I have first the array of cell structures and then 1 aa array and 1 bb array per structure in that order.
That is enough to get rid of the segmentation fault...