I am trying to debug a piece of code that reads binary files I created. It seems that when the number of elements to read is higher than a certain treshold, the script I wrote fails.
The file contains is structured as follow: for each time t it has the time t then N particles struct (each containing some info about that particle, position, speed and so on, probably not relevant).
File is generated with
int update_binary_file(const char *name, particle *P, parameters par, double *time)
{
FILE *file;
file = fopen(name, "a+");
if (!file)
{
return 1;
}
fwrite(time, sizeof(double), 1, file);
fwrite(P, sizeof(particle), par.N, file);
fclose(file);
return 0;
}
Where particle is a struct defined like this:
typedef struct
{
double x, y, z; // position
double vx, vy, vz; // velocity
double wx, wy, wz; // acceleration
int next_inbox;
} particle;
Parameters is another struct but is not relevant here, I use it just to pass the par.N value (number of particle in the system).
Finally I read the file as follow:
void read_binaryfile(parameters par, char *filename, double *x, double *y, double *z, double *vx, double *vy, double *vz, double *wx, double *wy, double *wz, double *t)
{
int dump;
FILE *file;
file = fopen(filename, "rb");
if (!file)
{
printf("ERROR: invalid file %s\n", filename);
return;
}
else
{
for (int line = 0; line < par.Nsnap; line++)
{
if (fread(&t[line], 1, sizeof(double), file) != sizeof(double))
break;
printf("%f\n", t[line]);
for(int n = 0; n < par.N; n++){
int lin_index = linear_index(n, line);
if (fread(&x[lin_index], 1, sizeof(double), file) != sizeof(double)){
printf("ERROR reading file!\n");
break;
}
if (fread(&y[lin_index], 1, sizeof(double), file) != sizeof(double))
{
printf("ERROR reading file!\n");
break;
}
if (fread(&z[lin_index], 1, sizeof(double), file) != sizeof(double))
{
printf("ERROR reading file!\n");
break;
}
if (fread(&vx[lin_index], 1, sizeof(double), file) != sizeof(double))
{
printf("ERROR reading file!\n");
break;
}
if (fread(&vy[lin_index], 1, sizeof(double), file) != sizeof(double))
{
printf("ERROR reading file!\n");
break;
}
if (fread(&vz[lin_index], 1, sizeof(double), file) != sizeof(double))
{
printf("ERROR reading file!\n");
break;
}
if (fread(&wx[lin_index], 1, sizeof(double), file) != sizeof(double))
{
printf("ERROR reading file!\n");
break;
}
if (fread(&wy[lin_index], 1, sizeof(double), file) != sizeof(double))
{
printf("ERROR reading file!\n");
break;
}
if (fread(&wz[lin_index], 1, sizeof(double), file) != sizeof(double))
{
printf("ERROR reading file!\n");
break;
}
if (fread(&dump, 1, sizeof(int), file) != sizeof(int))
{
printf("ERROR reading file!\n");
break;
}
if (fread(&dump, 1, sizeof(int), file) != sizeof(int))
{
printf("ERROR reading file!\n");
break;
}
}
}
}
fclose(file);
}
It basically reads all the particle struct values one by one, instead of reading the struct itself (because is more practical). The script works well for small files and I can read them with Python. When the total number of elements in the file is higher, I get segmentation fault error. More specifically in gdb:
Program received signal SIGSEGV, Segmentation fault.
__memmove_avx_unaligned_erms () at ../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:418
418 ../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S: No such file or directory.
(gdb) where
#0 __memmove_avx_unaligned_erms () at ../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:418
#1 0x00007ffff788b423 in __GI__IO_file_xsgetn (fp=0x55555556cea0, data=<optimized out>, n=8) at ./libio/fileops.c:1295
#2 0x00007ffff787fba9 in __GI__IO_fread (buf=0x7ffdf7ad6af8, size=1, count=8, fp=0x55555556cea0) at ./libio/iofread.c:38
#3 0x00005555555559ae in read_binaryfile (par=...,
filename=0x7fffffffe127 "snap/m0.68_J18.00_g80.00_n1.00_T7.60_r1.00_v2.00/rho4.0_L4_N256_dt0.010_correlations_ICrand_dtsave0.05_t10.00.bin", x=0x7ffff7e37010, y=0x7ffff7b9d010, z=0x7ffff7b3a010, vx=0x7ffff7ad7010, vy=0x7ffff7a74010,
vz=0x7ffff779d010, wx=0x7ffff773a010, wy=0x7ffff76d7010, wz=0x7ffff7674010, t=0x55555556beb0)
at spacetime_correlations.c:151
#4 0x000055555555602a in main (argc=2, argv=0x7fffffffdd28) at spacetime_correlations.c:226
All the arrays x,y,z,vx,vy,vz,wx,wy,wz are defined as:
double *x = (double *)calloc(par.N * par.Nsnap, sizeof(double));
and it seems to me that I never exceed their maximum dimension while assigning them their values read from the file with fread(&vz[lin_index], etc...).
I do not know where the problem comes from, usually Segmentation fault means a problem in the memory but here the indices seems all good to me.
UPDATE 1. Minimal code to produce output file called N100.bin.
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
typedef struct
{
double x, y, z; // position
double vx, vy, vz; // velocity
double wx, wy, wz; // acceleration
int next_inbox;
} particle;
int update_binary_file(const char *name, particle *P, int N, double *time)
{
FILE *file;
file = fopen(name, "a+");
if (!file)
{
return 1;
}
fwrite(time, sizeof(double), 1, file);
fwrite(P, sizeof(particle), N, file);
fclose(file);
return 0;
}
int main(int argc, char *argv[])
{
int N = 100;
particle *P = (particle *)calloc(N, sizeof(particle));
int Nsteps = 1000;
for (int i = 0; i < N; i++)
{
P[i].x = 1;
P[i].y = 1;
P[i].z = 1;
P[i].vx = 1;
P[i].vy = 1;
P[i].vz = 1;
P[i].wx = 1;
P[i].wy = 1;
P[i].wz = 1;
P[i].next_inbox = 1;
}
double time;
for (int i = 0; i < Nsteps; i++)
{
time = i*0.1;
update_binary_file("N100.bin", P, N, &time);
}
}
Minimal code to read the N100.bin file generated.
// gcc spacetime_correlations.c -o spacetime_correlations -lm -O3
#include <stdio.h>
#include <stdlib.h>
#define linear_index(n, t) ((n) + (t) * par.N )
typedef struct
{
int N, Nsnap, L;
} parameters;
typedef struct
{
double x, y, z; // position
double vx, vy, vz; // velocity
double wx, wy, wz; // acceleration
int next_inbox;
} particle;
void read_binaryfile(parameters par, char *filename, double *x, double *y, double *z, double *vx, double *vy, double *vz, double *wx, double *wy, double *wz, double *t)
{
int dump;
FILE *file;
file = fopen(filename, "rb");
if (!file)
{
printf("ERROR: invalid file %s\n", filename);
return;
}
else
{
for (int line = 0; line < par.Nsnap; line++)
{
if (fread(&t[line], 1, sizeof(double), file) != sizeof(double))
break;
printf("%f\n", t[line]);
for(int n = 0; n < par.N; n++){
int lin_index = linear_index(n, line);
if (fread(&x[lin_index], 1, sizeof(double), file) != sizeof(double)){
printf("ERROR reading file!\n");
break;
}
if (fread(&y[lin_index], 1, sizeof(double), file) != sizeof(double))
{
printf("ERROR reading file!\n");
break;
}
if (fread(&z[lin_index], 1, sizeof(double), file) != sizeof(double))
{
printf("ERROR reading file!\n");
break;
}
if (fread(&vx[lin_index], 1, sizeof(double), file) != sizeof(double))
{
printf("ERROR reading file!\n");
break;
}
if (fread(&vy[lin_index], 1, sizeof(double), file) != sizeof(double))
{
printf("ERROR reading file!\n");
break;
}
if (fread(&vz[lin_index], 1, sizeof(double), file) != sizeof(double))
{
printf("ERROR reading file!\n");
break;
}
if (fread(&wx[lin_index], 1, sizeof(double), file) != sizeof(double))
{
printf("ERROR reading file!\n");
break;
}
if (fread(&wy[lin_index], 1, sizeof(double), file) != sizeof(double))
{
printf("ERROR reading file!\n");
break;
}
if (fread(&wz[lin_index], 1, sizeof(double), file) != sizeof(double))
{
printf("ERROR reading file!\n");
break;
}
if (fread(&dump, 1, sizeof(int), file) != sizeof(int))
{
printf("ERROR reading file!\n");
break;
}
if (fread(&dump, 1, sizeof(int), file) != sizeof(int))
{
printf("ERROR reading file!\n");
break;
}
}
}
}
fclose(file);
}
int main(int argc, char **argv){
// Provide filename from command line
char *filename = argv[1], *dump;
parameters par;
double *time = (double *)calloc(par.Nsnap, sizeof(double));
double *C_t = (double *)calloc(par.Nsnap, sizeof(double));
double *Vx = (double *)calloc(par.Nsnap, sizeof(double));
double *Vy = (double *)calloc(par.Nsnap, sizeof(double));
double *Vz = (double *)calloc(par.Nsnap, sizeof(double));
double *x = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *y = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *z = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *vx = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *vy = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *vz = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *wx = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *wy = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *wz = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *delta_vx = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *delta_vy = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *delta_vz = (double *)calloc(par.N * par.Nsnap, sizeof(double));
// Count number of "lines" (e.g. the number of particles arrays at different time)
int count = 0;
FILE *file;
file = fopen(filename, "rb");
while(fgetc(file) != EOF)
{
count++;
}
fclose(file);
// Get number of particles from file name
sscanf(filename,"N%d", &par.N);
par.Nsnap = count/(sizeof(particle)*par.N+sizeof(double));
printf("%d\n", par.N);
read_binaryfile(par, filename, x, y, z, vx, vy, vz, wx, wy, wz, time);
free(time);
free(C_t);
free(Vx);
free(Vy);
free(Vz);
free(x);
free(y);
free(z);
free(vx);
free(vy);
free(vz);
free(wx);
free(wy);
free(wz);
free(delta_vx);
free(delta_vy);
free(delta_vz);
return 1;
}
Notice that if I change in the first script Nsteps to a lower value, than I do not have the segmentation fault error while using the second script.
The buffer sizes depend on an uninitialized variable leading to undefined behavior.
parameters par;
double *time = (double *)calloc(par.Nsnap, sizeof(double));
double *C_t = (double *)calloc(par.Nsnap, sizeof(double));
double *Vx = (double *)calloc(par.Nsnap, sizeof(double));
double *Vy = (double *)calloc(par.Nsnap, sizeof(double));
double *Vz = (double *)calloc(par.Nsnap, sizeof(double));
double *x = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *y = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *z = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *vx = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *vy = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *vz = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *wx = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *wy = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *wz = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *delta_vx = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *delta_vy = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *delta_vz = (double *)calloc(par.N * par.Nsnap, sizeof(double));
Move those buffer allocations to a point where par.N and par.Nsnap are valid:
// Get number of particles from file name
sscanf(filename,"N%d", &par.N);
par.Nsnap = count/(sizeof(particle)*par.N+sizeof(double));
printf("%d\n", par.N);
double *time = (double *)calloc(par.Nsnap, sizeof(double));
double *C_t = (double *)calloc(par.Nsnap, sizeof(double));
double *Vx = (double *)calloc(par.Nsnap, sizeof(double));
double *Vy = (double *)calloc(par.Nsnap, sizeof(double));
double *Vz = (double *)calloc(par.Nsnap, sizeof(double));
double *x = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *y = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *z = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *vx = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *vy = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *vz = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *wx = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *wy = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *wz = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *delta_vx = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *delta_vy = (double *)calloc(par.N * par.Nsnap, sizeof(double));
double *delta_vz = (double *)calloc(par.N * par.Nsnap, sizeof(double));
read_binaryfile(par, filename, x, y, z, vx, vy, vz, wx, wy, wz, time);