Memory allocation in IBM AIX fails using xlc compiler?

In my code is necessary to allocate several large arrays.

However, when I try to use the IBM xlc_r :

xlc_r -g -O -L. -qarch=pwr7 -qtune=pwr7 -lesslsmp -lm -qsmp -qthreaded -qmaxmem=-1 2.c

int main()
{
     int natom = 5000;
     while(1)
     {
        double *z =(double*) malloc(natom*natom*natom* sizeof(double));
        if(!z)
        {  
           printf("error memory vector z\n\n");
           exit(-1);
        }
     }
}

I receive either Killed an error for memory vector z.

This is the ulimit -a :

core file size          (blocks, -c) unlimited

data seg size           (kbytes, -d) unlimited
file size               (blocks, -f) unlimited
max memory size         (kbytes, -m) unlimited 
open files                      (-n) 102400   
pipe size            (512 bytes, -p) 64   
stack size              (kbytes, -s) unlimited   
cpu time               (seconds, -t) unlimited   
max user processes              (-u) 128  
virtual memory          (kbytes, -v) unlimited

Is there any flag necessary to allocate more memory?

I hit this issue on a POWER775 AIX machine a while back. You need to add the -q64 flag when compiling in order to allocate more than 2GiB of memory. Note that the comments by others about using int may be relevant to your code, which is why my example below uses size_t.

I recommend you run a simple test of allocation to investigate this. My tester that I wrote for this situation is below.

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

int main(int argc, char * argv[])
{
    size_t max = (argc>1) ? (size_t)atol(argv[1]) : ((size_t)256*1024)*((size_t)1024*1024);
    for (size_t n=1; n<=max; n*=2) {
        printf("attempting malloc of %zu bytes \n", n);
        fflush(0);
        void * ptr = malloc(n);
        if (ptr==NULL) {
            printf("malloc of %zu bytes failed \n", n);
            fflush(0);
            return 1;
        } else {
            printf("malloc of %zu bytes succeeded \n", n);
            fflush(0);
            memset(ptr,0,n);
            printf("memset of %zu bytes succeeded \n", n);
            fflush(0);
            free(ptr);
        }
    }
    return 0;
}

The associated build flags were as follows.

ifeq ($(TARGET),POWER7-AIX)
# Savba i.e. POWER775 AIX
    CC       = mpcc_r
    OPT      = -q64 -O3 -qhot -qsmp=omp -qtune=pwr7 -qarch=pwr7 -qstrict
    CFLAGS   = $(OPT) -qlanglvl=stdc99
endif