Hi there we (a colleague and I) stumbled upon an interesting behavior of the HDF5 library (we are using 1.8.11 on Debian Wheezy). We try to write a single 2D array of data to an HDF5 dataset with an absolutely stupid chunking scheme (see the test() function in the attached source file). As a result the library allocates quite a lot of memory (around 3 GByte). What surprised us is that this memory is not freed even after closing the file. Moreover, it does not grow when calling the test() several times as can be seen in the output of the attached program
./test
Startup ...
RSS - 6.880000e+02 kB
Shared Memory - 5.200000e+02 kB
Private Memory - 1.680000e+02 kB
After first write ...
RSS - 2.916884e+06 kB
Shared Memory - 2.160000e+03 kB
Private Memory - 2.914724e+06 kB
After second write ...
RSS - 2.921896e+06 kB
Shared Memory - 2.160000e+03 kB
Private Memory - 2.919736e+06 kB
Obviously this is not a resource leak in the classical sense. My
suspicion is that the memory is occupied by some persistent cache.
Which leads me to my question: is there a possibility to free this
memory?
regards
Eugen
--
---------------------------------------
DI. Dr. Eugen Wintersberger
FS-EC
DESY
Notkestr. 85
D-22607 Hamburg
Germany
E-Mail: [email protected]
Telefon: +49-40-8998-1917
---------------------------------------
#include <stdlib.h>
#include <stdio.h>
#include <hdf5.h>
#include <time.h>
#include <unistd.h>
#define NX 1000
#define NY 2048
#define NTOT NX*NY
#define NCX 1
#define NCY 4
void test()
{
hsize_t shape[2] = {NX,NY}; //shape of the dataset
hsize_t chunk[2] = {NCX,NCY}; //stupid chunk shape for the dataset
//create a new file
hid_t fid = H5Fcreate("test2.h5",H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT);
//create type and data space
hid_t type_id = H5Tcopy(H5T_NATIVE_USHORT);
hid_t space_id = H5Screate_simple(2,shape,shape);
//setup the creation property list for the dataset
hid_t cplist = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_layout(cplist,H5D_CHUNKED);
H5Pset_chunk(cplist,2,chunk);
//create the dataset
hid_t dset_id = H5Dcreate(fid,"test",type_id,space_id,H5P_DEFAULT,cplist,
H5P_DEFAULT);
//generate data
unsigned short *data = malloc(sizeof(unsigned short)*NTOT);
for(size_t i=0;i<NTOT;++i) data[i] = 123;
//write data to disk
H5Dwrite(dset_id,type_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,data);
//close everything and release memory
H5Dclose(dset_id);
H5Tclose(type_id);
H5Pclose(cplist);
H5Sclose(space_id);
H5Fflush(fid,H5F_SCOPE_GLOBAL);
H5Fclose(fid);
free(data);
}
void check()
{
//read the programs statm file from the proc file system
FILE *f = fopen("/proc/self/statm","r");
size_t size,resident,share;
fscanf(f,"%lu %lu %lu",&size,&resident,&share);
fclose(f);
long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024;
double rss = resident * page_size_kb;
fprintf(stdout,"RSS - %e kB\n",rss);
double shared_mem = share * page_size_kb;
fprintf(stdout,"Shared Memory - %e kB\n",shared_mem);
fprintf(stdout,"Private Memory - %e kB\n",rss - shared_mem);
}
int main(int argc,char **argv)
{
fprintf(stdout,"Startup ...\n");
check();
test();
fprintf(stdout,"\nAfter first write ...\n");
check();
test();
fprintf(stdout,"\nAfter second write ...\n");
check();
return 0;
}
signature.asc
Description: This is a digitally signed message part
_______________________________________________ Hdf-forum is for HDF software users discussion. [email protected] http://mail.lists.hdfgroup.org/mailman/listinfo/hdf-forum_lists.hdfgroup.org
