I finally found a way to reproduce the errors I sent in the first post
(pasted code below).
The problem is that the collective data transfer does not work after using
H5Dset_extent. This behaviour does not even depend on H5Sselect_elements,
as I initially thought.
Anybody knows if there is something wrong with my use of H5Dset_extent?
Anybody knows why I does not get any performance improvement when using
H5FD_MPIO_COLLECTIVE?
Thank you for your help
Fred
#include <mpi.h>
#include <hdf5.h>
#include <iostream>
#include <sstream>
#include <vector>
#include <algorithm>
#include <ctime>
using namespace std;
int main (int argc, char* argv[])
{
int mpi_provided, sz, rk;
MPI_Init_thread( &argc, &argv, MPI_THREAD_FUNNELED, &mpi_provided );
MPI_Comm_size( MPI_COMM_WORLD, &sz );
MPI_Comm_rank( MPI_COMM_WORLD, &rk );
hsize_t local_nElements = 10000;
hsize_t global_nElements = sz*local_nElements;
vector<int> vec;
vec.resize(1);
hsize_t dims[2] = {0, global_nElements};
if (rk==0) {
// create file
hid_t fid = H5Fcreate( "test.h5", H5F_ACC_TRUNC, H5P_DEFAULT,
H5P_DEFAULT);
// prepare file space
hsize_t max_dims[2] = {H5S_UNLIMITED, global_nElements}; // not
really needed but for future use
hid_t file_space = H5Screate_simple(2, dims, max_dims);
// prepare dataset
hid_t plist = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_layout(plist, H5D_CHUNKED);
hsize_t chunk_dims[2] = {1, global_nElements};
H5Pset_chunk(plist, 2, chunk_dims);
// create dataset
hid_t did = H5Dcreate(fid, "Id", H5T_NATIVE_INT, file_space,
H5P_DEFAULT, plist, H5P_DEFAULT);
H5Dclose(did);
H5Pclose(plist);
H5Sclose(file_space);
H5Fclose( fid );
// make a randomized vector
vec.resize(global_nElements);
for(int i=0; i<global_nElements; i++) vec[i]=i;
random_shuffle(vec.begin(), vec.end());
}
// Scatter the randomized vector
int * v = &vec[0];
int * data = new int[local_nElements];
MPI_Scatter( v, local_nElements, MPI_INT, data, local_nElements,
MPI_INT, 0, MPI_COMM_WORLD);
hsize_t * coords = new hsize_t[local_nElements*2];
for(int i=0; i<local_nElements; i++) {
coords[i*2 ] = 0;
coords[i*2+1] = data[i];
}
clock_t start = clock();
// define MPI file access
hid_t file_access = H5Pcreate(H5P_FILE_ACCESS);
H5Pset_fapl_mpio( file_access, MPI_COMM_WORLD, MPI_INFO_NULL );
// define MPI transfer mode
hid_t transfer = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio( transfer, H5FD_MPIO_COLLECTIVE);
// Open the file
hid_t fid = H5Fopen( "test.h5", H5F_ACC_RDWR, file_access);
// Open the existing dataset
hid_t did = H5Dopen( fid, "Id", H5P_DEFAULT );
dims[0] ++;
H5Dset_extent(did, dims);
// Get the file space
hid_t file_space = H5Dget_space(did);
// Define the memory space for this proc
hsize_t count[2] = {1, (hsize_t) local_nElements};
hid_t mem_space = H5Screate_simple(2, count, NULL);
// Select the elements for this particular proc (the `coords` array has
been created before)
H5Sselect_elements( file_space, H5S_SELECT_SET, local_nElements, coords
);
// Write the previously generated `data` array
H5Dwrite( did, H5T_NATIVE_INT, mem_space , file_space , transfer, data
);
H5D_mpio_actual_io_mode_t actual_io_mode;
H5Pget_mpio_actual_io_mode(transfer, &actual_io_mode);
cout << "rank " << rk << " - actual_io_mode " << actual_io_mode << endl;
// Close stuff
H5Sclose(file_space);
H5Dclose(did);
H5Fclose( fid );
double duration = ( clock() - start ) / (double) CLOCKS_PER_SEC;
cout << "rank " << rk << ", duration = " << duration << endl;
MPI_Finalize();
}
2015-09-04 0:21 GMT+02:00 Frederic Perez <[email protected]>:
> In response to Mohamad Chaarawi, here is a full code that compiles, but it
> surprisingly does not raise the errors that I obtained with my full code. I
> will investigate into that. In the meantime, I found that the short version
> below does not have improved performance when I remove the line that sets
> H5FD_MPIO_COLLECTIVE.
>
> Cheers
> Fred
>
>
> #include <mpi.h>
> #include <hdf5.h>
> #include <iostream>
> #include <sstream>
> #include <vector>
> #include <algorithm>
> #include <ctime>
>
> using namespace std;
>
> int main (int argc, char* argv[])
> {
> int mpi_provided, sz, rk;
> MPI_Init_thread( &argc, &argv, MPI_THREAD_FUNNELED, &mpi_provided );
> MPI_Comm_size( MPI_COMM_WORLD, &sz );
> MPI_Comm_rank( MPI_COMM_WORLD, &rk );
>
> hsize_t local_nElements = 10000;
> hsize_t global_nElements = sz*local_nElements;
>
> vector<int> vec;
> vec.resize(1);
>
> if (rk==0) {
> // create file
> hid_t fid = H5Fcreate( "test.h5", H5F_ACC_TRUNC, H5P_DEFAULT,
> H5P_DEFAULT);
> // prepare file space
> hsize_t dims[2] = {1, global_nElements};
> hsize_t max_dims[2] = {H5S_UNLIMITED, global_nElements}; // not
> really needed but for future use
> hid_t file_space = H5Screate_simple(2, dims, max_dims);
> // prepare dataset
> hid_t plist = H5Pcreate(H5P_DATASET_CREATE);
> H5Pset_layout(plist, H5D_CHUNKED);
> hsize_t chunk_dims[2] = {1, global_nElements};
> H5Pset_chunk(plist, 2, chunk_dims);
> // create dataset
> hid_t did = H5Dcreate(fid, "Id", H5T_NATIVE_INT, file_space,
> H5P_DEFAULT, plist, H5P_DEFAULT);
>
> H5Dclose(did);
> H5Pclose(plist);
> H5Sclose(file_space);
> H5Fclose( fid );
>
>
> // make a randomized vector
> vec.resize(global_nElements);
> for(int i=0; i<global_nElements; i++) vec[i]=i;
> random_shuffle(vec.begin(), vec.end());
> }
>
> // Scatter the randomized vector
> int * v = &vec[0];
> int * data = new int[local_nElements];
> MPI_Scatter( v, local_nElements, MPI_INT, data, local_nElements,
> MPI_INT, 0, MPI_COMM_WORLD);
>
> hsize_t * coords = new hsize_t[local_nElements*2];
> for(int i=0; i<local_nElements; i++) {
> coords[i*2 ] = 0;
> coords[i*2+1] = data[i];
> }
>
> clock_t start = clock();
>
> // define MPI file access
> hid_t file_access = H5Pcreate(H5P_FILE_ACCESS);
> H5Pset_fapl_mpio( file_access, MPI_COMM_WORLD, MPI_INFO_NULL );
> // define MPI transfer mode
> hid_t transfer = H5Pcreate(H5P_DATASET_XFER);
> H5Pset_dxpl_mpio( transfer, H5FD_MPIO_COLLECTIVE);
> // Open the file
> hid_t fid = H5Fopen( "test.h5", H5F_ACC_RDWR, file_access);
> // Open the existing dataset
> hid_t did = H5Dopen( fid, "Id", H5P_DEFAULT );
> // Get the file space
> hid_t file_space = H5Dget_space(did);
> // Define the memory space for this proc
> hsize_t count[2] = {1, (hsize_t) local_nElements};
> hid_t mem_space = H5Screate_simple(2, count, NULL);
> // Select the elements for this particular proc (the `coords` array
> has been created before)
> H5Sselect_elements( file_space, H5S_SELECT_SET, local_nElements,
> coords );
> // Write the previously generated `data` array
> H5Dwrite( did, H5T_NATIVE_INT, mem_space , file_space , transfer, data
> );
> // Close stuff
> H5Sclose(file_space);
> H5Dclose(did);
> H5Fclose( fid );
>
> double duration = ( clock() - start ) / (double) CLOCKS_PER_SEC;
> cout << rk << " " << duration << endl;
>
> MPI_Finalize();
> }
>
_______________________________________________
Hdf-forum is for HDF software users discussion.
[email protected]
http://lists.hdfgroup.org/mailman/listinfo/hdf-forum_lists.hdfgroup.org
Twitter: https://twitter.com/hdf5
