Thank you for the help. I tried your second solution and it works. Concerning the performance, I did not do thorough tests, but i actually got better results going towards higher number of procs (up to 32). It looks like it depends a lot on the particular case, however.
Fred Le 4 sept. 2015 5:03 PM, "Mohamad Chaarawi" <[email protected]> a écrit : > I haven’t checked the performance yet, but the error you are running into > is a known issue that you can fix by either: > > 1) Create the file on process 0 using the MPI-IO VFD with > MPI_COMM_SELF: > > hid_t file_access = H5Pcreate(H5P_FILE_ACCESS); > > H5Pset_fapl_mpio( file_access, MPI_COMM_SELF, MPI_INFO_NULL ); > > 2) Or Create the dataset on process 0 with space allocation set to > H5D_ALLOC_TIME_EARLY: > > hid_t plist = H5Pcreate(H5P_DATASET_CREATE); > > H5Pset_layout(plist, H5D_CHUNKED); > > H5Pset_alloc_time(plist, H5D_ALLOC_TIME_EARLY) > > > > This should fix the error you are seeing.. The problem is that when you > create the file on process 0 with the sec2 driver, the space allocation for > the dataset is set to LATE by default, however parallel access requires it > to be set to EARLY. We were debating whether this is a bug or just a > documentation issue that we should make more clear for dataset creation. I > believe we decided for the latter, but I couldn’t find where this was > documented, so I’ll follow up on that too internally. > > > > I’ll look into performance once I get some time.. > > > > Thanks > > Mohamad > > > > *From:* Hdf-forum [mailto:[email protected]] *On > Behalf Of *Frederic Perez > *Sent:* Friday, September 04, 2015 9:33 AM > *To:* HDF Users Discussion List > *Subject:* Re: [Hdf-forum] Slow or buggy using H5Sselect_elements > > > > I finally found a way to reproduce the errors I sent in the first post > (pasted code below). > > The problem is that the collective data transfer does not work after using > H5Dset_extent. This behaviour does not even depend on H5Sselect_elements, > as I initially thought. > > Anybody knows if there is something wrong with my use of H5Dset_extent? > > Anybody knows why I does not get any performance improvement when using > H5FD_MPIO_COLLECTIVE? > > Thank you for your help > > Fred > > > > > #include <mpi.h> > #include <hdf5.h> > #include <iostream> > #include <sstream> > #include <vector> > #include <algorithm> > #include <ctime> > > using namespace std; > > int main (int argc, char* argv[]) > { > int mpi_provided, sz, rk; > MPI_Init_thread( &argc, &argv, MPI_THREAD_FUNNELED, &mpi_provided ); > MPI_Comm_size( MPI_COMM_WORLD, &sz ); > MPI_Comm_rank( MPI_COMM_WORLD, &rk ); > > hsize_t local_nElements = 10000; > hsize_t global_nElements = sz*local_nElements; > > vector<int> vec; > vec.resize(1); > > hsize_t dims[2] = {0, global_nElements}; > > if (rk==0) { > // create file > hid_t fid = H5Fcreate( "test.h5", H5F_ACC_TRUNC, H5P_DEFAULT, > H5P_DEFAULT); > // prepare file space > hsize_t max_dims[2] = {H5S_UNLIMITED, global_nElements}; // not > really needed but for future use > hid_t file_space = H5Screate_simple(2, dims, max_dims); > // prepare dataset > hid_t plist = H5Pcreate(H5P_DATASET_CREATE); > H5Pset_layout(plist, H5D_CHUNKED); > hsize_t chunk_dims[2] = {1, global_nElements}; > H5Pset_chunk(plist, 2, chunk_dims); > // create dataset > hid_t did = H5Dcreate(fid, "Id", H5T_NATIVE_INT, file_space, > H5P_DEFAULT, plist, H5P_DEFAULT); > > H5Dclose(did); > H5Pclose(plist); > H5Sclose(file_space); > H5Fclose( fid ); > > > // make a randomized vector > vec.resize(global_nElements); > for(int i=0; i<global_nElements; i++) vec[i]=i; > random_shuffle(vec.begin(), vec.end()); > } > > // Scatter the randomized vector > int * v = &vec[0]; > int * data = new int[local_nElements]; > MPI_Scatter( v, local_nElements, MPI_INT, data, local_nElements, > MPI_INT, 0, MPI_COMM_WORLD); > > hsize_t * coords = new hsize_t[local_nElements*2]; > for(int i=0; i<local_nElements; i++) { > coords[i*2 ] = 0; > coords[i*2+1] = data[i]; > } > > clock_t start = clock(); > > // define MPI file access > hid_t file_access = H5Pcreate(H5P_FILE_ACCESS); > H5Pset_fapl_mpio( file_access, MPI_COMM_WORLD, MPI_INFO_NULL ); > // define MPI transfer mode > hid_t transfer = H5Pcreate(H5P_DATASET_XFER); > H5Pset_dxpl_mpio( transfer, H5FD_MPIO_COLLECTIVE); > // Open the file > hid_t fid = H5Fopen( "test.h5", H5F_ACC_RDWR, file_access); > // Open the existing dataset > hid_t did = H5Dopen( fid, "Id", H5P_DEFAULT ); > dims[0] ++; > H5Dset_extent(did, dims); > // Get the file space > hid_t file_space = H5Dget_space(did); > // Define the memory space for this proc > hsize_t count[2] = {1, (hsize_t) local_nElements}; > hid_t mem_space = H5Screate_simple(2, count, NULL); > // Select the elements for this particular proc (the `coords` array > has been created before) > H5Sselect_elements( file_space, H5S_SELECT_SET, local_nElements, > coords ); > // Write the previously generated `data` array > H5Dwrite( did, H5T_NATIVE_INT, mem_space , file_space , transfer, data > ); > H5D_mpio_actual_io_mode_t actual_io_mode; > H5Pget_mpio_actual_io_mode(transfer, &actual_io_mode); > cout << "rank " << rk << " - actual_io_mode " << actual_io_mode << > endl; > // Close stuff > H5Sclose(file_space); > H5Dclose(did); > H5Fclose( fid ); > > double duration = ( clock() - start ) / (double) CLOCKS_PER_SEC; > cout << "rank " << rk << ", duration = " << duration << endl; > > MPI_Finalize(); > } > > > > 2015-09-04 0:21 GMT+02:00 Frederic Perez <[email protected]>: > > In response to Mohamad Chaarawi, here is a full code that compiles, but it > surprisingly does not raise the errors that I obtained with my full code. I > will investigate into that. In the meantime, I found that the short version > below does not have improved performance when I remove the line that sets > H5FD_MPIO_COLLECTIVE. > > Cheers > > Fred > > > #include <mpi.h> > #include <hdf5.h> > #include <iostream> > #include <sstream> > #include <vector> > #include <algorithm> > #include <ctime> > > using namespace std; > > int main (int argc, char* argv[]) > { > int mpi_provided, sz, rk; > MPI_Init_thread( &argc, &argv, MPI_THREAD_FUNNELED, &mpi_provided ); > MPI_Comm_size( MPI_COMM_WORLD, &sz ); > MPI_Comm_rank( MPI_COMM_WORLD, &rk ); > > hsize_t local_nElements = 10000; > hsize_t global_nElements = sz*local_nElements; > > vector<int> vec; > vec.resize(1); > > if (rk==0) { > // create file > hid_t fid = H5Fcreate( "test.h5", H5F_ACC_TRUNC, H5P_DEFAULT, > H5P_DEFAULT); > // prepare file space > hsize_t dims[2] = {1, global_nElements}; > hsize_t max_dims[2] = {H5S_UNLIMITED, global_nElements}; // not > really needed but for future use > hid_t file_space = H5Screate_simple(2, dims, max_dims); > // prepare dataset > hid_t plist = H5Pcreate(H5P_DATASET_CREATE); > H5Pset_layout(plist, H5D_CHUNKED); > hsize_t chunk_dims[2] = {1, global_nElements}; > H5Pset_chunk(plist, 2, chunk_dims); > // create dataset > hid_t did = H5Dcreate(fid, "Id", H5T_NATIVE_INT, file_space, > H5P_DEFAULT, plist, H5P_DEFAULT); > > H5Dclose(did); > H5Pclose(plist); > H5Sclose(file_space); > H5Fclose( fid ); > > > // make a randomized vector > vec.resize(global_nElements); > for(int i=0; i<global_nElements; i++) vec[i]=i; > random_shuffle(vec.begin(), vec.end()); > } > > // Scatter the randomized vector > int * v = &vec[0]; > int * data = new int[local_nElements]; > MPI_Scatter( v, local_nElements, MPI_INT, data, local_nElements, > MPI_INT, 0, MPI_COMM_WORLD); > > hsize_t * coords = new hsize_t[local_nElements*2]; > for(int i=0; i<local_nElements; i++) { > coords[i*2 ] = 0; > coords[i*2+1] = data[i]; > } > > clock_t start = clock(); > > // define MPI file access > hid_t file_access = H5Pcreate(H5P_FILE_ACCESS); > H5Pset_fapl_mpio( file_access, MPI_COMM_WORLD, MPI_INFO_NULL ); > // define MPI transfer mode > hid_t transfer = H5Pcreate(H5P_DATASET_XFER); > H5Pset_dxpl_mpio( transfer, H5FD_MPIO_COLLECTIVE); > // Open the file > hid_t fid = H5Fopen( "test.h5", H5F_ACC_RDWR, file_access); > // Open the existing dataset > hid_t did = H5Dopen( fid, "Id", H5P_DEFAULT ); > // Get the file space > hid_t file_space = H5Dget_space(did); > // Define the memory space for this proc > hsize_t count[2] = {1, (hsize_t) local_nElements}; > hid_t mem_space = H5Screate_simple(2, count, NULL); > // Select the elements for this particular proc (the `coords` array > has been created before) > H5Sselect_elements( file_space, H5S_SELECT_SET, local_nElements, > coords ); > // Write the previously generated `data` array > H5Dwrite( did, H5T_NATIVE_INT, mem_space , file_space , transfer, data > ); > // Close stuff > H5Sclose(file_space); > H5Dclose(did); > H5Fclose( fid ); > > double duration = ( clock() - start ) / (double) CLOCKS_PER_SEC; > cout << rk << " " << duration << endl; > > MPI_Finalize(); > } > > > > _______________________________________________ > Hdf-forum is for HDF software users discussion. > [email protected] > http://lists.hdfgroup.org/mailman/listinfo/hdf-forum_lists.hdfgroup.org > Twitter: https://twitter.com/hdf5 >
_______________________________________________ Hdf-forum is for HDF software users discussion. [email protected] http://lists.hdfgroup.org/mailman/listinfo/hdf-forum_lists.hdfgroup.org Twitter: https://twitter.com/hdf5
