Dear Petsc Team! I had an issue when writing out DMPlex objects through hdf5.
This comes from a DMLabel that has only entries on non-local mesh points. The DMLabel write only includes local parts of the label and so leads to a zero sized write for the index set. This seems to be fine except that the hdf5 chunksize is set to zero which is not allowed. I added a minimal example to illustrate the error. It creates a 2D DMPlex in serial, distributes it, labels the nonlocal points in the mesh and dumps it via PetscObjectViewer to HDF5. Run with: make plex.h5 I also attached a quick fix to override the chunksize. Please let me know if you anything extra and also if this is expected behavior... I could certainly with the fact that DMLabel is not supposed to work this way. Many thanks, Fabian
From a8fb918b6f1ef49b8b14c5c492581ff84d484eb6 Mon Sep 17 00:00:00 2001 From: "Fabian.Jakub" <fab...@jakub.com> Date: Thu, 14 Feb 2019 13:21:28 +0100 Subject: [PATCH] fix hdf5 chunksizes of 0 chunksize must not be 0. H5Pset_chunk,(chunkspace, dim, chunkDims) will otherwise give an error. This happened for example when dumping a dmlabel inside a dmplex which has only entries on non-local points. --- src/vec/is/is/impls/general/general.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vec/is/is/impls/general/general.c b/src/vec/is/is/impls/general/general.c index c6930cf..476a149 100644 --- a/src/vec/is/is/impls/general/general.c +++ b/src/vec/is/is/impls/general/general.c @@ -264,7 +264,7 @@ static PetscErrorCode ISView_General_HDF5(IS is, PetscViewer viewer) ierr = PetscHDF5IntCast(N/bs,dims + dim);CHKERRQ(ierr); maxDims[dim] = dims[dim]; - chunkDims[dim] = dims[dim]; + chunkDims[dim] = PetscMax(1,dims[dim]); ++dim; if (bs >= 1) { dims[dim] = bs; -- 2.7.4
program main #include "petsc/finclude/petsc.h" use petsc implicit none PetscErrorCode :: ierr PetscInt, parameter :: petscint_dummy=0 integer, parameter :: pi=kind(petscint_dummy) type(tDM) :: dm, dmdist call PetscInitialize(PETSC_NULL_CHARACTER,ierr); CHKERRQ(ierr) call create_plex(PETSC_COMM_WORLD, dm) call PetscObjectViewFromOptions(dm, PETSC_NULL_VEC, "-show_serial_plex", ierr); CHKERRQ(ierr) call distribute_dmplex(dm, dmdist) call PetscObjectViewFromOptions(dmdist, PETSC_NULL_VEC, "-show_dist_plex", ierr); CHKERRQ(ierr) call label_non_local_points(dmdist) call PetscObjectViewFromOptions(dmdist, PETSC_NULL_VEC, "-show_labeled_plex", ierr); CHKERRQ(ierr) call DMDestroy(dmdist, ierr);CHKERRQ(ierr) call DMDestroy(dm, ierr);CHKERRQ(ierr) call PetscFinalize(ierr) contains subroutine create_plex(comm, dm) integer, intent(in) :: comm type(tDM), intent(out) :: dm integer :: myid PetscInt :: i, k, Nfaces, Nedges, Nverts, chartsize call mpi_comm_rank(comm, myid, ierr);CHKERRQ(ierr) call DMPlexCreate(comm, dm, ierr);CHKERRQ(ierr) call PetscObjectSetName(dm, 'testplex', ierr);CHKERRQ(ierr) call DMSetDimension(dm, 2_pi, ierr);CHKERRQ(ierr) if(myid.eq.0) then ! 16____11_____17_____12___18 ! | /|\ | ! | / | \ | ! | / | \ 3 | ! | 0 / | \ | ! | / | \ | ! 8 7 | 9 10 ! | / | \ | ! | / 8 \ | ! | / 1 | 2 \ | ! | / | \ | ! |/ | \| ! / | \ ! 13-----4----14-----5-----15 Nfaces = 4 Nedges = 9 Nverts = 6 chartsize = 19 else Nfaces = 0 Nedges = 0 Nverts = 0 chartsize = 0 endif call DMPlexSetChart(dm, 0_pi, chartsize, ierr); CHKERRQ(ierr) ! Preallocation k=0 ! cell has 3 edges do i = 1, Nfaces call DMPlexSetConeSize(dm, k, 3_pi, ierr); CHKERRQ(ierr) k = k+1 enddo ! Edges have 2 vertices do i = 1, Nedges call DMPlexSetConeSize(dm, k, 2_pi, ierr); CHKERRQ(ierr) k = k+1 enddo call DMSetUp(dm, ierr); CHKERRQ(ierr) ! Allocate space for cones if(myid.eq.0) then ! Setup Connections call DMPlexSetCone(dm, 0_pi, [6_pi, 7_pi,11_pi], ierr); CHKERRQ(ierr) call DMPlexSetCone(dm, 1_pi, [4_pi, 8_pi, 7_pi], ierr); CHKERRQ(ierr) call DMPlexSetCone(dm, 2_pi, [5_pi, 9_pi, 8_pi], ierr); CHKERRQ(ierr) call DMPlexSetCone(dm, 3_pi, [9_pi,10_pi,12_pi], ierr); CHKERRQ(ierr) call DMPlexSetCone(dm, 4_pi, [13_pi,14_pi], ierr); CHKERRQ(ierr) call DMPlexSetCone(dm, 5_pi, [14_pi,15_pi], ierr); CHKERRQ(ierr) call DMPlexSetCone(dm, 6_pi, [13_pi,16_pi], ierr); CHKERRQ(ierr) call DMPlexSetCone(dm, 7_pi, [13_pi,17_pi], ierr); CHKERRQ(ierr) call DMPlexSetCone(dm, 8_pi, [14_pi,17_pi], ierr); CHKERRQ(ierr) call DMPlexSetCone(dm, 9_pi, [15_pi,17_pi], ierr); CHKERRQ(ierr) call DMPlexSetCone(dm, 10_pi, [15_pi,18_pi], ierr); CHKERRQ(ierr) call DMPlexSetCone(dm, 11_pi, [16_pi,17_pi], ierr); CHKERRQ(ierr) call DMPlexSetCone(dm, 12_pi, [17_pi,18_pi], ierr); CHKERRQ(ierr) endif call DMPlexSymmetrize(dm, ierr); CHKERRQ(ierr) call DMPlexStratify(dm, ierr); CHKERRQ(ierr) call set_coords(comm) end subroutine subroutine set_coords(comm) integer, intent(in) :: comm integer :: myid, ierr PetscScalar, pointer :: coords(:) Vec :: coordinates PetscInt :: i, dimEmbed, coordSize, vStart, vEnd, pStart, pEnd, voff PetscSection :: coordSection call mpi_comm_rank(comm, myid, ierr); CHKERRQ(ierr) call DMGetCoordinateDim(dm, dimEmbed, ierr); CHKERRQ(ierr) call DMGetCoordinateSection(dm, coordSection, ierr); CHKERRQ(ierr) call PetscSectionSetNumFields(coordSection, 1, ierr); CHKERRQ(ierr) call PetscSectionSetUp(coordSection, ierr); CHKERRQ(ierr) call PetscSectionSetFieldComponents(coordSection, 0, dimEmbed, ierr); CHKERRQ(ierr) call DMPlexGetChart(dm, pStart, pEnd, ierr); CHKERRQ(ierr) call DMPlexGetDepthStratum (dm, 0, vStart, vEnd, ierr); CHKERRQ(ierr) ! vertices call PetscSectionSetChart(coordSection, pStart, pEnd, ierr);CHKERRQ(ierr) do i = vStart, vEnd-1 call PetscSectionSetDof(coordSection, i, dimEmbed, ierr); CHKERRQ(ierr) call PetscSectionSetFieldDof(coordSection, i, 0, dimEmbed, ierr); CHKERRQ(ierr) enddo call PetscSectionSetUp(coordSection, ierr); CHKERRQ(ierr) call PetscSectionGetStorageSize(coordSection, coordSize, ierr); CHKERRQ(ierr) call VecCreate(PETSC_COMM_SELF, coordinates, ierr); CHKERRQ(ierr) call VecSetSizes(coordinates, coordSize, PETSC_DETERMINE, ierr);CHKERRQ(ierr) call VecSetBlockSize(coordinates, dimEmbed, ierr);CHKERRQ(ierr) call VecSetType(coordinates, VECSTANDARD, ierr);CHKERRQ(ierr) call PetscObjectSetName(coordinates, "coordinates", ierr); CHKERRQ(ierr) call VecGetArrayF90(coordinates, coords, ierr); CHKERRQ(ierr) if(myid.eq.0) then i =13; call PetscSectionGetOffset(coordSection, i, voff, ierr); coords(voff+1:voff+2) = [0,0] i =14; call PetscSectionGetOffset(coordSection, i, voff, ierr); coords(voff+1:voff+2) = [1,0] i =15; call PetscSectionGetOffset(coordSection, i, voff, ierr); coords(voff+1:voff+2) = [2,0] i =16; call PetscSectionGetOffset(coordSection, i, voff, ierr); coords(voff+1:voff+2) = [0,1] i =17; call PetscSectionGetOffset(coordSection, i, voff, ierr); coords(voff+1:voff+2) = [1,1] i =18; call PetscSectionGetOffset(coordSection, i, voff, ierr); coords(voff+1:voff+2) = [2,1] endif call VecRestoreArrayF90(coordinates, coords, ierr); CHKERRQ(ierr) call DMSetCoordinatesLocal(dm, coordinates, ierr);CHKERRQ(ierr) call PetscObjectViewFromOptions(coordinates, PETSC_NULL_VEC, "-show_plex_coordinates", ierr); CHKERRQ(ierr) call VecDestroy(coordinates, ierr);CHKERRQ(ierr) end subroutine subroutine distribute_dmplex(dm, dmdist) type(tDM), intent(in) :: dm type(tDM), intent(out) :: dmdist type(tPetscSF) :: msf call DMPlexSetAdjacencyUseCone(dm, PETSC_TRUE, ierr); CHKERRQ(ierr) call DMPlexSetAdjacencyUseClosure(dm, PETSC_FALSE, ierr); CHKERRQ(ierr) msf = PETSC_NULL_SF call DMPlexDistribute(dm, 0_pi, msf, dmdist, ierr); CHKERRQ(ierr) if(dmdist.eq.PETSC_NULL_DM) then print *,'Did not distribute the mesh' call DMClone(dm, dmdist, ierr); CHKERRQ(ierr) endif end subroutine subroutine label_non_local_points(dm) type(tDM), intent(inout) :: dm integer :: comm, myid, ierr integer(pi) :: i type(tDMLabel) :: label_nonlocal type(tPetscSF) :: sf integer(pi) :: nleaves, nroots integer(pi), pointer :: myidx(:) ! list of my indices that we do not own type(PetscSFNode), pointer :: remote(:) ! rank and remote idx of those points call PetscObjectGetComm(dm, comm, ierr); CHKERRQ(ierr) call mpi_comm_rank(comm, myid, ierr); CHKERRQ(ierr) call DMCreateLabel(dm, "nonlocal", ierr); CHKERRQ(ierr) call DMGetLabel(dm, "nonlocal", label_nonlocal, ierr); CHKERRQ(ierr) call DMLabelSetDefaultValue(label_nonlocal, int(myid, pi), ierr); CHKERRQ(ierr) call DMGetPointSF(dm, sf, ierr); CHKERRQ(ierr) call PetscSFGetGraph(sf, nroots, nleaves, myidx, remote, ierr); CHKERRQ(ierr) do i = 1, nleaves call DMLabelSetValue(label_nonlocal, myidx(i), remote(i)%rank, ierr); CHKERRQ(ierr) enddo myidx=>NULL(); remote=>NULL() end subroutine end program
include ${PETSC_DIR}/lib/petsc/conf/variables include ${PETSC_DIR}/lib/petsc/conf/rules plex.xmf:: plex.h5 ${PETSC_DIR}/lib/petsc/bin/petsc_gen_xdmf.py plex.h5 plex.h5:: hdf5_issue_nonlocal_only_labels ./hdf5_issue_nonlocal_only_labels -show_labeled_plex ::ascii_info_detail ./hdf5_issue_nonlocal_only_labels -show_labeled_plex hdf5:plex.h5 hdf5_issue_nonlocal_only_labels:: hdf5_issue_nonlocal_only_labels.F90 ${PETSC_FCOMPILE} -c hdf5_issue_nonlocal_only_labels.F90 ${FLINKER} hdf5_issue_nonlocal_only_labels.o -o hdf5_issue_nonlocal_only_labels ${PETSC_LIB} clean:: rm -rf *.o plex.h5 plex.xmf hdf5_issue_nonlocal_only_labels
HDF5-DIAG: Error detected in HDF5 (1.8.18) MPI-process 0: #000: H5Pdcpl.c line 870 in H5Pset_chunk(): all chunk dimensions must be positive major: Invalid arguments to routine minor: Out of range [0]PETSC ERROR: --------------------- Error Message -------------------------------------------------------------- [0]PETSC ERROR: Error in external library [0]PETSC ERROR: Error in HDF5 call H5Pset_chunk() Status -1 [0]PETSC ERROR: See http://www.mcs.anl.gov/petsc/documentation/faq.html for trouble shooting. [0]PETSC ERROR: Petsc Development GIT revision: v3.10.3-1549-g9b78045 GIT Date: 2019-02-13 11:10:12 -0600 [0]PETSC ERROR: plex_prism on a debug_gcc named met-ws-970r17 by Fabian.Jakub Thu Feb 14 14:39:13 2019 [0]PETSC ERROR: Configure options --with-cc=/usr/bin/mpicc --with-fc=/usr/bin/mpif90 --with-cxx=/usr/bin/mpicxx --with-fortran --with-fortran-interfaces --with-shared-libraries=1 --download-hdf5 --download-netcdf=netcdf-c-4.6.2.tar.gz --download-zlib --download-szlib --download-openblas --download-metis --download-parmetis --download-ptscotch --no-download-petsc4py --download-ml --download-hypre --with-debugging=1 COPTFLAGS="-O2 -g" FOPTFLAGS="-O2 -g" [0]PETSC ERROR: #1 ISView_General_HDF5() line 288 in /software/meteo/xenial/x86_64/petsc/master/src/vec/is/is/impls/general/general.c [0]PETSC ERROR: #2 ISView_General() line 513 in /software/meteo/xenial/x86_64/petsc/master/src/vec/is/is/impls/general/general.c [0]PETSC ERROR: #3 ISView() line 1015 in /software/meteo/xenial/x86_64/petsc/master/src/vec/is/is/interface/index.c [0]PETSC ERROR: #4 DMPlexWriteLabels_HDF5_Static() line 668 in /software/meteo/xenial/x86_64/petsc/master/src/dm/impls/plex/plexhdf5.c [0]PETSC ERROR: #5 DMPlexView_HDF5_Internal() line 692 in /software/meteo/xenial/x86_64/petsc/master/src/dm/impls/plex/plexhdf5.c [0]PETSC ERROR: #6 DMView_Plex() line 1103 in /software/meteo/xenial/x86_64/petsc/master/src/dm/impls/plex/plex.c [0]PETSC ERROR: #7 DMView() line 904 in /software/meteo/xenial/x86_64/petsc/master/src/dm/interface/dm.c [0]PETSC ERROR: #8 PetscObjectView() line 100 in /software/meteo/xenial/x86_64/petsc/master/src/sys/objects/destroy.c [0]PETSC ERROR: #9 PetscObjectViewFromOptions() line 136 in /software/meteo/xenial/x86_64/petsc/master/src/sys/objects/destroy.c [0]PETSC ERROR: #10 User provided function() line 0 in User file HDF5-DIAG: Error detected in HDF5 (1.8.18) MPI-process 1: #000: H5Pdcpl.c line 870 in H5Pset_chunk(): all chunk dimensions must be positive major: Invalid arguments to routine minor: Out of range [1]PETSC ERROR: --------------------- Error Message -------------------------------------------------------------- [1]PETSC ERROR: Error in external library [1]PETSC ERROR: Error in HDF5 call H5Pset_chunk() Status -1 [1]PETSC ERROR: See http://www.mcs.anl.gov/petsc/documentation/faq.html for trouble shooting. [1]PETSC ERROR: Petsc Development GIT revision: v3.10.3-1549-g9b78045 GIT Date: 2019-02-13 11:10:12 -0600 [1]PETSC ERROR: plex_prism on a debug_gcc named met-ws-970r17 by Fabian.Jakub Thu Feb 14 14:39:13 2019 [1]PETSC ERROR: Configure options --with-cc=/usr/bin/mpicc --with-fc=/usr/bin/mpif90 --with-cxx=/usr/bin/mpicxx --with-fortran --with-fortran-interfaces --with-shared-libraries=1 --download-hdf5 --download-netcdf=netcdf-c-4.6.2.tar.gz --download-zlib --download-szlib --download-openblas --download-metis --download-parmetis --download-ptscotch --no-download-petsc4py --download-ml --download-hypre --with-debugging=1 COPTFLAGS="-O2 -g" FOPTFLAGS="-O2 -g" [1]PETSC ERROR: #1 ISView_General_HDF5() line 288 in /software/meteo/xenial/x86_64/petsc/master/src/vec/is/is/impls/general/general.c [1]PETSC ERROR: #2 ISView_General() line 513 in /software/meteo/xenial/x86_64/petsc/master/src/vec/is/is/impls/general/general.c [1]PETSC ERROR: #3 ISView() line 1015 in /software/meteo/xenial/x86_64/petsc/master/src/vec/is/is/interface/index.c [1]PETSC ERROR: #4 DMPlexWriteLabels_HDF5_Static() line 668 in /software/meteo/xenial/x86_64/petsc/master/src/dm/impls/plex/plexhdf5.c [1]PETSC ERROR: #5 DMPlexView_HDF5_Internal() line 692 in /software/meteo/xenial/x86_64/petsc/master/src/dm/impls/plex/plexhdf5.c [1]PETSC ERROR: #6 DMView_Plex() line 1103 in /software/meteo/xenial/x86_64/petsc/master/src/dm/impls/plex/plex.c [1]PETSC ERROR: #7 DMView() line 904 in /software/meteo/xenial/x86_64/petsc/master/src/dm/interface/dm.c [1]PETSC ERROR: #8 PetscObjectView() line 100 in /software/meteo/xenial/x86_64/petsc/master/src/sys/objects/destroy.c [1]PETSC ERROR: #9 PetscObjectViewFromOptions() line 136 in /software/meteo/xenial/x86_64/petsc/master/src/sys/objects/destroy.c [1]PETSC ERROR: #10 User provided function() line 0 in User file --------------------------------------------------------------------------
signature.asc
Description: OpenPGP digital signature