Packet tables have regularly given me grief over the years I've used
them - always regarding the same subject: structure padding.
Previously I've believed it to be in other libraries, such as h5py's
handling of compound types - and there have been bugs found and fixed
regarding those - but I'm still getting some confusing behavior as of
late. So I've attached a test case where we can see something funky
is happening. Maybe it's only on packet tables though. The test case
uses C++11 (for brevity of the structure initialization), but
otherwise uses the C-api.
Build:
g++ -std=c++11 h5compound_test.cpp -lhdf5 -lhdf5_hl -o h5compound_test_unpacked
g++ -std=c++11 -DPACKED_STRUCT h5compound_test.cpp -lhdf5 -lhdf5_hl -o
h5compound_test_packed
Run both binaries - they will produce unpacked_test.h5 and
packed_test.h5 respectively.
With both 1.8.18 and 1.10, you will observe the
packed_test.h5/pt_testdata has some shifting (and thus corruption)
occurring at field 30 on my system (64-bit linux). However while I
was not able to reproduce in this test case, I have seen simliar
behavior in unpacked structures as well, but I was not able to
reproduce it here - it may take certain definitions to trigger the bug
- in this later case the size of the structure was correct but
somewhere along the lines inside a struct the data gets skewed up (so
f1 is always good, but field fx may not be. In contrast to the packed
case where the there is a cumulative skew such that record 2 doesn't
have field f1 correct.
Well it was about this late that I figured I'd check your git
repository logs for commits on H5PT.c - it looks like you fixed this
in git but haven't provided a release containing the fix yet. Wahoo -
imagine my enthusiasm constructing a test case for something already
fixed and trying to understand the source of this bug for the nth
time, especially yesterday. 14e308b2e6eada778818abf53949ceef0e7b2a34
. I saw this H5T_get_native_type call in the H5PT.c source but
wasn't sure it was the bug...
By the way - is the mailing list the preferred (or should it be the
preferred) way to report bugs given you guys expose your bitbucket git
repository to the public now?
-Jason
#include <hdf5.h>
#include <hdf5_hl.h>
#include <cstdint>
#include <vector>
#include <string>
#define _H5Tinsertp(field, type) H5Tinsert(hid, #field, offsetof(HDFType, field), type)
#define _H5Tinsertp_native(type_suffix, field) _H5Tinsertp(field, H5T_NATIVE_##type_suffix)
#define _H5Tinsertp_mat(field, rows, cols, type) \
do{ \
hsize_t dims[] = { rows, cols}; \
hid_t mat_hid = H5Tarray_create2(type, 2, dims); \
_H5Tinsertp(field, mat_hid); \
H5Tclose(mat_hid); \
}while(false) \
#define _H5Tinsertp_vec(field, N, type) \
do{ \
hsize_t dims[] = { N}; \
hid_t vec_hid = H5Tarray_create2(type, 1, dims); \
_H5Tinsertp(field, vec_hid); \
H5Tclose(vec_hid); \
}while(false) \
#define _H5Tinsertp_native_mat(type_suffix, field, rows, cols) _H5Tinsertp_mat(field, rows, cols, H5T_NATIVE_##type_suffix)
#define _H5Tinsertp_native_vec(type_suffix, field, N) _H5Tinsertp_vec(field, N, H5T_NATIVE_##type_suffix)
struct CompoundTest{
uint8_t f1 = 1;
uint8_t f2 = 2;
uint8_t f3 = 3;
uint8_t f4 = 4;
uint8_t f5 = 5;
uint8_t f6 = 6;
uint8_t f7_27[3][7] = {{7, 8, 9, 10, 11, 12, 13}, {14, 15, 16, 17, 18, 19, 20}, {21, 22, 23, 24, 25, 26, 27}};
uint8_t f28 = 28;
int16_t f29 = 29;
int f30 = 30;
int f31 = 31;
long f32 = 32;
uint64_t f33 = 33;
int64_t f34 = 34;
float f35 = 35;
double f36_44[3][3] = {{ 36, 37, 38}, {39, 40, 41}, {42, 43, 44}};
double f45_47[3] = { 45, 46, 47};
double f48_72[5][5] = {{48, 49, 50, 51, 52}, {53, 54, 55, 56, 57}, {58, 59, 60, 61, 62}, {63, 64, 65, 66, 67}, {68, 69, 70, 71, 72}};
double f73_77[5] = {73, 74, 75, 76, 77};
uint8_t f78 = 78;
CompoundTest(){}
static hid_t make_type(){
typedef CompoundTest HDFType;
hid_t hid = H5Tcreate(H5T_COMPOUND, sizeof(HDFType));
_H5Tinsertp_native(UINT8, f1);
_H5Tinsertp_native(UINT8, f2);
_H5Tinsertp_native(UINT8, f3);
_H5Tinsertp_native(UINT8, f4);
_H5Tinsertp_native(UINT8, f5);
_H5Tinsertp_native(UINT8, f6);
_H5Tinsertp_native_mat(UINT8, f7_27, 3, 7);
_H5Tinsertp_native(UINT8, f28);
_H5Tinsertp_native(INT16, f29);
_H5Tinsertp_native(INT, f30);
_H5Tinsertp_native(INT, f31);
_H5Tinsertp_native(LONG, f32);
_H5Tinsertp_native(UINT64, f33);
_H5Tinsertp_native(INT64, f34);
_H5Tinsertp_native(FLOAT, f35);
_H5Tinsertp_native_mat(DOUBLE, f36_44, 3, 3);
_H5Tinsertp_native_vec(DOUBLE, f45_47, 3);
_H5Tinsertp_native_mat(DOUBLE, f48_72, 5, 5);
_H5Tinsertp_native_vec(DOUBLE, f73_77, 5);
_H5Tinsertp_native(UINT8, f78);
return hid;
}
}
#ifdef PACKED_STRUCT
__attribute__((packed))
#endif
;
int main(int argc, char **argv){
#ifdef PACKED_STRUCT
const std::string filename = "packed_test.h5";
#else
const std::string filename = "unpacked_test.h5";
#endif
hid_t root_hid = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
hid_t compound_hid = CompoundTest::make_type();
H5Tcommit2(root_hid, "CompoundTest", compound_hid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
std::vector<CompoundTest> data(131);
{
hid_t compound_pt_hid = H5PTcreate(root_hid, "pt_testdata", compound_hid, 100, H5P_DEFAULT);
H5PTappend(compound_pt_hid, (hsize_t) data.size(), (void *) data.data());
H5PTclose(compound_pt_hid);
}
{
hsize_t dims[] = { (hsize_t) data.size() };
hid_t dataspace = H5Screate_simple(1, dims, NULL);
hid_t dataset = H5Dcreate2(root_hid, "d_testdata", compound_hid, dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Sclose(dataspace);
H5Dwrite(dataset, compound_hid, H5S_ALL, H5S_ALL, H5P_DEFAULT, (void *) data.data());
H5Dclose(dataset);
}
H5Tclose(compound_hid);
H5Fclose(root_hid);
}
_______________________________________________
Hdf-forum is for HDF software users discussion.
[email protected]
http://lists.hdfgroup.org/mailman/listinfo/hdf-forum_lists.hdfgroup.org
Twitter: https://twitter.com/hdf5
