Hello HDF5 group!
Zstandard is a real-time compression algorithm, providing high
compression ratios. It offers a very wide range of compression / speed
trade-off, while being backed by a very fast decoder. Zstandard library
is provided as open source software using a BSD license.
www.zstd.net
In attachment you can find an implementation of Zstd HDF5 gilter
plug-in. My tests confirm the good properties of Zstd compression, even
on small chunks.
I'd like the filter binary format to be registered in
https://support.hdfgroup.org/services/contributions.html#filters
Is anything else needed to get the filter ID registered? I think the
filter code is trivial, but if explicit license is needed, please let me
know.
Best wishes,
Andrey Paramonov
--
This message has been scanned for viruses and
dangerous content by MailScanner, and is
believed to be clean.
#include <stdlib.h>
#include "zstd_h5plugin.h"
#include "zstd.h"
#define ZSTD_FILTER 48001
DLL_EXPORT size_t zstd_filter(unsigned int flags, size_t cd_nelmts,
const unsigned int cd_values[], size_t nbytes,
size_t *buf_size, void **buf)
{
void *outbuf = NULL; /* Pointer to new output buffer */
void *inbuf = NULL; /* Pointer to input buffer */
inbuf = *buf;
size_t ret_value;
size_t origSize = nbytes; /* Number of bytes for output
(compressed) buffer */
if (flags & H5Z_FLAG_REVERSE)
{
size_t decompSize = ZSTD_getDecompressedSize(*buf, origSize);
if (NULL == (outbuf = malloc(decompSize)))
goto error;
decompSize = ZSTD_decompress(outbuf, decompSize, inbuf,
origSize);
free(*buf);
*buf = outbuf;
outbuf = NULL;
ret_value = (size_t)decompSize;
}
else
{
int aggression = 0;
if (cd_nelmts > 0)
aggression = (int)cd_values[0];
if (aggression > 22)
aggression = 22;
size_t compSize = ZSTD_compressBound(origSize);
if (NULL == (outbuf = malloc(compSize)))
goto error;
compSize = ZSTD_compress(outbuf, compSize, inbuf, origSize,
aggression);
free(*buf);
*buf = outbuf;
*buf_size = compSize;
outbuf = NULL;
ret_value = compSize;
}
if (outbuf != NULL)
free(outbuf);
return ret_value;
error:
return 0;
}
const H5Z_class_t zstd_H5Filter =
{
H5Z_CLASS_T_VERS,
(H5Z_filter_t)(ZSTD_FILTER),
1, 1,
"Zstandard compression: http://www.zstd.net";,
NULL, NULL,
(H5Z_func_t)(zstd_filter)
};
DLL_EXPORT H5PL_type_t H5PLget_plugin_type(void)
{
return H5PL_TYPE_FILTER;
}
DLL_EXPORT const void* H5PLget_plugin_info(void)
{
return &zstd_H5Filter;
}
#include "hdf5.h"
#define DLL_EXPORT __declspec(dllexport)
#ifdef __cplusplus
extern "C" {
#endif
DLL_EXPORT size_t zstd_filter(unsigned int flags, size_t cd_nelmts,
const unsigned int cd_values[], size_t nbytes,
size_t *buf_size, void **buf);
DLL_EXPORT H5PL_type_t H5PLget_plugin_type(void);
DLL_EXPORT const void* H5PLget_plugin_info(void);
#ifdef __cplusplus
}
#endif
_______________________________________________
Hdf-forum is for HDF software users discussion.
[email protected]
http://lists.hdfgroup.org/mailman/listinfo/hdf-forum_lists.hdfgroup.org
Twitter: https://twitter.com/hdf5
