This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 4c35e2fe fix(parquet/compress): Gzip CompressBound (#886)
4c35e2fe is described below
commit 4c35e2fe1209912b29583ef84a68b2f6e3740a09
Author: Minh Vu <[email protected]>
AuthorDate: Thu Jul 2 21:08:04 2026 +0200
fix(parquet/compress): Gzip CompressBound (#886)
### What changed
`gzipCodec.CompressBound` now includes gzip framing overhead on top of
the existing deflate growth estimate. The bound also leaves room for the
stateless close block accepted by `EncodeLevel`.
The regression test checks empty, one-byte, small, and random
incompressible payloads across the gzip levels accepted by the codec.
### Why
The previous bound could be smaller than real gzip output for tiny
inputs because it did not account for gzip header and trailer bytes.
Callers using `CompressBound` to size destination buffers could
therefore under-allocate.
### Validation
- `go test ./parquet/compress`
- `go test -tags assert ./parquet/compress`
-
`PARQUET_TEST_DATA=/Users/hoangvu/Code/OSS/arrow-go/parquet-testing/data
PARQUET_TEST_BAD_DATA=/Users/hoangvu/Code/OSS/arrow-go/parquet-testing/bad_data
go test ./parquet/...`
---
parquet/compress/compress_test.go | 43 +++++++++++++++++++++++++++++++++++++++
parquet/compress/gzip.go | 9 +++++++-
2 files changed, 51 insertions(+), 1 deletion(-)
diff --git a/parquet/compress/compress_test.go
b/parquet/compress/compress_test.go
index c60c65d7..6b65f23b 100644
--- a/parquet/compress/compress_test.go
+++ b/parquet/compress/compress_test.go
@@ -24,6 +24,7 @@ import (
"testing"
"github.com/apache/arrow-go/v18/parquet/compress"
+ "github.com/klauspost/compress/gzip"
"github.com/klauspost/compress/zstd"
"github.com/stretchr/testify/assert"
)
@@ -91,6 +92,48 @@ func TestCompressDataOneShot(t *testing.T) {
}
}
+func TestGzipCompressBound(t *testing.T) {
+ codec, err := compress.GetCodec(compress.Codecs.Gzip)
+ assert.NoError(t, err)
+
+ payloads := []struct {
+ name string
+ data []byte
+ }{
+ {"empty", nil},
+ {"one byte", []byte{0xab}},
+ {"small", []byte("gzip")},
+ {"incompressible", makeRandomData(128)},
+ }
+
+ levels := []struct {
+ name string
+ level int
+ }{
+ {"default", gzip.DefaultCompression},
+ {"no compression", gzip.NoCompression},
+ {"best speed", gzip.BestSpeed},
+ {"best compression", gzip.BestCompression},
+ {"huffman only", gzip.HuffmanOnly},
+ {"stateless", gzip.StatelessCompression},
+ }
+
+ for _, payload := range payloads {
+ t.Run(payload.name, func(t *testing.T) {
+ bound := codec.CompressBound(int64(len(payload.data)))
+ for _, level := range levels {
+ t.Run(level.name, func(t *testing.T) {
+ compressed :=
codec.EncodeLevel(make([]byte, 0, bound), payload.data, level.level)
+ assert.LessOrEqual(t,
int64(len(compressed)), bound)
+
+ uncompressed := codec.Decode(nil,
compressed)
+ assert.Equal(t, payload.data,
append([]byte(nil), uncompressed...))
+ })
+ }
+ })
+ }
+}
+
func TestCompressReaderWriter(t *testing.T) {
tests := []struct {
c compress.Compression
diff --git a/parquet/compress/gzip.go b/parquet/compress/gzip.go
index 4b43f8e9..562b5f4e 100644
--- a/parquet/compress/gzip.go
+++ b/parquet/compress/gzip.go
@@ -26,6 +26,12 @@ import (
type gzipCodec struct{}
+const (
+ gzipHeaderSize = 10
+ gzipTrailerSize = 8
+ gzipStatelessCloseBlockSize = 5
+)
+
func (gzipCodec) NewReader(r io.Reader) io.ReadCloser {
ret, err := gzip.NewReader(r)
if err != nil {
@@ -81,7 +87,8 @@ func (g gzipCodec) Encode(dst, src []byte) []byte {
}
func (gzipCodec) CompressBound(len int64) int64 {
- return len + ((len + 7) >> 3) + ((len + 63) >> 6) + 5
+ return len + ((len + 7) >> 3) + ((len + 63) >> 6) + 5 +
+ gzipHeaderSize + gzipTrailerSize + gzipStatelessCloseBlockSize
}
func (gzipCodec) NewWriter(w io.Writer) io.WriteCloser {
