This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new bb3f9800 fix(parquet/metadata): compare bloom filter caps in bits
(#996)
bb3f9800 is described below
commit bb3f980016d3cabee782cb4768568db666876ed1
Author: Minh Vu <[email protected]>
AuthorDate: Mon Jul 27 19:08:10 2026 +0200
fix(parquet/metadata): compare bloom filter caps in bits (#996)
The bloom filter sizing path computes `m` in bits, but the cap check
compared it to the byte limit shifted the wrong way. In practice that
can send a moderate-sized filter straight to the 128 MiB cap.
This switches the comparison to bits and adds a regression that covers
both the capped case and a case that should stay in the low-megabyte
range.
Tests: `go test ./parquet/metadata`
---
parquet/metadata/bloom_filter.go | 2 +-
parquet/metadata/bloom_filter_test.go | 4 +++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/parquet/metadata/bloom_filter.go b/parquet/metadata/bloom_filter.go
index 72371aab..cd2c79ff 100644
--- a/parquet/metadata/bloom_filter.go
+++ b/parquet/metadata/bloom_filter.go
@@ -69,7 +69,7 @@ func optimalNumBits(ndv uint32, fpp float64) uint32 {
numBits uint32
)
- if m < 0 || m > maximumBloomFilterBytes>>3 {
+ if m < 0 || m > maximumBloomFilterBytes<<3 {
numBits = maximumBloomFilterBytes << 3
} else {
numBits = uint32(m)
diff --git a/parquet/metadata/bloom_filter_test.go
b/parquet/metadata/bloom_filter_test.go
index 1eb43287..955b7324 100644
--- a/parquet/metadata/bloom_filter_test.go
+++ b/parquet/metadata/bloom_filter_test.go
@@ -122,9 +122,11 @@ func TestNewBloomFilter(t *testing.T) {
}{
{1, 0.09, 0, 0},
// cap at maximumBloomFilterBytes
- {1024 * 1024 * 128, 0.9, maximumBloomFilterBytes + 1,
maximumBloomFilterBytes},
+ {1 << 30, 0.9, maximumBloomFilterBytes + 1,
maximumBloomFilterBytes},
// round to power of 2
{1024 * 1024, 0.01, maximumBloomFilterBytes, 1 << 21},
+ // keep values between 2MB and the maximum from being capped
+ {2_000_000, 0.01, maximumBloomFilterBytes, 4 << 20},
}
for _, tt := range tests {
