This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new c0be13fc fix(parquet/metadata): preserve infinite floating-point
statistics (#1007)
c0be13fc is described below
commit c0be13fcf8e39579cf10f705c84b2a28d29eca38
Author: Minh Vu <[email protected]>
AuthorDate: Mon Jul 27 19:21:53 2026 +0200
fix(parquet/metadata): preserve infinite floating-point statistics (#1007)
### Rationale for this change
Floating-point statistics use the largest finite values as their initial
min/max bounds. Those are not identity values when a column contains
only positive or negative infinity, so the writer records a finite bound
that was not present in the data.
### What changes are included in this PR?
- Seed FLOAT, DOUBLE, and FLOAT16 statistics with positive and negative
infinity.
- Treat an inverted min/max pair as the empty or all-NaN sentinel.
- Cover all-positive-infinity and all-negative-infinity inputs for all
three types.
### Are these changes tested?
Yes. The focused statistics tests, including the existing NaN and
signed-zero coverage, pass.
---
parquet/metadata/statistics.go | 22 +++++++++++-----------
parquet/metadata/statistics_test.go | 36 ++++++++++++++++++++++++++++++++++++
2 files changed, 47 insertions(+), 11 deletions(-)
diff --git a/parquet/metadata/statistics.go b/parquet/metadata/statistics.go
index 9e568403..0aa9ef46 100644
--- a/parquet/metadata/statistics.go
+++ b/parquet/metadata/statistics.go
@@ -388,8 +388,8 @@ var (
defaultMaxInt96 parquet.Int96
defaultMaxUInt96 parquet.Int96
- defaultMinFloat16 parquet.FixedLenByteArray = float16.MaxNum.ToLEBytes()
- defaultMaxFloat16 parquet.FixedLenByteArray = float16.MinNum.ToLEBytes()
+ defaultMinFloat16 parquet.FixedLenByteArray = float16.Inf().ToLEBytes()
+ defaultMaxFloat16 parquet.FixedLenByteArray =
float16.Inf().Negate().ToLEBytes()
)
func init() {
@@ -432,10 +432,10 @@ func (Float16Statistics) defaultMax()
parquet.FixedLenByteArray {
return defaultMaxFloat16
}
-func (Float32Statistics) defaultMin() float32 {
return math.MaxFloat32 }
-func (Float32Statistics) defaultMax() float32 {
return -math.MaxFloat32 }
-func (Float64Statistics) defaultMin() float64 {
return math.MaxFloat64 }
-func (Float64Statistics) defaultMax() float64 {
return -math.MaxFloat64 }
+func (Float32Statistics) defaultMin() float32 {
return float32(math.Inf(1)) }
+func (Float32Statistics) defaultMax() float32 {
return float32(math.Inf(-1)) }
+func (Float64Statistics) defaultMin() float64 {
return math.Inf(1) }
+func (Float64Statistics) defaultMax() float64 {
return math.Inf(-1) }
func (ByteArrayStatistics) defaultMin() parquet.ByteArray {
return nil }
func (ByteArrayStatistics) defaultMax() parquet.ByteArray {
return nil }
func (FixedLenByteArrayStatistics) defaultMin() parquet.FixedLenByteArray {
return nil }
@@ -531,7 +531,7 @@ func (Float32Statistics) cleanStat(minMax
minmaxPairFloat32) *minmaxPairFloat32
return nil
}
- if minMax[0] == math.MaxFloat32 && minMax[1] == -math.MaxFloat32 {
+ if minMax[0] > minMax[1] {
return nil
}
@@ -552,7 +552,7 @@ func (Float64Statistics) cleanStat(minMax
minmaxPairFloat64) *minmaxPairFloat64
return nil
}
- if minMax[0] == math.MaxFloat64 && minMax[1] == -math.MaxFloat64 {
+ if minMax[0] > minMax[1] {
return nil
}
@@ -576,7 +576,7 @@ func (Float16Statistics) cleanStat(minMax
minmaxPairFloat16) *minmaxPairFloat16
return nil
}
- if min.Equal(float16.MaxNum) && max.Equal(float16.MinNum) {
+ if min.Greater(max) {
return nil
}
@@ -864,9 +864,9 @@ func (c *floatComparator[T]) defaultMin() T {
var z T
switch any(z).(type) {
case float32:
- return math.MaxFloat32
+ return T(math.Inf(1))
case float64:
- v := math.MaxFloat64
+ v := math.Inf(1)
return T(v)
}
panic("unreachable")
diff --git a/parquet/metadata/statistics_test.go
b/parquet/metadata/statistics_test.go
index 6809eeef..f7fcbda6 100644
--- a/parquet/metadata/statistics_test.go
+++ b/parquet/metadata/statistics_test.go
@@ -175,6 +175,42 @@ func TestCheckNaNs(t *testing.T) {
assertMinMaxAreSpaced(someNanStatsf16, someNansf16, validBitmap,
f16Min, f16Max)
}
+func TestFloatingPointInfinityStatistics(t *testing.T) {
+ f32Col := schema.NewColumn(schema.NewFloat32Node("f32",
parquet.Repetitions.Required, -1), 0, 0)
+ f64Col := schema.NewColumn(schema.NewFloat64Node("f64",
parquet.Repetitions.Required, -1), 0, 0)
+ f16Col := schema.NewColumn(newFloat16Node("f16",
parquet.Repetitions.Required, -1), 0, 0)
+
+ t.Run("float32", func(t *testing.T) {
+ for _, value := range []float32{float32(math.Inf(1)),
float32(math.Inf(-1))} {
+ stats := metadata.NewStatistics(f32Col,
memory.DefaultAllocator).(*metadata.Float32Statistics)
+ stats.Update([]float32{value, value}, 0)
+ require.True(t, stats.HasMinMax())
+ assert.Equal(t, value, stats.Min())
+ assert.Equal(t, value, stats.Max())
+ }
+ })
+
+ t.Run("float64", func(t *testing.T) {
+ for _, value := range []float64{math.Inf(1), math.Inf(-1)} {
+ stats := metadata.NewStatistics(f64Col,
memory.DefaultAllocator).(*metadata.Float64Statistics)
+ stats.Update([]float64{value, value}, 0)
+ require.True(t, stats.HasMinMax())
+ assert.Equal(t, value, stats.Min())
+ assert.Equal(t, value, stats.Max())
+ }
+ })
+
+ t.Run("float16", func(t *testing.T) {
+ for _, value := range []float16.Num{float16.Inf(),
float16.Inf().Negate()} {
+ stats := metadata.NewStatistics(f16Col,
memory.DefaultAllocator).(*metadata.Float16Statistics)
+
stats.Update([]parquet.FixedLenByteArray{value.ToLEBytes(), value.ToLEBytes()},
0)
+ require.True(t, stats.HasMinMax())
+ assert.True(t,
value.Equal(float16.FromLEBytes(stats.Min())))
+ assert.True(t,
value.Equal(float16.FromLEBytes(stats.Max())))
+ }
+ })
+}
+
func TestCheckNegativeZeroStats(t *testing.T) {
assertMinMaxZeroesSign := func(stats metadata.TypedStatistics, values
interface{}) {
switch s := stats.(type) {
