This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new edf981e1f refactor!: remove ManifestFile.FetchEntries and
DataFile.DistinctValueCounts (#2179)
edf981e1f is described below
commit edf981e1f1ca21ce03b416f4eac8b536f44388f0
Author: Andrei Tserakhau <[email protected]>
AuthorDate: Tue Oct 13 14:31:11 2026 +0200
refactor!: remove ManifestFile.FetchEntries and
DataFile.DistinctValueCounts (#2179)
refactor!: remove deprecated ManifestFile.FetchEntries and
DataFile.DistinctValueCounts
ManifestFile.FetchEntries loaded every entry into memory and took no
context. Use ManifestFile.Entries, which streams entries as an iterator:
for entry, err := range mf.Entries(fs, discardDeleted) {
if err != nil {
return err
}
// use entry
}
or ReadManifest when a slice is needed.
DataFile.DistinctValueCounts and DataFileBuilder.DistinctValueCounts exposed
distinct_counts (field 111), which the spec deprecates in every format
version. Writers already omitted the field, so the setter had no effect on
the wire. The Avro field stays on the data file struct, so legacy manifests
that carry it still decode, and writers continue to omit it. The tests now
check both on the struct, and a codec test checks that the entry schema has
no field 111.
This is a breaking change for callers of the removed methods.
Part of the v1.0 API cleanup in #2062.
Signed-off-by: laskoviymishka <[email protected]>
---
codec/data_file.go | 8 ++--
codec/data_file_test.go | 14 ------
data_file_codec.go | 10 ++---
data_file_codec_test.go | 47 ++++++++++++++++----
data_file_refs.go | 2 +-
manifest.go | 88 ++++++++++---------------------------
manifest_projection_bench_test.go | 3 --
manifest_test.go | 28 +++++-------
table/compaction/compaction_test.go | 1 -
table/dv/deletion_vector_test.go | 1 -
table/evaluators_test.go | 1 -
11 files changed, 80 insertions(+), 123 deletions(-)
diff --git a/codec/data_file.go b/codec/data_file.go
index 4df07d9d6..b77ee339a 100644
--- a/codec/data_file.go
+++ b/codec/data_file.go
@@ -63,11 +63,9 @@ import (
// [DecodeDataFile].
//
// distinct_counts (field 111) is deprecated in the spec for every
-// version (apache/iceberg#12182). EncodeDataFile drops the field on
-// encode for v1, v2, and v3 alike — values populated on the source
-// DataFile are not transported. Legacy manifests that already carry
-// the field on the wire still decode correctly through
-// [DecodeDataFile]. New DataFiles should not set distinct counts.
+// version (apache/iceberg#12182). EncodeDataFile never encodes it,
+// for v1, v2, and v3 alike. Legacy manifests that already carry the
+// field on the wire still decode correctly through [DecodeDataFile].
func EncodeDataFile(df iceberg.DataFile, spec iceberg.PartitionSpec, schema
*iceberg.Schema, version int) ([]byte, error) {
if version < 1 || version > 3 {
return nil, fmt.Errorf("codec: EncodeDataFile: unsupported
format version %d", version)
diff --git a/codec/data_file_test.go b/codec/data_file_test.go
index 3eb9ca21d..335295c55 100644
--- a/codec/data_file_test.go
+++ b/codec/data_file_test.go
@@ -183,13 +183,6 @@ func fullyPopulatedDataFile(t *testing.T, version int)
(iceberg.PartitionSpec, *
SplitOffsets([]int64{0, 4096}).
SortOrderID(0).
KeyMetadata([]byte("kms-key-1"))
- // distinct_counts (field 111) is deprecated in the spec for every
- // version (apache/iceberg#12182). The fixture populates it on every
- // version to assert that the wire codec drops the field on encode
- // regardless of what the in-memory DataFile carries — strict readers
- // (e.g. PyIceberg) reject manifests that emit it. The deprecated
- // setter is invoked here intentionally to exercise that drop path.
- builder.DistinctValueCounts(map[int]int64{1: 64, 2: 128})
//nolint:staticcheck // SA1019: deliberate use of deprecated setter to assert
the encoder drops the field
if version >= 2 {
builder.EqualityFieldIDs([]int{1})
}
@@ -221,11 +214,6 @@ func assertDataFileEqual(t *testing.T, want, got
iceberg.DataFile, version int)
require.Equal(t, want.SortOrderID(), got.SortOrderID())
require.Equal(t, want.SpecID(), got.SpecID())
require.Equal(t, want.ContentType(), got.ContentType())
- require.Empty(t, got.DistinctValueCounts(),
- "distinct_counts (field 111) is deprecated in every version "+
- "(apache/iceberg#12182); the manifest-entry schemas in
"+
- "internal/avro_schemas.go omit it for v1/v2/v3 so the
encoder "+
- "drops it on the wire regardless of what the source
DataFile carries")
if version >= 2 {
require.Equal(t, want.EqualityFieldIDs(),
got.EqualityFieldIDs())
}
@@ -253,7 +241,6 @@ var expectedDataFileMethods = []string{
"ContentSizeInBytes",
"ContentType",
"Count",
- "DistinctValueCounts",
"EqualityFieldIDs",
"FileFormat",
"FilePath",
@@ -299,7 +286,6 @@ func (stubDataFile) ColumnSizes() map[int]int64
{ return nil }
func (stubDataFile) ValueCounts() map[int]int64 { return nil }
func (stubDataFile) NullValueCounts() map[int]int64 { return nil }
func (stubDataFile) NaNValueCounts() map[int]int64 { return nil }
-func (stubDataFile) DistinctValueCounts() map[int]int64 { return nil }
func (stubDataFile) LowerBoundValues() map[int][]byte { return nil }
func (stubDataFile) UpperBoundValues() map[int][]byte { return nil }
func (stubDataFile) KeyMetadata() []byte { return nil }
diff --git a/data_file_codec.go b/data_file_codec.go
index 1aeb0c83c..15c6c84ba 100644
--- a/data_file_codec.go
+++ b/data_file_codec.go
@@ -96,12 +96,10 @@ type AvroEntryMarshaler interface {
// matching decoder.
//
// distinct_counts (field 111) is deprecated in the spec for every
-// version (apache/iceberg#12182). MarshalAvroEntry drops the field
-// on encode for v1, v2, and v3 alike — values populated on the
-// source DataFile are not transported. The Avro tag on the dataFile
-// struct is intentionally retained so legacy manifests that already
-// carry the field on the wire still decode through the matching
-// reader. New DataFiles should not set distinct counts.
+// version (apache/iceberg#12182). MarshalAvroEntry never encodes it,
+// for v1, v2, and v3 alike. The Avro tag on the dataFile struct is
+// intentionally retained so legacy manifests that already carry the
+// field on the wire still decode through the matching reader.
func (d *dataFile) MarshalAvroEntry(spec PartitionSpec, schema *Schema,
version int) ([]byte, error) {
if err := d.validateForWrite(); err != nil {
return nil, err
diff --git a/data_file_codec_test.go b/data_file_codec_test.go
index 6ac4d16c5..6126e6975 100644
--- a/data_file_codec_test.go
+++ b/data_file_codec_test.go
@@ -95,6 +95,35 @@ func TestDataFileCodecWithDroppedPartitionSource(t
*testing.T) {
require.Equal(t, map[int]any{1000: nil, 1001: int32(3)},
decoded.Partition())
}
+// TestMarshalAvroEntryOmitsDistinctCounts checks that the data_file schema the
+// entry codec encodes with has no distinct_counts (field 111), which the spec
+// deprecates in every version (apache/iceberg#12182). The encoder and decoder
+// share this schema, so a round trip alone could not show the field leaking;
+// strict readers such as PyIceberg reject manifests that emit it.
+func TestMarshalAvroEntryOmitsDistinctCounts(t *testing.T) {
+ for _, version := range []int{1, 2, 3} {
+ t.Run("v"+strconv.Itoa(version), func(t *testing.T) {
+ spec, schema, _ := fullyPopulatedDataFileForCodec(t,
version)
+ s, _, err := manifestEntrySchemaFor(spec, schema,
version)
+ require.NoError(t, err)
+
+ root := s.Root()
+ found := false
+ for _, field := range root.Fields {
+ if field.Name != "data_file" {
+ continue
+ }
+ found = true
+ for _, dfField := range field.Type.Fields {
+ require.NotEqual(t, "distinct_counts",
dfField.Name)
+ require.NotEqual(t, 111,
dfField.Props["field-id"])
+ }
+ }
+ require.True(t, found, "data_file field not found on
manifest_entry v%d schema", version)
+ })
+ }
+}
+
// TestManifestEntrySchemaForDistinguishesSameSpecIDDifferentPartitionTypes
// guards against the cache-key collision that would occur if the cache
// were keyed only by spec.ID(). Two tables can both use
@@ -292,13 +321,6 @@ func fullyPopulatedDataFileForCodec(t testing.TB, version
int) (PartitionSpec, *
SplitOffsets([]int64{0, 4096}).
SortOrderID(0).
KeyMetadata([]byte("kms-key-1"))
- if version < 3 {
- // distinct_counts is deprecated in the spec for all versions
- // (apache/iceberg#12182); the fixture sets it to cover the
- // read-compatibility round-trip path for legacy manifests, not
- // to model what a new DataFile should carry.
- builder.DistinctValueCounts(map[int]int64{1: 64, 2: 128})
- }
if version >= 2 {
builder.EqualityFieldIDs([]int{1})
}
@@ -309,7 +331,16 @@ func fullyPopulatedDataFileForCodec(t testing.TB, version
int) (PartitionSpec, *
ContentSizeInBytes(2048)
}
- return spec, schema, builder.Build()
+ df := builder.Build()
+ if version < 3 {
+ // distinct_counts is deprecated in the spec for all versions
+ // (apache/iceberg#12182) and has no public setter. The fixture
sets
+ // the Avro field directly to cover the read-compatibility path
for
+ // legacy manifests, not to model what a new DataFile should
carry.
+ df.(*dataFile).DistinctCounts =
mapToAvroColMap(map[int]int64{1: 64, 2: 128})
+ }
+
+ return spec, schema, df
}
var (
diff --git a/data_file_refs.go b/data_file_refs.go
index 8b5445746..723f400fa 100644
--- a/data_file_refs.go
+++ b/data_file_refs.go
@@ -28,7 +28,7 @@ import "github.com/apache/iceberg-go/internal"
// not introduce mutable values through this borrowed view.
//
// This view intentionally contains only the maps consumed by metric
-// evaluators. ColumnSizes and DistinctValueCounts are excluded because the
+// evaluators. ColumnSizes and distinct_counts are excluded because the
// evaluator does not read them, and cloning those maps would add work to the
// hot path without changing its result. Partition has a separate borrowed
// accessor because partition data is read during scan planning.
diff --git a/manifest.go b/manifest.go
index 65eb82f56..e344b14a9 100644
--- a/manifest.go
+++ b/manifest.go
@@ -419,16 +419,6 @@ func (m *manifestFile) Entries(fs iceio.IO, discardDeleted
bool) iter.Seq2[Manif
return manifestEntries(fs, m, discardDeleted, nil)
}
-func (m *manifestFile) FetchEntries(fs iceio.IO, discardDeleted bool) (_
[]ManifestEntry, err error) {
- f, openErr := fs.Open(m.FilePath())
- if openErr != nil {
- return nil, openErr
- }
- defer internal.CheckedClose(f, &err)
-
- return ReadManifest(m, f, discardDeleted)
-}
-
func getFieldIDMap(sc *avro.Schema) dataFileFieldMaps {
root := sc.Root()
getField := func(node avro.SchemaNode, name string) *avro.SchemaField {
@@ -582,8 +572,8 @@ type ManifestFile interface {
// the provided file system IO interface. Entries that have been
// marked as deleted are skipped if discardDeleted is true.
//
- // Prefer Entries over FetchEntries when walking large manifests
- // since it avoids loading every entry into memory at once.
+ // Entries avoids loading every manifest entry into memory at once,
+ // making it suitable for walking large manifests.
//
// Iteration contract:
//
@@ -603,14 +593,6 @@ type ManifestFile interface {
// longer interested in further values, so an extra synthetic
// (nil, closeErr) tail would be discarded anyway.
Entries(fs iceio.IO, discardDeleted bool) iter.Seq2[ManifestEntry,
error]
- // FetchEntries reads the manifest list file to fetch the list of
- // manifest entries using the provided file system IO interface.
- // If discardDeleted is true, entries for files containing deleted rows
- // will be skipped.
- //
- // Deprecated: Use Entries instead, which streams manifest entries via
an
- // iterator and avoids loading every entry into memory at once.
- FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error)
// // WriteEntries writes a list of manifest entries to a provided
// // io.Writer. The version of the manifest file is used to determine
the
// // schema to use for writing the entries.
@@ -2613,18 +2595,21 @@ func fitDecimalBytes(bytes []byte, size int) ([]byte,
error) {
}
type dataFile struct {
- Content ManifestEntryContent `avro:"content"`
- projected bool
- Path string `avro:"file_path"`
- Format FileFormat `avro:"file_format"`
- PartitionData map[string]any `avro:"partition"`
- RecordCount int64 `avro:"record_count"`
- FileSize int64
`avro:"file_size_in_bytes"`
- BlockSizeInBytes int64
`avro:"block_size_in_bytes"`
- ColSizes *[]colMap[int, int64] `avro:"column_sizes"`
- ValCounts *[]colMap[int, int64] `avro:"value_counts"`
- NullCounts *[]colMap[int, int64]
`avro:"null_value_counts"`
- NaNCounts *[]colMap[int, int64] `avro:"nan_value_counts"`
+ Content ManifestEntryContent `avro:"content"`
+ projected bool
+ Path string `avro:"file_path"`
+ Format FileFormat `avro:"file_format"`
+ PartitionData map[string]any `avro:"partition"`
+ RecordCount int64 `avro:"record_count"`
+ FileSize int64 `avro:"file_size_in_bytes"`
+ BlockSizeInBytes int64 `avro:"block_size_in_bytes"`
+ ColSizes *[]colMap[int, int64] `avro:"column_sizes"`
+ ValCounts *[]colMap[int, int64] `avro:"value_counts"`
+ NullCounts *[]colMap[int, int64] `avro:"null_value_counts"`
+ NaNCounts *[]colMap[int, int64] `avro:"nan_value_counts"`
+ // DistinctCounts holds distinct_counts (field 111), deprecated in the
+ // spec. It has no accessor and writers never encode it; it stays so
+ // legacy manifests that carry the field still decode.
DistinctCounts *[]colMap[int, int64] `avro:"distinct_counts"`
LowerBounds *[]colMap[int, []byte] `avro:"lower_bounds"`
UpperBounds *[]colMap[int, []byte] `avro:"upper_bounds"`
@@ -2637,13 +2622,12 @@ type dataFile struct {
ContentOffsetField *int64 `avro:"content_offset"`
ContentSizeInBytesField *int64
`avro:"content_size_in_bytes"`
- colSizeMap map[int]int64
- valCntMap map[int]int64
- nullCntMap map[int]int64
- nanCntMap map[int]int64
- distinctCntMap map[int]int64
- lowerBoundMap map[int][]byte
- upperBoundMap map[int][]byte
+ colSizeMap map[int]int64
+ valCntMap map[int]int64
+ nullCntMap map[int]int64
+ nanCntMap map[int]int64
+ lowerBoundMap map[int][]byte
+ upperBoundMap map[int][]byte
// used for partition retrieval
fieldNameToID map[string]int
@@ -2676,7 +2660,6 @@ func (d *dataFile) initColumnStatsData() {
d.valCntMap = avroColMapToMap(d.ValCounts)
d.nullCntMap = avroColMapToMap(d.NullCounts)
d.nanCntMap = avroColMapToMap(d.NaNCounts)
- d.distinctCntMap = avroColMapToMap(d.DistinctCounts)
d.lowerBoundMap = avroColMapToMap(d.LowerBounds)
d.upperBoundMap = avroColMapToMap(d.UpperBounds)
})
@@ -2907,12 +2890,6 @@ func (d *dataFile) NaNValueCounts() map[int]int64 {
return maps.Clone(d.nanCntMap)
}
-func (d *dataFile) DistinctValueCounts() map[int]int64 {
- d.initColumnStatsData()
-
- return maps.Clone(d.distinctCntMap)
-}
-
func (d *dataFile) LowerBoundValues() map[int][]byte {
d.initColumnStatsData()
@@ -3199,20 +3176,6 @@ func (b *DataFileBuilder) NaNValueCounts(counts
map[int]int64) *DataFileBuilder
return b
}
-// DistinctValueCounts sets the distinct value counts for the data file.
-//
-// Deprecated: distinct_counts (field 111) is deprecated in every
-// version of the Iceberg spec (apache/iceberg#12182). The Avro
-// manifest-entry schemas omit the field for v1, v2, and v3, so values
-// set here are not transported in manifests written by this library.
-// The setter is retained for round-tripping legacy DataFiles read from
-// older manifests; new code should not call it.
-func (b *DataFileBuilder) DistinctValueCounts(counts map[int]int64)
*DataFileBuilder {
- b.d.DistinctCounts = mapToAvroColMap(counts)
-
- return b
-}
-
// LowerBoundValues sets the lower bound values for the data file.
func (b *DataFileBuilder) LowerBoundValues(bounds map[int][]byte)
*DataFileBuilder {
b.d.LowerBounds = mapToAvroColMapClonedBytes(bounds)
@@ -3318,11 +3281,6 @@ type DataFile interface {
// NaNValueCounts is a mapping from column id to the number of NaN
// values in the column.
NaNValueCounts() map[int]int64
- // DistictValueCounts is a mapping from column id to the number of
- // distinct values in the column. Distinct counts must be derived
- // using values in the file by counting or using sketches, but not
- // using methods like merging existing distinct counts.
- DistinctValueCounts() map[int]int64
// LowerBoundValues is a mapping from column id to the lower bounded
// value of the column, serialized as binary. Each value in the column
// must be less than or requal to all non-null, non-NaN values in the
diff --git a/manifest_projection_bench_test.go
b/manifest_projection_bench_test.go
index 97d65087b..105a95d8e 100644
--- a/manifest_projection_bench_test.go
+++ b/manifest_projection_bench_test.go
@@ -65,7 +65,6 @@ func newManifestProjectionBenchmarkFixture(
lowerBounds := make(map[int][]byte, fieldCount)
upperBounds := make(map[int][]byte, fieldCount)
columnSizes := make(map[int]int64, fieldCount)
- distinctCounts := make(map[int]int64, fieldCount)
for i := range fieldCount {
id := i + 1
fields[i] = NestedField{
@@ -77,7 +76,6 @@ func newManifestProjectionBenchmarkFixture(
lowerBounds[id] = []byte{0, 0, 0, 0, 0, 0, 0, 0}
upperBounds[id] = []byte{0, 0, 0, 0, 0, 0, 0, 1}
columnSizes[id] = 64
- distinctCounts[id] = 1
}
schema := NewSchema(1, fields...)
builder, err := NewDataFileBuilder(
@@ -99,7 +97,6 @@ func newManifestProjectionBenchmarkFixture(
ValueCounts(valueCounts).
NullValueCounts(nullCounts).
NaNValueCounts(nanCounts).
- DistinctValueCounts(distinctCounts).
LowerBoundValues(lowerBounds).
UpperBoundValues(upperBounds)
diff --git a/manifest_test.go b/manifest_test.go
index 5b126ebc4..49a63aac4 100644
--- a/manifest_test.go
+++ b/manifest_test.go
@@ -2447,9 +2447,6 @@ func (m *ManifestTestSuite) TestManifestEntryBuilder() {
}).ValueCounts(map[int]int64{
1: 1,
2: 2,
- }).DistinctValueCounts(map[int]int64{
- 1: 1,
- 2: 2,
}).NullValueCounts(map[int]int64{
1: 0,
2: 0,
@@ -2494,10 +2491,6 @@ func (m *ManifestTestSuite) TestManifestEntryBuilder() {
1: 0,
2: 0,
}, data.NullValueCounts())
- m.Assert().Equal(map[int]int64{
- 1: 1,
- 2: 2,
- }, data.DistinctValueCounts())
m.Assert().Equal(map[int]int64{
1: 0,
2: 0,
@@ -3895,7 +3888,6 @@ func (m *ManifestTestSuite)
TestDataFileMetadataIsIsolatedFromExternalMutation()
valueCounts := map[int]int64{1: 2}
nullCounts := map[int]int64{1: 1}
nanCounts := map[int]int64{1: 0}
- distinctCounts := map[int]int64{1: 2}
lower := map[int][]byte{1: {0x03, 0x04}}
upper := map[int][]byte{1: {0x05, 0x06}}
key := []byte{0x07, 0x08}
@@ -3907,7 +3899,6 @@ func (m *ManifestTestSuite)
TestDataFileMetadataIsIsolatedFromExternalMutation()
ValueCounts(valueCounts).
NullValueCounts(nullCounts).
NaNValueCounts(nanCounts).
- DistinctValueCounts(distinctCounts).
LowerBoundValues(lower).
UpperBoundValues(upper).
KeyMetadata(key).
@@ -3922,7 +3913,7 @@ func (m *ManifestTestSuite)
TestDataFileMetadataIsIsolatedFromExternalMutation()
partition[0], lower[1][0], upper[1][0], key[0], splits[0],
equalityIDs[0] = 0xff, 0xff, 0xff, 0xff, 99, 99
partitionData[1000] = []byte{0xff}
- columnSizes[1], valueCounts[1], nullCounts[1], nanCounts[1],
distinctCounts[1] = 99, 99, 99, 99, 99
+ columnSizes[1], valueCounts[1], nullCounts[1], nanCounts[1] = 99, 99,
99, 99
m.Equal([]byte{0x01, 0x02}, dataFile.Partition()[1000])
dataFile.Partition()[1000].([]byte)[0] = 0xff
@@ -3930,7 +3921,6 @@ func (m *ManifestTestSuite)
TestDataFileMetadataIsIsolatedFromExternalMutation()
dataFile.ValueCounts()[1] = 99
dataFile.NullValueCounts()[1] = 99
dataFile.NaNValueCounts()[1] = 99
- dataFile.DistinctValueCounts()[1] = 99
dataFile.LowerBoundValues()[1][0] = 0xff
dataFile.UpperBoundValues()[1][0] = 0xff
dataFile.KeyMetadata()[0] = 0xff
@@ -3947,7 +3937,6 @@ func (m *ManifestTestSuite)
TestDataFileMetadataIsIsolatedFromExternalMutation()
m.Equal(map[int]int64{1: 2}, dataFile.ValueCounts())
m.Equal(map[int]int64{1: 1}, dataFile.NullValueCounts())
m.Equal(map[int]int64{1: 0}, dataFile.NaNValueCounts())
- m.Equal(map[int]int64{1: 2}, dataFile.DistinctValueCounts())
m.Equal([]byte{0x03, 0x04}, dataFile.LowerBoundValues()[1])
m.Equal([]byte{0x05, 0x06}, dataFile.UpperBoundValues()[1])
m.Equal([]byte{0x07, 0x08}, dataFile.KeyMetadata())
@@ -4006,7 +3995,6 @@ func (m *ManifestTestSuite)
TestDecodedDataFileMetadataIsIsolatedFromSourceAndGe
m.Equal(map[int]int64{1: 2}, decoded.ValueCounts())
m.Equal(map[int]int64{1: 1}, decoded.NullValueCounts())
m.Equal(map[int]int64{1: 0}, decoded.NaNValueCounts())
- m.Empty(decoded.DistinctValueCounts())
m.Equal([]byte{0x03, 0x04}, decoded.LowerBoundValues()[1])
m.Equal([]byte{0x05, 0x06}, decoded.UpperBoundValues()[1])
m.Equal([]byte{0x07, 0x08}, decoded.KeyMetadata())
@@ -4048,7 +4036,6 @@ func (m *ManifestTestSuite)
TestDecodedDataFileMetadataIsIsolatedFromSourceAndGe
m.Equal(map[int]int64{1: 2}, decoded.ValueCounts())
m.Equal(map[int]int64{1: 1}, decoded.NullValueCounts())
m.Equal(map[int]int64{1: 0}, decoded.NaNValueCounts())
- m.Empty(decoded.DistinctValueCounts())
m.Equal([]byte{0x03, 0x04}, decoded.LowerBoundValues()[1])
m.Equal([]byte{0x05, 0x06}, decoded.UpperBoundValues()[1])
m.Equal([]byte{0x07, 0x08}, decoded.KeyMetadata())
@@ -4399,7 +4386,11 @@ func (m *ManifestTestSuite)
assertWriteOmitsDistinctCounts(version int) {
1,
)
m.Require().NoError(err)
- dataFileBuilder.DistinctValueCounts(map[int]int64{1: 42})
+ // There is no public setter for distinct_counts any more, but a
DataFile
+ // decoded from a legacy manifest can still carry it, so set the Avro
+ // field directly.
+ df := dataFileBuilder.Build()
+ df.(*dataFile).DistinctCounts = mapToAvroColMap(map[int]int64{1: 42})
var buf bytes.Buffer
file, err := WriteManifest(
@@ -4414,7 +4405,7 @@ func (m *ManifestTestSuite)
assertWriteOmitsDistinctCounts(version int) {
EntryStatusADDED,
&snapshotID,
&seqNum, &seqNum,
- dataFileBuilder.Build(),
+ df,
)},
)
m.Require().NoError(err)
@@ -4422,7 +4413,7 @@ func (m *ManifestTestSuite)
assertWriteOmitsDistinctCounts(version int) {
entries, err := ReadManifest(file, &buf, false)
m.Require().NoError(err)
m.Require().Len(entries, 1)
- m.Empty(entries[0].DataFile().DistinctValueCounts(),
+
m.Empty(avroColMapToMap(entries[0].DataFile().(*dataFile).DistinctCounts),
"manifest writer must drop distinct_counts on the wire for
every format version; "+
"see internal/avro_schemas.go
data_file_v"+strconv.Itoa(version))
}
@@ -4490,7 +4481,8 @@ func (m *ManifestTestSuite)
TestReadManifestLegacyDistinctCounts() {
entries, err := ReadManifest(file, &buf, false)
m.Require().NoError(err)
m.Require().Len(entries, 1)
- m.Equal(map[int]int64{1: 99, 2: 88},
entries[0].DataFile().DistinctValueCounts(),
+ m.Equal(map[int]int64{1: 99, 2: 88},
+
avroColMapToMap(entries[0].DataFile().(*dataFile).DistinctCounts),
"ReadManifest must decode distinct_counts (field 111) from
legacy "+
"manifests where the file's writer schema still carries
the field")
}
diff --git a/table/compaction/compaction_test.go
b/table/compaction/compaction_test.go
index ddbb78c3e..dbba3cf9a 100644
--- a/table/compaction/compaction_test.go
+++ b/table/compaction/compaction_test.go
@@ -52,7 +52,6 @@ func (f *testDataFile) ColumnSizes() map[int]int64
{ return nil }
func (f *testDataFile) ValueCounts() map[int]int64 { return nil
}
func (f *testDataFile) NullValueCounts() map[int]int64 { return nil
}
func (f *testDataFile) NaNValueCounts() map[int]int64 { return nil
}
-func (f *testDataFile) DistinctValueCounts() map[int]int64 { return nil
}
func (f *testDataFile) LowerBoundValues() map[int][]byte { return
f.lowerBounds }
func (f *testDataFile) UpperBoundValues() map[int][]byte { return
f.upperBounds }
func (f *testDataFile) KeyMetadata() []byte { return nil
}
diff --git a/table/dv/deletion_vector_test.go b/table/dv/deletion_vector_test.go
index e9333d234..15239908a 100644
--- a/table/dv/deletion_vector_test.go
+++ b/table/dv/deletion_vector_test.go
@@ -62,7 +62,6 @@ func (*mockDVFile) ColumnSizes() map[int]int64
{ return nil }
func (*mockDVFile) ValueCounts() map[int]int64 { return nil }
func (*mockDVFile) NullValueCounts() map[int]int64 { return nil }
func (*mockDVFile) NaNValueCounts() map[int]int64 { return nil }
-func (*mockDVFile) DistinctValueCounts() map[int]int64 { return nil }
func (*mockDVFile) LowerBoundValues() map[int][]byte { return nil }
func (*mockDVFile) UpperBoundValues() map[int][]byte { return nil }
func (*mockDVFile) KeyMetadata() []byte { return nil }
diff --git a/table/evaluators_test.go b/table/evaluators_test.go
index 0d584fe07..03c2ae56a 100644
--- a/table/evaluators_test.go
+++ b/table/evaluators_test.go
@@ -1308,7 +1308,6 @@ func (m *mockDataFile) ColumnSizes() map[int]int64
{ return m.col
func (m *mockDataFile) ValueCounts() map[int]int64 { return
m.valueCounts }
func (m *mockDataFile) NullValueCounts() map[int]int64 { return
m.nullCounts }
func (m *mockDataFile) NaNValueCounts() map[int]int64 { return
m.nanCounts }
-func (*mockDataFile) DistinctValueCounts() map[int]int64 { return nil
}
func (m *mockDataFile) LowerBoundValues() map[int][]byte { return
m.lowerBounds }
func (m *mockDataFile) UpperBoundValues() map[int][]byte { return
m.upperBounds }
func (*mockDataFile) KeyMetadata() []byte { return nil
}
