Copilot commented on code in PR #1588:
URL: https://github.com/apache/arrow-go/pull/1588#discussion_r4591903853


##########
parquet/file/column_reader_test.go:
##########
@@ -856,6 +859,206 @@ func TestFullSeekRow(t *testing.T) {
        }
 }
 
+type seekBatchReader[T any] interface {
+       ReadBatch(int64, []T, []int16, []int16) (int64, int, error)
+}
+
+// readRows reads n flat values, returning nil entries for nulls.
+func readRows[T any](t *testing.T, r seekBatchReader[T], n int, maxDef int16) 
[]*T {
+       t.Helper()
+       vals := make([]T, n)
+       defs := make([]int16, n)
+       got, nvals, err := r.ReadBatch(int64(n), vals, defs, nil)
+       require.NoError(t, err)
+       require.EqualValues(t, n, got)
+
+       out := make([]*T, n)
+       next := 0
+       for i := range out {
+               if maxDef == 0 || defs[i] == maxDef {
+                       out[i] = &vals[next]
+                       next++
+               }
+       }
+       require.Equal(t, nvals, next)
+       return out
+}
+
+// Both files hold dictionary encoded columns whose chunks have no
+// dictionary_page_offset, so the dictionary page sits at data_page_offset.
+func TestSeekToRowDictionaryAtDataPageOffset(t *testing.T) {
+       dir := os.Getenv("PARQUET_TEST_DATA")
+       if dir == "" {
+               dir = "../../parquet-testing/data"
+               t.Log("PARQUET_TEST_DATA not set, using 
../../parquet-testing/data")
+       }
+       require.DirExists(t, dir)
+
+       t.Run("offset index", func(t *testing.T) {
+               fr, err := file.OpenParquetFile(filepath.Join(dir, 
"alltypes_tiny_pages.parquet"), false)
+               require.NoError(t, err)
+               defer fr.Close()
+               rg := fr.RowGroup(0)
+
+               const col, nrows = 2, 100
+               maxDef := fr.MetaData().Schema.Column(col).MaxDefinitionLevel()
+               open := func() *file.Int32ColumnChunkReader {
+                       cr, err := rg.Column(col)
+                       require.NoError(t, err)
+                       return cr.(*file.Int32ColumnChunkReader)
+               }
+               want := readRows[int32](t, open(), nrows, maxDef)
+
+               // Reading past the seek target crosses data pages, which only 
works if the
+               // dictionary load left the page stream where it was.
+               cr := open()
+               require.NoError(t, cr.SeekToRow(0))
+               require.Equal(t, want, readRows[int32](t, cr, nrows, maxDef))
+       })
+
+       t.Run("no offset index", func(t *testing.T) {
+               fr, err := file.OpenParquetFile(filepath.Join(dir, 
"datapage_v2.snappy.parquet"), false)
+               require.NoError(t, err)
+               defer fr.Close()
+               rg := fr.RowGroup(0)
+
+               const col, nrows = 2, 5
+               maxDef := fr.MetaData().Schema.Column(col).MaxDefinitionLevel()
+               open := func() *file.Float64ColumnChunkReader {
+                       cr, err := rg.Column(col)
+                       require.NoError(t, err)
+                       return cr.(*file.Float64ColumnChunkReader)
+               }
+               want := readRows[float64](t, open(), nrows, maxDef)
+
+               t.Run("fresh reader", func(t *testing.T) {
+                       cr := open()
+                       require.NoError(t, cr.SeekToRow(4))
+                       require.Equal(t, want[4:], readRows[float64](t, cr, 1, 
maxDef))
+               })
+
+               t.Run("reseek after read", func(t *testing.T) {
+                       cr := open()
+                       readRows[float64](t, cr, nrows, maxDef)
+                       require.NoError(t, cr.SeekToRow(0))
+                       require.Equal(t, want, readRows[float64](t, cr, nrows, 
maxDef))
+               })
+       })
+}
+
+func sharedOffsetFile(t *testing.T, values []int32, pageIndex bool) []byte {
+       t.Helper()
+       sc := schema.MustGroup(schema.NewGroupNode("schema", 
parquet.Repetitions.Required, schema.FieldList{
+               schema.NewInt32Node("x", parquet.Repetitions.Required, -1),
+       }, -1))
+       var out bytes.Buffer
+       w, err := file.NewParquetWriterWithError(&out, sc, 
file.WithWriterProps(parquet.NewWriterProperties(
+               parquet.WithDictionaryDefault(true),
+               parquet.WithDictionaryPageSizeLimit(1<<20),
+               parquet.WithDictionaryCostFallbackFor("x", false),
+               parquet.WithDataPageSize(256),
+               parquet.WithBatchSize(50),
+               parquet.WithPageIndexEnabled(pageIndex))))
+       require.NoError(t, err)
+       rg := w.AppendRowGroup()
+       cw, err := rg.NextColumn()
+       require.NoError(t, err)
+       _, err = cw.(*file.Int32ColumnChunkWriter).WriteBatch(values, nil, nil)
+       require.NoError(t, err)
+       require.NoError(t, cw.Close())
+       require.NoError(t, rg.Close())
+       require.NoError(t, w.Close())
+
+       data := out.Bytes()
+       footerStart := len(data) - 8 - 
int(binary.LittleEndian.Uint32(data[len(data)-8:]))
+       md := format.NewFileMetaData()
+       _, err = thrift.DeserializeThrift(md, data[footerStart:len(data)-8])
+       require.NoError(t, err)
+       chunk := md.RowGroups[0].Columns[0].MetaData
+       require.True(t, chunk.IsSetDictionaryPageOffset())
+       require.Greater(t, chunk.DataPageOffset, 
chunk.GetDictionaryPageOffset())
+       chunk.DataPageOffset = chunk.GetDictionaryPageOffset()
+
+       var footer bytes.Buffer
+       _, err = thrift.NewThriftSerializer().Serialize(md, &footer, nil)
+       require.NoError(t, err)
+       var footerLen [4]byte
+       binary.LittleEndian.PutUint32(footerLen[:], uint32(footer.Len()))
+       return append(append(append(bytes.Clone(data[:footerStart]), 
footer.Bytes()...), footerLen[:]...), "PAR1"...)
+}
+
+// Unreleased parquet-mr builds between PARQUET-1850 and PARQUET-1977 wrote
+// chunks whose data_page_offset equals dictionary_page_offset, so the
+// dictionary page sits at both offsets and a seek has to read it from there.
+func TestSeekToRowDictionaryPageOffsetEqualsDataPageOffset(t *testing.T) {
+       const nrows = 2000
+       values := make([]int32, nrows)
+       for i := range values {
+               values[i] = int32((i * 37) % 1000)
+       }
+
+       for _, pageIndex := range []bool{false, true} {
+               t.Run(fmt.Sprintf("offset index %v", pageIndex), func(t 
*testing.T) {
+                       rdr, err := 
file.NewParquetReader(bytes.NewReader(sharedOffsetFile(t, values, pageIndex)))
+                       require.NoError(t, err)
+                       t.Cleanup(func() { require.NoError(t, rdr.Close()) })
+                       open := func() *file.Int32ColumnChunkReader {
+                               cr, err := rdr.RowGroup(0).Column(0)
+                               require.NoError(t, err)
+                               t.Cleanup(func() { require.NoError(t, 
cr.Close()) })
+                               return cr.(*file.Int32ColumnChunkReader)
+                       }
+                       read := func(cr *file.Int32ColumnChunkReader, n int) 
[]int32 {
+                               got := make([]int32, n)
+                               read, _, err := cr.ReadBatch(int64(n), got, 
nil, nil)
+                               require.NoError(t, err)
+                               require.EqualValues(t, n, read)
+                               return got
+                       }
+
+                       require.Equal(t, values, read(open(), nrows))
+
+                       const row = 150
+                       cr := open()
+                       require.NoError(t, cr.SeekToRow(row))
+                       require.Equal(t, values[row:row+250], read(cr, 250))
+
+                       if pageIndex {
+                               return
+                       }
+                       t.Run("seek to row 0 after a read", func(t *testing.T) {
+                               cr := open()
+                               read(cr, 10)
+                               require.NoError(t, cr.SeekToRow(0))
+                               require.Equal(t, values[:100], read(cr, 100))
+                       })
+               })
+       }
+}
+
+// The dictionary page offset of each column chunk in this file is past its
+// data page offset of 0.
+func TestGetDictionaryPageOffsetPastDataPageOffset(t *testing.T) {
+       dir := os.Getenv("PARQUET_TEST_DATA")
+       if dir == "" {
+               dir = "../../parquet-testing/data"
+               t.Log("PARQUET_TEST_DATA not set, using 
../../parquet-testing/data")
+       }
+       require.DirExists(t, dir)
+
+       fr, err := file.OpenParquetFile(filepath.Join(dir, 
"column_chunk_key_value_metadata.parquet"), false)
+       require.NoError(t, err)
+       defer fr.Close()
+
+       for col := range fr.MetaData().Schema.NumColumns() {

Review Comment:
   `range fr.MetaData().Schema.NumColumns()` does not compile because 
`NumColumns()` returns an integer, not a slice/map/channel. Use a standard 
indexed loop (`for col := 0; col < fr.MetaData().Schema.NumColumns(); col++ { 
... }`).



##########
parquet/file/page_reader.go:
##########
@@ -737,17 +738,40 @@ func extractStats(dataHeader dataheader) (pageStats 
metadata.EncodedStatistics)
        return
 }
 
+// dictionaryAtDataOffset reports whether the page at data_page_offset may be a
+// dictionary page, which is so when the chunk records no dictionary offset or
+// the same offset as the data offset. Encrypted chunks are excluded because 
the
+// AAD of a page header depends on the page type, which is unknown before the
+// header is read.
+func (p *serializedPageReader) dictionaryAtDataOffset() bool {
+       return p.dataOffset <= p.baseOffset && p.cryptoCtx.MetaDecryptor == nil
+}
+
 func (p *serializedPageReader) GetDictionaryPage() (*DictionaryPage, error) {
-       if p.dictOffset > 0 {
+       probe := p.dictOffset == 0 && p.dictionaryAtDataOffset()
+       if p.dictOffset > 0 || probe {
+               span := p.dataOffset - p.baseOffset
+               readBufSize := int(min(span, int64(p.r.BufferSize())))
+               // The dictionary page starts the stream, and the data offset 
marks its end
+               // only when it is past the dictionary offset. Otherwise the 
length is
+               // unknown, as it is when no dictionary offset is recorded. 
Buffer only a
+               // page header, since the stream of a default reader holds the 
whole chunk.
+               if span <= 0 {
+                       span = math.MaxInt64
+                       readBufSize = min(p.r.BufferSize(), 
defaultPageHeaderSize)
+               }
                hdr := format.NewPageHeader()
-               readBufSize := min(int(p.dataOffset-p.baseOffset), 
p.r.BufferSize())
-               rd := utils.NewBufferedReader(
-                       io.NewSectionReader(p.r.Outer(), 
p.dictOffset-p.baseOffset, p.dataOffset-p.baseOffset),
-                       readBufSize)
+               rd := utils.NewBufferedReader(io.NewSectionReader(p.r.Outer(), 
0, span), readBufSize)

Review Comment:
   Using `io.NewSectionReader(p.r.Outer(), 0, span)` with `span = 
math.MaxInt64` can read past the column chunk boundary (into subsequent file 
data) when `span <= 0`, which can produce incorrect headers and 
nondeterministic behavior. This should be bounded to the actual chunk length 
(or at least to the buffered reader's stream limits) and should use the correct 
start offset relative to the chunk (typically based on 
`baseOffset`/`dictOffset`), rather than hard-coding `0` against the `Outer()` 
reader.



##########
parquet/pqarrow/file_reader_test.go:
##########
@@ -670,6 +670,44 @@ func TestRecordReaderSeekToRowMultiRowGroup(t *testing.T) {
        assert.Truef(t, array.RecordEqual(first, rec), "expected: %s\ngot: %s", 
first, rec)
 }
 
+// string_col in alltypes_tiny_pages is dictionary encoded and its chunk has no
+// dictionary_page_offset.
+func TestRecordReaderSeekToRowDictionaryAtDataPageOffset(t *testing.T) {
+       mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+       defer mem.AssertSize(t, 0)
+
+       pf, err := file.OpenParquetFile(path.Join(getDataDir(), 
"alltypes_tiny_pages.parquet"), false,
+               file.WithReadProps(parquet.NewReaderProperties(mem)))
+       require.NoError(t, err)
+       defer pf.Close()
+
+       fr, err := pqarrow.NewFileReader(pf, 
pqarrow.ArrowReadProperties{BatchSize: 100}, mem)
+       require.NoError(t, err)
+
+       const stringCol, batch = 9, 36
+       newReader := func() pqarrow.RecordReader {
+               rr, err := fr.GetRecordReader(context.Background(), 
[]int{stringCol}, nil)
+               require.NoError(t, err)
+               return rr
+       }
+
+       seq := newReader()
+       defer seq.Release()
+       for range batch {

Review Comment:
   `for range batch` does not compile because `batch` is an integer constant. 
Replace with an index loop (e.g., `for i := 0; i < batch; i++ { ... }`).



##########
parquet/file/page_reader.go:
##########
@@ -737,17 +738,40 @@ func extractStats(dataHeader dataheader) (pageStats 
metadata.EncodedStatistics)
        return
 }
 
+// dictionaryAtDataOffset reports whether the page at data_page_offset may be a
+// dictionary page, which is so when the chunk records no dictionary offset or
+// the same offset as the data offset. Encrypted chunks are excluded because 
the
+// AAD of a page header depends on the page type, which is unknown before the
+// header is read.
+func (p *serializedPageReader) dictionaryAtDataOffset() bool {
+       return p.dataOffset <= p.baseOffset && p.cryptoCtx.MetaDecryptor == nil
+}
+
 func (p *serializedPageReader) GetDictionaryPage() (*DictionaryPage, error) {

Review Comment:
   The comment describes the dictionary page being *at* `data_page_offset`, but 
the implementation returns true for `dataOffset < baseOffset` as well. That 
includes malformed metadata cases (e.g., `data_page_offset` preceding 
`dictionary_page_offset`), and can incorrectly treat a corrupted offset 
ordering as a probe/skip scenario. Consider tightening this to the equality 
case for the intended scenario, and explicitly erroring out when offsets are 
inconsistent (e.g., when a recorded dictionary offset exists but 
`data_page_offset` is before it).



##########
parquet/pqarrow/file_reader_test.go:
##########
@@ -670,6 +670,44 @@ func TestRecordReaderSeekToRowMultiRowGroup(t *testing.T) {
        assert.Truef(t, array.RecordEqual(first, rec), "expected: %s\ngot: %s", 
first, rec)
 }
 
+// string_col in alltypes_tiny_pages is dictionary encoded and its chunk has no
+// dictionary_page_offset.
+func TestRecordReaderSeekToRowDictionaryAtDataPageOffset(t *testing.T) {
+       mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+       defer mem.AssertSize(t, 0)
+
+       pf, err := file.OpenParquetFile(path.Join(getDataDir(), 
"alltypes_tiny_pages.parquet"), false,

Review Comment:
   This new test relies on `getDataDir()`, which panics if `PARQUET_TEST_DATA` 
is unset. Other tests added in this PR fall back to 
`../../parquet-testing/data` and log a message; using the same fallback/skip 
pattern here would avoid hard panics and make local runs more robust.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to