hcrosse commented on code in PR #1588:
URL: https://github.com/apache/arrow-go/pull/1588#discussion_r4591945497


##########
parquet/file/column_reader_test.go:
##########
@@ -856,6 +859,206 @@ func TestFullSeekRow(t *testing.T) {
        }
 }
 
+type seekBatchReader[T any] interface {
+       ReadBatch(int64, []T, []int16, []int16) (int64, int, error)
+}
+
+// readRows reads n flat values, returning nil entries for nulls.
+func readRows[T any](t *testing.T, r seekBatchReader[T], n int, maxDef int16) 
[]*T {
+       t.Helper()
+       vals := make([]T, n)
+       defs := make([]int16, n)
+       got, nvals, err := r.ReadBatch(int64(n), vals, defs, nil)
+       require.NoError(t, err)
+       require.EqualValues(t, n, got)
+
+       out := make([]*T, n)
+       next := 0
+       for i := range out {
+               if maxDef == 0 || defs[i] == maxDef {
+                       out[i] = &vals[next]
+                       next++
+               }
+       }
+       require.Equal(t, nvals, next)
+       return out
+}
+
+// Both files hold dictionary encoded columns whose chunks have no
+// dictionary_page_offset, so the dictionary page sits at data_page_offset.
+func TestSeekToRowDictionaryAtDataPageOffset(t *testing.T) {
+       dir := os.Getenv("PARQUET_TEST_DATA")
+       if dir == "" {
+               dir = "../../parquet-testing/data"
+               t.Log("PARQUET_TEST_DATA not set, using 
../../parquet-testing/data")
+       }
+       require.DirExists(t, dir)
+
+       t.Run("offset index", func(t *testing.T) {
+               fr, err := file.OpenParquetFile(filepath.Join(dir, 
"alltypes_tiny_pages.parquet"), false)
+               require.NoError(t, err)
+               defer fr.Close()
+               rg := fr.RowGroup(0)
+
+               const col, nrows = 2, 100
+               maxDef := fr.MetaData().Schema.Column(col).MaxDefinitionLevel()
+               open := func() *file.Int32ColumnChunkReader {
+                       cr, err := rg.Column(col)
+                       require.NoError(t, err)
+                       return cr.(*file.Int32ColumnChunkReader)
+               }
+               want := readRows[int32](t, open(), nrows, maxDef)
+
+               // Reading past the seek target crosses data pages, which only 
works if the
+               // dictionary load left the page stream where it was.
+               cr := open()
+               require.NoError(t, cr.SeekToRow(0))
+               require.Equal(t, want, readRows[int32](t, cr, nrows, maxDef))
+       })
+
+       t.Run("no offset index", func(t *testing.T) {
+               fr, err := file.OpenParquetFile(filepath.Join(dir, 
"datapage_v2.snappy.parquet"), false)
+               require.NoError(t, err)
+               defer fr.Close()
+               rg := fr.RowGroup(0)
+
+               const col, nrows = 2, 5
+               maxDef := fr.MetaData().Schema.Column(col).MaxDefinitionLevel()
+               open := func() *file.Float64ColumnChunkReader {
+                       cr, err := rg.Column(col)
+                       require.NoError(t, err)
+                       return cr.(*file.Float64ColumnChunkReader)
+               }
+               want := readRows[float64](t, open(), nrows, maxDef)
+
+               t.Run("fresh reader", func(t *testing.T) {
+                       cr := open()
+                       require.NoError(t, cr.SeekToRow(4))
+                       require.Equal(t, want[4:], readRows[float64](t, cr, 1, 
maxDef))
+               })
+
+               t.Run("reseek after read", func(t *testing.T) {
+                       cr := open()
+                       readRows[float64](t, cr, nrows, maxDef)
+                       require.NoError(t, cr.SeekToRow(0))
+                       require.Equal(t, want, readRows[float64](t, cr, nrows, 
maxDef))
+               })
+       })
+}
+
+func sharedOffsetFile(t *testing.T, values []int32, pageIndex bool) []byte {
+       t.Helper()
+       sc := schema.MustGroup(schema.NewGroupNode("schema", 
parquet.Repetitions.Required, schema.FieldList{
+               schema.NewInt32Node("x", parquet.Repetitions.Required, -1),
+       }, -1))
+       var out bytes.Buffer
+       w, err := file.NewParquetWriterWithError(&out, sc, 
file.WithWriterProps(parquet.NewWriterProperties(
+               parquet.WithDictionaryDefault(true),
+               parquet.WithDictionaryPageSizeLimit(1<<20),
+               parquet.WithDictionaryCostFallbackFor("x", false),
+               parquet.WithDataPageSize(256),
+               parquet.WithBatchSize(50),
+               parquet.WithPageIndexEnabled(pageIndex))))
+       require.NoError(t, err)
+       rg := w.AppendRowGroup()
+       cw, err := rg.NextColumn()
+       require.NoError(t, err)
+       _, err = cw.(*file.Int32ColumnChunkWriter).WriteBatch(values, nil, nil)
+       require.NoError(t, err)
+       require.NoError(t, cw.Close())
+       require.NoError(t, rg.Close())
+       require.NoError(t, w.Close())
+
+       data := out.Bytes()
+       footerStart := len(data) - 8 - 
int(binary.LittleEndian.Uint32(data[len(data)-8:]))
+       md := format.NewFileMetaData()
+       _, err = thrift.DeserializeThrift(md, data[footerStart:len(data)-8])
+       require.NoError(t, err)
+       chunk := md.RowGroups[0].Columns[0].MetaData
+       require.True(t, chunk.IsSetDictionaryPageOffset())
+       require.Greater(t, chunk.DataPageOffset, 
chunk.GetDictionaryPageOffset())
+       chunk.DataPageOffset = chunk.GetDictionaryPageOffset()
+
+       var footer bytes.Buffer
+       _, err = thrift.NewThriftSerializer().Serialize(md, &footer, nil)
+       require.NoError(t, err)
+       var footerLen [4]byte
+       binary.LittleEndian.PutUint32(footerLen[:], uint32(footer.Len()))
+       return append(append(append(bytes.Clone(data[:footerStart]), 
footer.Bytes()...), footerLen[:]...), "PAR1"...)
+}
+
+// Unreleased parquet-mr builds between PARQUET-1850 and PARQUET-1977 wrote
+// chunks whose data_page_offset equals dictionary_page_offset, so the
+// dictionary page sits at both offsets and a seek has to read it from there.
+func TestSeekToRowDictionaryPageOffsetEqualsDataPageOffset(t *testing.T) {
+       const nrows = 2000
+       values := make([]int32, nrows)
+       for i := range values {
+               values[i] = int32((i * 37) % 1000)
+       }
+
+       for _, pageIndex := range []bool{false, true} {
+               t.Run(fmt.Sprintf("offset index %v", pageIndex), func(t 
*testing.T) {
+                       rdr, err := 
file.NewParquetReader(bytes.NewReader(sharedOffsetFile(t, values, pageIndex)))
+                       require.NoError(t, err)
+                       t.Cleanup(func() { require.NoError(t, rdr.Close()) })
+                       open := func() *file.Int32ColumnChunkReader {
+                               cr, err := rdr.RowGroup(0).Column(0)
+                               require.NoError(t, err)
+                               t.Cleanup(func() { require.NoError(t, 
cr.Close()) })
+                               return cr.(*file.Int32ColumnChunkReader)
+                       }
+                       read := func(cr *file.Int32ColumnChunkReader, n int) 
[]int32 {
+                               got := make([]int32, n)
+                               read, _, err := cr.ReadBatch(int64(n), got, 
nil, nil)
+                               require.NoError(t, err)
+                               require.EqualValues(t, n, read)
+                               return got
+                       }
+
+                       require.Equal(t, values, read(open(), nrows))
+
+                       const row = 150
+                       cr := open()
+                       require.NoError(t, cr.SeekToRow(row))
+                       require.Equal(t, values[row:row+250], read(cr, 250))
+
+                       if pageIndex {
+                               return
+                       }
+                       t.Run("seek to row 0 after a read", func(t *testing.T) {
+                               cr := open()
+                               read(cr, 10)
+                               require.NoError(t, cr.SeekToRow(0))
+                               require.Equal(t, values[:100], read(cr, 100))
+                       })
+               })
+       }
+}
+
+// The dictionary page offset of each column chunk in this file is past its
+// data page offset of 0.
+func TestGetDictionaryPageOffsetPastDataPageOffset(t *testing.T) {
+       dir := os.Getenv("PARQUET_TEST_DATA")
+       if dir == "" {
+               dir = "../../parquet-testing/data"
+               t.Log("PARQUET_TEST_DATA not set, using 
../../parquet-testing/data")
+       }
+       require.DirExists(t, dir)
+
+       fr, err := file.OpenParquetFile(filepath.Join(dir, 
"column_chunk_key_value_metadata.parquet"), false)
+       require.NoError(t, err)
+       defer fr.Close()
+
+       for col := range fr.MetaData().Schema.NumColumns() {

Review Comment:
   Range over an integer is valid since Go 1.22, and the module requires Go 
1.26. The test compiles and passes.



##########
parquet/pqarrow/file_reader_test.go:
##########
@@ -670,6 +670,44 @@ func TestRecordReaderSeekToRowMultiRowGroup(t *testing.T) {
        assert.Truef(t, array.RecordEqual(first, rec), "expected: %s\ngot: %s", 
first, rec)
 }
 
+// string_col in alltypes_tiny_pages is dictionary encoded and its chunk has no
+// dictionary_page_offset.
+func TestRecordReaderSeekToRowDictionaryAtDataPageOffset(t *testing.T) {
+       mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+       defer mem.AssertSize(t, 0)
+
+       pf, err := file.OpenParquetFile(path.Join(getDataDir(), 
"alltypes_tiny_pages.parquet"), false,
+               file.WithReadProps(parquet.NewReaderProperties(mem)))
+       require.NoError(t, err)
+       defer pf.Close()
+
+       fr, err := pqarrow.NewFileReader(pf, 
pqarrow.ArrowReadProperties{BatchSize: 100}, mem)
+       require.NoError(t, err)
+
+       const stringCol, batch = 9, 36
+       newReader := func() pqarrow.RecordReader {
+               rr, err := fr.GetRecordReader(context.Background(), 
[]int{stringCol}, nil)
+               require.NoError(t, err)
+               return rr
+       }
+
+       seq := newReader()
+       defer seq.Release()
+       for range batch {

Review Comment:
   Range over an integer constant is valid since Go 1.22, and the module 
requires Go 1.26.



##########
parquet/file/page_reader.go:
##########
@@ -737,17 +738,40 @@ func extractStats(dataHeader dataheader) (pageStats 
metadata.EncodedStatistics)
        return
 }
 
+// dictionaryAtDataOffset reports whether the page at data_page_offset may be a
+// dictionary page, which is so when the chunk records no dictionary offset or
+// the same offset as the data offset. Encrypted chunks are excluded because 
the
+// AAD of a page header depends on the page type, which is unknown before the
+// header is read.
+func (p *serializedPageReader) dictionaryAtDataOffset() bool {
+       return p.dataOffset <= p.baseOffset && p.cryptoCtx.MetaDecryptor == nil
+}
+
 func (p *serializedPageReader) GetDictionaryPage() (*DictionaryPage, error) {
-       if p.dictOffset > 0 {
+       probe := p.dictOffset == 0 && p.dictionaryAtDataOffset()
+       if p.dictOffset > 0 || probe {
+               span := p.dataOffset - p.baseOffset
+               readBufSize := int(min(span, int64(p.r.BufferSize())))
+               // The dictionary page starts the stream, and the data offset 
marks its end
+               // only when it is past the dictionary offset. Otherwise the 
length is
+               // unknown, as it is when no dictionary offset is recorded. 
Buffer only a
+               // page header, since the stream of a default reader holds the 
whole chunk.
+               if span <= 0 {
+                       span = math.MaxInt64
+                       readBufSize = min(p.r.BufferSize(), 
defaultPageHeaderSize)
+               }
                hdr := format.NewPageHeader()
-               readBufSize := min(int(p.dataOffset-p.baseOffset), 
p.r.BufferSize())
-               rd := utils.NewBufferedReader(
-                       io.NewSectionReader(p.r.Outer(), 
p.dictOffset-p.baseOffset, p.dataOffset-p.baseOffset),
-                       readBufSize)
+               rd := utils.NewBufferedReader(io.NewSectionReader(p.r.Outer(), 
0, span), readBufSize)

Review Comment:
   `p.r.Outer()` is the column chunk's own stream, a section reader or a byte 
reader that starts at the chunk, so offset 0 is the chunk start and reads stop 
at the chunk end. Only a page header is buffered.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to