Copilot commented on code in PR #1594:
URL: https://github.com/apache/arrow-go/pull/1594#discussion_r4603387780
##########
parquet/pqarrow/file_reader_test.go:
##########
@@ -708,6 +708,70 @@ func TestRecordReaderSeekToRowDictionaryAtDataPageOffset(t
*testing.T) {
require.Truef(t, array.RecordEqual(want, got), "expected: %s\ngot: %s",
want, got)
}
+// A record reader builds a new page reader for each seek, which starts at the
+// dictionary page of an encrypted chunk.
+func TestRecordReaderSeekToRowEncryptedDictionary(t *testing.T) {
+ const (
+ footerKey = "0123456789012345"
+ rowGroups = 2
+ rowsPerRG = 1500
+ row = 2200
+ )
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ bldr := array.NewInt64Builder(mem)
+ defer bldr.Release()
+ for i := range rowGroups * rowsPerRG {
+ bldr.Append(int64((i*31 + i/rowsPerRG) % 97))
+ }
+ arr := bldr.NewArray()
+ defer arr.Release()
+ sc := arrow.NewSchema([]arrow.Field{{Name: "x", Type:
arrow.PrimitiveTypes.Int64}}, nil)
+ tbl := array.NewTableFromSlice(sc, [][]arrow.Array{{arr}})
+ defer tbl.Release()
+
+ var buf bytes.Buffer
+ require.NoError(t, pqarrow.WriteTable(tbl, &buf, rowsPerRG,
parquet.NewWriterProperties(
+
parquet.WithEncryptionProperties(parquet.NewFileEncryptionProperties(footerKey)),
+ parquet.WithDictionaryDefault(true),
+ parquet.WithDictionaryCostFallbackFor("x", false),
+ parquet.WithDictionaryPageSizeLimit(1<<20),
+ parquet.WithDataPageSize(400),
+ parquet.WithBatchSize(16)),
+ pqarrow.NewArrowWriterProperties(pqarrow.WithAllocator(mem))))
+
+ props := parquet.NewReaderProperties(mem)
+ props.FileDecryptProps =
parquet.NewFileDecryptionProperties(parquet.WithFooterKey(footerKey))
+ pf, err := file.NewParquetReader(bytes.NewReader(buf.Bytes()),
file.WithReadProps(props))
+ require.NoError(t, err)
+ defer pf.Close()
Review Comment:
`pf.Close()` errors are currently ignored, and the `arrow.Record` returned
from `rr.Read()` is not released. In Arrow Go, records are reference-counted;
missing `rec.Release()` can leak memory and make allocator-based assertions
flaky. Prefer `t.Cleanup(func() { require.NoError(t, pf.Close()) })` (or `defer
func(){...}()`) and `defer rec.Release()` after a successful read.
##########
parquet/pqarrow/file_reader_test.go:
##########
@@ -708,6 +708,70 @@ func TestRecordReaderSeekToRowDictionaryAtDataPageOffset(t
*testing.T) {
require.Truef(t, array.RecordEqual(want, got), "expected: %s\ngot: %s",
want, got)
}
+// A record reader builds a new page reader for each seek, which starts at the
+// dictionary page of an encrypted chunk.
+func TestRecordReaderSeekToRowEncryptedDictionary(t *testing.T) {
+ const (
+ footerKey = "0123456789012345"
+ rowGroups = 2
+ rowsPerRG = 1500
+ row = 2200
+ )
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ bldr := array.NewInt64Builder(mem)
+ defer bldr.Release()
+ for i := range rowGroups * rowsPerRG {
+ bldr.Append(int64((i*31 + i/rowsPerRG) % 97))
+ }
+ arr := bldr.NewArray()
+ defer arr.Release()
+ sc := arrow.NewSchema([]arrow.Field{{Name: "x", Type:
arrow.PrimitiveTypes.Int64}}, nil)
+ tbl := array.NewTableFromSlice(sc, [][]arrow.Array{{arr}})
+ defer tbl.Release()
+
+ var buf bytes.Buffer
+ require.NoError(t, pqarrow.WriteTable(tbl, &buf, rowsPerRG,
parquet.NewWriterProperties(
+
parquet.WithEncryptionProperties(parquet.NewFileEncryptionProperties(footerKey)),
+ parquet.WithDictionaryDefault(true),
+ parquet.WithDictionaryCostFallbackFor("x", false),
+ parquet.WithDictionaryPageSizeLimit(1<<20),
+ parquet.WithDataPageSize(400),
+ parquet.WithBatchSize(16)),
+ pqarrow.NewArrowWriterProperties(pqarrow.WithAllocator(mem))))
+
+ props := parquet.NewReaderProperties(mem)
+ props.FileDecryptProps =
parquet.NewFileDecryptionProperties(parquet.WithFooterKey(footerKey))
+ pf, err := file.NewParquetReader(bytes.NewReader(buf.Bytes()),
file.WithReadProps(props))
+ require.NoError(t, err)
+ defer pf.Close()
+ chunk, err := pf.MetaData().RowGroup(rowGroups - 1).ColumnChunk(0)
+ require.NoError(t, err)
+ require.True(t, chunk.HasDictionaryPage())
+
+ fr, err := pqarrow.NewFileReader(pf,
pqarrow.ArrowReadProperties{BatchSize: 100}, mem)
+ require.NoError(t, err)
+ seq, err := fr.ReadTable(context.Background())
+ require.NoError(t, err)
+ defer seq.Release()
+ var want []int64
+ for _, c := range seq.Column(0).Data().Chunks() {
+ want = append(want, c.(*array.Int64).Int64Values()...)
+ }
+ require.Len(t, want, rowGroups*rowsPerRG)
+
+ rr, err := fr.GetRecordReader(context.Background(), []int{0}, nil)
+ require.NoError(t, err)
+ defer rr.Release()
+ require.NoError(t, rr.SeekToRow(row))
+ rec, err := rr.Read()
+ require.NoError(t, err)
+ got := rec.Column(0).(*array.Int64).Int64Values()
Review Comment:
`pf.Close()` errors are currently ignored, and the `arrow.Record` returned
from `rr.Read()` is not released. In Arrow Go, records are reference-counted;
missing `rec.Release()` can leak memory and make allocator-based assertions
flaky. Prefer `t.Cleanup(func() { require.NoError(t, pf.Close()) })` (or `defer
func(){...}()`) and `defer rec.Release()` after a successful read.
##########
parquet/encryption_read_config_test.go:
##########
@@ -622,6 +622,38 @@ func (d *TestDecryptionSuite) TestDecryption() {
}
}
+// uniform_encryption.parquet.encrypted was written by Arrow C++, with a
+// dictionary page and an offset index in each chunk.
+func TestSeekToRowEncryptedTestData(t *testing.T) {
+ props := parquet.NewReaderProperties(memory.DefaultAllocator)
+ props.FileDecryptProps =
parquet.NewFileDecryptionProperties(parquet.WithFooterKey(FooterEncryptionKey))
+ rdr, err := file.OpenParquetFile(path.Join(getDataDir(),
"uniform_encryption.parquet.encrypted"), false, file.WithReadProps(props))
+ require.NoError(t, err)
+ defer rdr.Close()
+
+ const col, nrows, row = 1, 50, 24
+ chunk, err := rdr.MetaData().RowGroup(0).ColumnChunk(col)
+ require.NoError(t, err)
+ require.Equal(t, parquet.Types.Int32, chunk.Type())
+ require.True(t, chunk.HasDictionaryPage())
+
+ read := func(seekTo, n int) []int32 {
+ cr, err := rdr.RowGroup(0).Column(col)
+ require.NoError(t, err)
+ defer cr.Close()
Review Comment:
These defers ignore `Close()` errors. To avoid silently missing
close/flush/decryption issues, assert `Close()` returns `nil` (e.g., via
`t.Cleanup(func(){ require.NoError(t, rdr.Close()) })` and similarly for
`cr.Close()`).
##########
parquet/encryption_read_config_test.go:
##########
@@ -622,6 +622,38 @@ func (d *TestDecryptionSuite) TestDecryption() {
}
}
+// uniform_encryption.parquet.encrypted was written by Arrow C++, with a
+// dictionary page and an offset index in each chunk.
+func TestSeekToRowEncryptedTestData(t *testing.T) {
+ props := parquet.NewReaderProperties(memory.DefaultAllocator)
+ props.FileDecryptProps =
parquet.NewFileDecryptionProperties(parquet.WithFooterKey(FooterEncryptionKey))
+ rdr, err := file.OpenParquetFile(path.Join(getDataDir(),
"uniform_encryption.parquet.encrypted"), false, file.WithReadProps(props))
+ require.NoError(t, err)
+ defer rdr.Close()
Review Comment:
These defers ignore `Close()` errors. To avoid silently missing
close/flush/decryption issues, assert `Close()` returns `nil` (e.g., via
`t.Cleanup(func(){ require.NoError(t, rdr.Close()) })` and similarly for
`cr.Close()`).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]