hcrosse commented on code in PR #1594:
URL: https://github.com/apache/arrow-go/pull/1594#discussion_r4607665756
##########
parquet/file/page_reader_internal_test.go:
##########
@@ -247,6 +247,28 @@ func
TestSeekToRowZeroOnEncryptedChunkStartingWithDictionaryPage(t *testing.T) {
require.Equal(t, []int32{10, 20, 20, 10, 10, 10, 20, 10}, vals[:n])
}
+// With data_page_offset equal to dictionary_page_offset a seek goes back to
the
+// dictionary page, whose header takes the dictionary AAD again.
+func TestReseekOnEncryptedChunkWithSharedPageOffset(t *testing.T) {
+ // RLE_DICTIONARY indices 0,1,1,0,0,0,1,0 at bit width 1
+ indices := []byte{1, 3, 0x46}
+ p := encryptedChunk(t, true, 8, dictPageSpec(10, 20), dataPageSpec(8,
format.Encoding_RLE_DICTIONARY, indices))
Review Comment:
The test now runs as two subtests, `SharedPageOffset` with the offsets set
and `NoDictionaryOffset` without them. `NoDictionaryOffset` fails on the
previous head with `cipher: message authentication failed`, and with only the
seek change it fails with `column chunk cannot have more than one dictionary`.
##########
parquet/pqarrow/file_reader_test.go:
##########
@@ -708,6 +708,70 @@ func TestRecordReaderSeekToRowDictionaryAtDataPageOffset(t
*testing.T) {
require.Truef(t, array.RecordEqual(want, got), "expected: %s\ngot: %s",
want, got)
}
+// A record reader builds a new page reader for each seek, which starts at the
+// dictionary page of an encrypted chunk.
+func TestRecordReaderSeekToRowEncryptedDictionary(t *testing.T) {
+ const (
+ footerKey = "0123456789012345"
+ rowGroups = 2
+ rowsPerRG = 1500
+ row = 2200
+ )
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ bldr := array.NewInt64Builder(mem)
+ defer bldr.Release()
+ for i := range rowGroups * rowsPerRG {
+ bldr.Append(int64((i*31 + i/rowsPerRG) % 97))
+ }
+ arr := bldr.NewArray()
+ defer arr.Release()
+ sc := arrow.NewSchema([]arrow.Field{{Name: "x", Type:
arrow.PrimitiveTypes.Int64}}, nil)
+ tbl := array.NewTableFromSlice(sc, [][]arrow.Array{{arr}})
+ defer tbl.Release()
+
+ var buf bytes.Buffer
+ require.NoError(t, pqarrow.WriteTable(tbl, &buf, rowsPerRG,
parquet.NewWriterProperties(
+
parquet.WithEncryptionProperties(parquet.NewFileEncryptionProperties(footerKey)),
+ parquet.WithDictionaryDefault(true),
+ parquet.WithDictionaryCostFallbackFor("x", false),
+ parquet.WithDictionaryPageSizeLimit(1<<20),
+ parquet.WithDataPageSize(400),
+ parquet.WithBatchSize(16)),
+ pqarrow.NewArrowWriterProperties(pqarrow.WithAllocator(mem))))
+
+ props := parquet.NewReaderProperties(mem)
+ props.FileDecryptProps =
parquet.NewFileDecryptionProperties(parquet.WithFooterKey(footerKey))
+ pf, err := file.NewParquetReader(bytes.NewReader(buf.Bytes()),
file.WithReadProps(props))
+ require.NoError(t, err)
+ defer pf.Close()
Review Comment:
`defer pf.Close()` matches the other tests in this file. The record from
`rr.Read()` belongs to the reader, which releases it on the next `Read` or on
`Release`, and the checked allocator ends at zero.
##########
parquet/pqarrow/file_reader_test.go:
##########
@@ -708,6 +708,70 @@ func TestRecordReaderSeekToRowDictionaryAtDataPageOffset(t
*testing.T) {
require.Truef(t, array.RecordEqual(want, got), "expected: %s\ngot: %s",
want, got)
}
+// A record reader builds a new page reader for each seek, which starts at the
+// dictionary page of an encrypted chunk.
+func TestRecordReaderSeekToRowEncryptedDictionary(t *testing.T) {
+ const (
+ footerKey = "0123456789012345"
+ rowGroups = 2
+ rowsPerRG = 1500
+ row = 2200
+ )
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ bldr := array.NewInt64Builder(mem)
+ defer bldr.Release()
+ for i := range rowGroups * rowsPerRG {
+ bldr.Append(int64((i*31 + i/rowsPerRG) % 97))
+ }
+ arr := bldr.NewArray()
+ defer arr.Release()
+ sc := arrow.NewSchema([]arrow.Field{{Name: "x", Type:
arrow.PrimitiveTypes.Int64}}, nil)
+ tbl := array.NewTableFromSlice(sc, [][]arrow.Array{{arr}})
+ defer tbl.Release()
+
+ var buf bytes.Buffer
+ require.NoError(t, pqarrow.WriteTable(tbl, &buf, rowsPerRG,
parquet.NewWriterProperties(
+
parquet.WithEncryptionProperties(parquet.NewFileEncryptionProperties(footerKey)),
+ parquet.WithDictionaryDefault(true),
+ parquet.WithDictionaryCostFallbackFor("x", false),
+ parquet.WithDictionaryPageSizeLimit(1<<20),
+ parquet.WithDataPageSize(400),
+ parquet.WithBatchSize(16)),
+ pqarrow.NewArrowWriterProperties(pqarrow.WithAllocator(mem))))
+
+ props := parquet.NewReaderProperties(mem)
+ props.FileDecryptProps =
parquet.NewFileDecryptionProperties(parquet.WithFooterKey(footerKey))
+ pf, err := file.NewParquetReader(bytes.NewReader(buf.Bytes()),
file.WithReadProps(props))
+ require.NoError(t, err)
+ defer pf.Close()
+ chunk, err := pf.MetaData().RowGroup(rowGroups - 1).ColumnChunk(0)
+ require.NoError(t, err)
+ require.True(t, chunk.HasDictionaryPage())
+
+ fr, err := pqarrow.NewFileReader(pf,
pqarrow.ArrowReadProperties{BatchSize: 100}, mem)
+ require.NoError(t, err)
+ seq, err := fr.ReadTable(context.Background())
+ require.NoError(t, err)
+ defer seq.Release()
+ var want []int64
+ for _, c := range seq.Column(0).Data().Chunks() {
+ want = append(want, c.(*array.Int64).Int64Values()...)
+ }
+ require.Len(t, want, rowGroups*rowsPerRG)
+
+ rr, err := fr.GetRecordReader(context.Background(), []int{0}, nil)
+ require.NoError(t, err)
+ defer rr.Release()
+ require.NoError(t, rr.SeekToRow(row))
+ rec, err := rr.Read()
+ require.NoError(t, err)
+ got := rec.Column(0).(*array.Int64).Int64Values()
Review Comment:
`Read` returns the reader's current record, which the reader releases on the
next `Read` or on `Release`, so the test doesn't release it. The checked
allocator ends at zero.
##########
parquet/encryption_read_config_test.go:
##########
@@ -622,6 +622,38 @@ func (d *TestDecryptionSuite) TestDecryption() {
}
}
+// uniform_encryption.parquet.encrypted was written by Arrow C++, with a
+// dictionary page and an offset index in each chunk.
+func TestSeekToRowEncryptedTestData(t *testing.T) {
+ props := parquet.NewReaderProperties(memory.DefaultAllocator)
+ props.FileDecryptProps =
parquet.NewFileDecryptionProperties(parquet.WithFooterKey(FooterEncryptionKey))
+ rdr, err := file.OpenParquetFile(path.Join(getDataDir(),
"uniform_encryption.parquet.encrypted"), false, file.WithReadProps(props))
+ require.NoError(t, err)
+ defer rdr.Close()
Review Comment:
This follows the other tests in this file, which close readers with `defer`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]