Copilot commented on code in PR #51362:
URL: https://github.com/apache/arrow/pull/51362#discussion_r4034462782
##########
python/pyarrow/tests/parquet/test_encryption.py:
##########
@@ -501,45 +501,58 @@ def validate_kms_connection_config(kms_connection_config):
validate_kms_connection_config(kms_connection_config_1)
[email protected](reason="Plaintext footer - reading plaintext column subset"
- " reads encrypted columns too")
def test_encrypted_parquet_write_read_plain_footer_single_wrapping(
tempdir, data_table):
- """Write an encrypted parquet, with plaintext footer
- and with single wrapping,
- verify it's encrypted, and then read plaintext columns."""
+ """
+ Write an encrypted parquet, with plaintext footer and with single wrapping,
+ verify it's encrypted, and then read plaintext columns. Runs once with a
+ flat schema and once where the encrypted column `b` is itself a nested
+ (struct) field.
+ """
path = tempdir / PARQUET_NAME
- # Encrypt the footer with the footer key,
- # encrypt column `a` and column `b` with another key,
- # keep `c` plaintext
- encryption_config = pe.EncryptionConfiguration(
- footer_key=FOOTER_KEY_NAME,
- column_keys={
- COL_KEY_NAME: ["a", "b"],
- },
- plaintext_footer=True,
- double_wrapping=False)
+ for nested in [False, True]:
+ if nested:
+ table = pa.Table.from_pydict({
+ 'a': pa.array([1, 2, 3]),
+ 'b': pa.array(
+ [{'x': 1, 'y': 2}, {'x': 3, 'y': 4}, {'x': 5, 'y': 6}],
+ type=pa.struct([('x', pa.int32()), ('y', pa.int32())])),
+ 'c': pa.array(['x', 'y', 'z'])
+ })
+ else:
+ table = data_table
+
+ # Encrypt the footer with the footer key,
+ # encrypt column `a` and column `b` with another key, keep `c`
plaintext
+ encryption_config = pe.EncryptionConfiguration(
+ footer_key=FOOTER_KEY_NAME,
+ column_keys={
+ COL_KEY_NAME: ["a", "b"],
+ },
+ plaintext_footer=True,
+ double_wrapping=False)
- kms_connection_config = pe.KmsConnectionConfig(
- custom_kms_conf={
- FOOTER_KEY_NAME: FOOTER_KEY.decode("UTF-8"),
- COL_KEY_NAME: COL_KEY.decode("UTF-8"),
- }
- )
+ kms_connection_config = pe.KmsConnectionConfig(
+ custom_kms_conf={
+ FOOTER_KEY_NAME: FOOTER_KEY.decode("UTF-8"),
+ COL_KEY_NAME: COL_KEY.decode("UTF-8"),
+ }
+ )
- def kms_factory(kms_connection_configuration):
- return InMemoryKmsClient(kms_connection_configuration)
+ def kms_factory(kms_connection_configuration):
+ return InMemoryKmsClient(kms_connection_configuration)
- crypto_factory = pe.CryptoFactory(kms_factory)
- # Write with encryption properties
- write_encrypted_parquet(path, data_table, encryption_config,
- kms_connection_config, crypto_factory)
+ crypto_factory = pe.CryptoFactory(kms_factory)
+ # Write with encryption properties
+ write_encrypted_parquet(path, table, encryption_config,
+ kms_connection_config, crypto_factory)
- # # Read without decryption properties only the plaintext column
- # result = pq.ParquetFile(path)
- # result_table = result.read(columns='c', use_threads=False)
- # assert table.num_rows == result_table.num_rows
+ # Read the plaintext column without decryption properties
+ result = pq.ParquetFile(path)
+ result_table = result.read(columns='c', use_threads=False)
+ assert table.num_rows == result_table.num_rows
+ assert table.select(['c']).equals(result_table)
Review Comment:
This loop rewrites `path` on the second iteration while the `ParquetFile`
from the first iteration is still open: a path-backed `ParquetFile` remains
open after `read()` unless it is closed (as covered by
`test_parquet_file.py:312-316`). On Windows, the writer can therefore fail to
reopen/truncate the file. Use a `with pq.ParquetFile(path) as result:` block or
explicitly close `result` before the next iteration.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
