This is an automated email from the ASF dual-hosted git repository.
ggershinsky pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-java.git
The following commit(s) were added to refs/heads/master by this push:
new a7e78d8b1 Add parameter to enable reading KMS URL from key material
(#3745)
a7e78d8b1 is described below
commit a7e78d8b154d0ce36684895c2c8ba578bd9d00fb
Author: ggershinsky <[email protected]>
AuthorDate: Mon Aug 31 11:17:39 2026 +0300
Add parameter to enable reading KMS URL from key material (#3745)
* inital commit
* spotless
* update readme
* cleanup
* add unitest
* update junit
* clean up
---
parquet-hadoop/README.md | 10 ++
.../parquet/crypto/keytools/FileKeyUnwrapper.java | 9 +-
.../apache/parquet/crypto/keytools/KeyToolkit.java | 11 ++
.../org/apache/parquet/crypto/TestKmsUrlRead.java | 197 +++++++++++++++++++++
4 files changed, 224 insertions(+), 3 deletions(-)
diff --git a/parquet-hadoop/README.md b/parquet-hadoop/README.md
index 4826d49e2..51fd34b0c 100644
--- a/parquet-hadoop/README.md
+++ b/parquet-hadoop/README.md
@@ -469,6 +469,16 @@ If `false`, write files in encrypted footer mode, that
fully encrypts the footer
---
+**Property:** `parquet.encryption.kms.enable.url.read`
+**Description:** If a KMS URL is set by writers, it will be stored in the key
material. However, by default, it will not be
+given to readers because the storage is untrusted. Readers that need the URL
should set the KMS URL property.
+If they cannot do so, they can enable retrieving the KMS URL from the stored
key material by setting this parameter
+to `true`. KMS client implementations must validate the URL value and use
authentication to prevent key material
+tampering attacks that could, for example, result in a KMS access token being
sent to a malicious URL endpoint.
+**Default value:** `false`
+
+---
+
**Property:** `parquet.encryption.key.access.token`
**Description:** Authorization token that will be passed to KMS.
**Default value:** `DEFAULT`
diff --git
a/parquet-hadoop/src/main/java/org/apache/parquet/crypto/keytools/FileKeyUnwrapper.java
b/parquet-hadoop/src/main/java/org/apache/parquet/crypto/keytools/FileKeyUnwrapper.java
index c3eb97875..b681187de 100644
---
a/parquet-hadoop/src/main/java/org/apache/parquet/crypto/keytools/FileKeyUnwrapper.java
+++
b/parquet-hadoop/src/main/java/org/apache/parquet/crypto/keytools/FileKeyUnwrapper.java
@@ -157,10 +157,13 @@ public class FileKeyUnwrapper implements
DecryptionKeyRetriever {
String kmsInstanceURL =
hadoopConfiguration.getTrimmed(KeyToolkit.KMS_INSTANCE_URL_PROPERTY_NAME);
if (stringIsEmpty(kmsInstanceURL)) {
- kmsInstanceURL = keyMaterial.getKmsInstanceURL();
+ if (hadoopConfiguration.getBoolean(
+ KeyToolkit.KMS_ENABLE_URL_READ_PROPERTY_NAME,
KeyToolkit.KMS_ENABLE_URL_READ_DEFAULT)) {
+ kmsInstanceURL = keyMaterial.getKmsInstanceURL();
+ }
+
if (null == kmsInstanceURL) {
- throw new ParquetCryptoRuntimeException(
- "KMS instance URL is missing both in properties and file key
material");
+ kmsInstanceURL = KmsClient.KMS_INSTANCE_URL_DEFAULT;
}
}
diff --git
a/parquet-hadoop/src/main/java/org/apache/parquet/crypto/keytools/KeyToolkit.java
b/parquet-hadoop/src/main/java/org/apache/parquet/crypto/keytools/KeyToolkit.java
index c53b65afb..854976d37 100644
---
a/parquet-hadoop/src/main/java/org/apache/parquet/crypto/keytools/KeyToolkit.java
+++
b/parquet-hadoop/src/main/java/org/apache/parquet/crypto/keytools/KeyToolkit.java
@@ -52,6 +52,17 @@ public class KeyToolkit {
* URL of the KMS instance.
*/
public static final String KMS_INSTANCE_URL_PROPERTY_NAME =
"parquet.encryption.kms.instance.url";
+ /**
+ * If a KMS URL is set by writers, it will be stored in the key material.
However, by default, it will not be
+ * provided to readers because the storage is untrusted. Readers that need
the URL should set the
+ * KMS URL property. If they cannot do so, they can enable retrieving the
KMS URL from the stored material
+ * by setting this parameter to true. KMS client implementations must
validate the URL value and
+ * use authentication to prevent key material tampering attacks, which
could, for example, send a KMS
+ * access token to a malicious URL endpoint.
+ */
+ public static final String KMS_ENABLE_URL_READ_PROPERTY_NAME =
"parquet.encryption.kms.enable.url.read";
+
+ public static final boolean KMS_ENABLE_URL_READ_DEFAULT = false;
/**
* Authorization token that will be passed to KMS.
*/
diff --git
a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestKmsUrlRead.java
b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestKmsUrlRead.java
new file mode 100644
index 000000000..5d3d3e04f
--- /dev/null
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestKmsUrlRead.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet.crypto;
+
+import static org.apache.parquet.hadoop.ParquetFileWriter.Mode.OVERWRITE;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.crypto.keytools.KeyToolkit;
+import org.apache.parquet.crypto.keytools.KmsClient;
+import org.apache.parquet.crypto.keytools.PropertiesDrivenCryptoFactory;
+import org.apache.parquet.crypto.keytools.mocks.InMemoryKMS;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.example.data.simple.SimpleGroupFactory;
+import org.apache.parquet.hadoop.ParquetReader;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.example.ExampleParquetWriter;
+import org.apache.parquet.hadoop.example.GroupReadSupport;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.MessageType;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+public class TestKmsUrlRead {
+
+ private static final List<SingleRow> DATA =
Collections.unmodifiableList(SingleRow.generateRandomData(5000));
+ private static final String UNIFORM_MASTER_KEY =
+
Base64.getEncoder().encodeToString("0123456789012346".getBytes(StandardCharsets.UTF_8));
+ private static final String UNIFORM_MASTER_KEY_ID = "ku";
+ private static final String KEY_LIST = UNIFORM_MASTER_KEY_ID + ": " +
UNIFORM_MASTER_KEY;
+ private static final String STORED_KMS_URL = "stored-kms-url";
+
+ private static Path filePath;
+
+ public static class UnitestUrlReadKMS extends InMemoryKMS {
+ private static String staticKmsURL;
+
+ @Override
+ public synchronized void initialize(
+ Configuration configuration, String kmsInstanceID, String
kmsInstanceURL, String accessToken) {
+ staticKmsURL = kmsInstanceURL;
+ super.initialize(configuration, kmsInstanceID, kmsInstanceURL,
accessToken);
+ }
+
+ static String getStaticKmsURL() {
+ return staticKmsURL;
+ }
+ }
+
+ @BeforeAll
+ public static void writeEncryptedFile() throws IOException {
+ Configuration writeConf = new Configuration();
+ writeConf.set(
+ EncryptionPropertiesFactory.CRYPTO_FACTORY_CLASS_PROPERTY_NAME,
+ PropertiesDrivenCryptoFactory.class.getName());
+ writeConf.set(PropertiesDrivenCryptoFactory.UNIFORM_KEY_PROPERTY_NAME,
UNIFORM_MASTER_KEY_ID);
+ writeConf.set(KeyToolkit.KMS_CLIENT_CLASS_PROPERTY_NAME,
InMemoryKMS.class.getName());
+ writeConf.set(KeyToolkit.KMS_INSTANCE_URL_PROPERTY_NAME, STORED_KMS_URL);
+ writeConf.set(InMemoryKMS.KEY_LIST_PROPERTY_NAME, KEY_LIST);
+ writeConf.set(KeyToolkit.KEY_ACCESS_TOKEN_PROPERTY_NAME, "writer-token");
+
+ filePath = new Path(Files.createTempFile("test-kms-url_", ".parquet")
+ .toAbsolutePath()
+ .toString());
+
+ MessageType schema = SingleRow.getSchema();
+ SimpleGroupFactory f = new SimpleGroupFactory(schema);
+
+ try (ParquetWriter<Group> writer = ExampleParquetWriter.builder(filePath)
+ .withConf(writeConf)
+ .withWriteMode(OVERWRITE)
+ .withType(schema)
+ .build()) {
+
+ for (SingleRow singleRow : DATA) {
+ writer.write(f.newGroup()
+ .append(SingleRow.BOOLEAN_FIELD_NAME, singleRow.boolean_field)
+ .append(SingleRow.INT32_FIELD_NAME, singleRow.int32_field)
+ .append(SingleRow.FLOAT_FIELD_NAME, singleRow.float_field)
+ .append(SingleRow.DOUBLE_FIELD_NAME, singleRow.double_field)
+ .append(SingleRow.BINARY_FIELD_NAME,
Binary.fromConstantByteArray(singleRow.ba_field))
+ .append(
+ SingleRow.FIXED_LENGTH_BINARY_FIELD_NAME,
+ Binary.fromConstantByteArray(singleRow.flba_field))
+ .append(SingleRow.PLAINTEXT_INT32_FIELD_NAME,
singleRow.plaintext_int32_field));
+ }
+ }
+ }
+
+ @Test
+ public void testReadWithoutKeys() throws IOException {
+ Configuration readConf = new Configuration();
+ try (ParquetReader<Group> reader = ParquetReader.builder(new
GroupReadSupport(), filePath)
+ .withConf(readConf)
+ .build()) {
+ assertThatThrownBy(reader::read)
+ .isInstanceOf(ParquetCryptoRuntimeException.class)
+ .hasMessageContaining("Trying to read file with encrypted footer. No
keys available");
+ }
+ }
+
+ @Test
+ public void testDefaultKmsUrl() throws IOException {
+ // Reader with basic decryption properties
+ Configuration readConf = basicDecryptionConfig();
+ // triggers creation of new kms client instance
+ readConf.set(KeyToolkit.KEY_ACCESS_TOKEN_PROPERTY_NAME, "reader1-token");
+
+ try (ParquetReader<Group> reader = ParquetReader.builder(new
GroupReadSupport(), filePath)
+ .withConf(readConf)
+ .build()) {
+ reader.read();
+ }
+
+ // Make sure KMS URL is the default string (not taken from storage).
+
assertThat(KmsClient.KMS_INSTANCE_ID_DEFAULT.equals(UnitestUrlReadKMS.getStaticKmsURL()));
+ }
+
+ @Test
+ public void testStoredKmsUrl() throws IOException {
+ Configuration readConf = basicDecryptionConfig();
+
+ // Enable reading KMS URL from storage
+ readConf.set(KeyToolkit.KMS_ENABLE_URL_READ_PROPERTY_NAME, "true");
+ readConf.set(KeyToolkit.KEY_ACCESS_TOKEN_PROPERTY_NAME, "reader2-token");
+
+ try (ParquetReader<Group> reader = ParquetReader.builder(new
GroupReadSupport(), filePath)
+ .withConf(readConf)
+ .build()) {
+ reader.read();
+ }
+
+ // Verify the stored value
+ assertThat(STORED_KMS_URL.equals(UnitestUrlReadKMS.getStaticKmsURL()));
+ }
+
+ @Test
+ public void testSetKmsUrl() throws IOException {
+ Configuration readConf = basicDecryptionConfig();
+
+ // Set KMS URL value in the reader
+ String readerSetURL = "reader-set-kms-url";
+ readConf.set(KeyToolkit.KMS_INSTANCE_URL_PROPERTY_NAME, readerSetURL);
+ readConf.set(KeyToolkit.KEY_ACCESS_TOKEN_PROPERTY_NAME, "reader3-token");
+
+ try (ParquetReader<Group> reader = ParquetReader.builder(new
GroupReadSupport(), filePath)
+ .withConf(readConf)
+ .build()) {
+ reader.read();
+ }
+
+ // Verify the set value
+ assertThat(readerSetURL.equals(UnitestUrlReadKMS.getStaticKmsURL()));
+ }
+
+ @AfterAll
+ public static void deleteFile() throws IOException {
+ filePath.getFileSystem(new Configuration()).delete(filePath, false);
+ }
+
+ private Configuration basicDecryptionConfig() {
+ Configuration readConf = new Configuration();
+ readConf.set(
+ EncryptionPropertiesFactory.CRYPTO_FACTORY_CLASS_PROPERTY_NAME,
+ PropertiesDrivenCryptoFactory.class.getName());
+ readConf.set(KeyToolkit.KMS_CLIENT_CLASS_PROPERTY_NAME,
UnitestUrlReadKMS.class.getName());
+ readConf.set(InMemoryKMS.KEY_LIST_PROPERTY_NAME, KEY_LIST);
+
+ return readConf;
+ }
+}
