This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 09a7f4c389 [#12297] feat(secret): Add SecretManager and typed
SecretBinding/SecretReference (#12375)
09a7f4c389 is described below
commit 09a7f4c3891b0601edc1ecada4ce1fd05a2cacc4
Author: MaSai <[email protected]>
AuthorDate: Thu Aug 6 09:17:05 2026 +0800
[#12297] feat(secret): Add SecretManager and typed
SecretBinding/SecretReference (#12375)
### What changes were proposed in this pull request?
- Add `SecretManager` for write-through / external-ref lifecycle
(`getSecretBindingUrns`, `getSecretReferenceUrns`, `writeSecrets`,
`rollbackWritten`).
- Add `SecretPropertyUtils` for request checks, URN recognition, and
applying URNs into properties.
- Add typed `@Evolving` API models: `SecretBinding` (`provider` +
`plaintext`) and `SecretReference` (`provider` + non-null `attributes`).
- Extend `SecretProvider` with optional `bindExternalReference` (default
rejects; write-through-only providers need not override).
- Wire `SecretManager` into `GravitinoEnv` / `CatalogManager` /
`OperationDispatcher` (including hidden-property detection for secret
URNs).
- Create APIs / REST request shapes are **unchanged** in this PR
### Why are the changes needed?
Entity create needs a shared secret lifecycle layer and typed models
before public create APIs can accept `secretBindings` /
`secretReferences`.
Fix: #12297
### Does this PR introduce _any_ user-facing change?
- No create API / REST request shape change yet.
- Adds typed secret models in the public API module for follow-up create
wiring.
### How was this patch tested?
- `./gradlew :api:test --tests
org.apache.gravitino.secret.TestSecretBinding --tests
org.apache.gravitino.secret.TestSecretReference :core:test --tests
org.apache.gravitino.secret.TestSecretManager --tests
org.apache.gravitino.TestGravitinoEnvSecretProviderRegistry -PskipITs`
---------
Co-authored-by: Cursor <[email protected]>
---
api/build.gradle.kts | 2 -
.../org/apache/gravitino/secret/SecretBinding.java | 89 ++++++++
.../apache/gravitino/secret/SecretReference.java | 92 +++++++++
.../gravitino/secret/TestSecretBinding.java} | 33 ++-
.../gravitino/secret/TestSecretReference.java} | 35 ++--
.../apache/gravitino/secret/SecretProvider.java | 17 ++
.../java/org/apache/gravitino/GravitinoEnv.java | 59 ++++--
.../apache/gravitino/catalog/CatalogManager.java | 11 +-
.../catalog/FilesetOperationDispatcher.java | 9 +-
.../catalog/FunctionOperationDispatcher.java | 8 +-
.../catalog/ModelOperationDispatcher.java | 8 +-
.../gravitino/catalog/OperationDispatcher.java | 19 +-
.../catalog/PartitionOperationDispatcher.java | 9 +-
.../catalog/SchemaOperationDispatcher.java | 9 +-
.../catalog/TableOperationDispatcher.java | 20 +-
.../catalog/TopicOperationDispatcher.java | 9 +-
.../gravitino/catalog/ViewOperationDispatcher.java | 20 +-
.../credential/CredentialOperationDispatcher.java | 8 +-
.../org/apache/gravitino/secret/SecretManager.java | 230 +++++++++++++++++++++
.../gravitino/secret/SecretPropertyUtils.java | 118 +++++++++++
.../TestGravitinoEnvSecretProviderRegistry.java | 26 ++-
.../gravitino/catalog/TestCatalogManager.java | 13 +-
.../catalog/TestCatalogNormalizeDispatcher.java | 10 +-
.../catalog/TestClassLoaderPoolIntegration.java | 16 +-
.../catalog/TestFilesetOperationDispatcher.java | 4 +-
.../catalog/TestFunctionOperationDispatcher.java | 5 +-
.../catalog/TestModelOperationDispatcher.java | 4 +-
.../gravitino/catalog/TestOperationDispatcher.java | 6 +-
.../catalog/TestPartitionOperationDispatcher.java | 6 +-
.../catalog/TestSchemaOperationDispatcher.java | 3 +-
.../catalog/TestTableOperationDispatcher.java | 14 +-
.../catalog/TestTopicOperationDispatcher.java | 4 +-
.../catalog/TestViewOperationDispatcher.java | 14 +-
.../TestCredentialOperationDispatcher.java | 2 +-
.../apache/gravitino/secret/TestSecretManager.java | 110 ++++++++++
35 files changed, 919 insertions(+), 123 deletions(-)
diff --git a/api/build.gradle.kts b/api/build.gradle.kts
index f0fe3ba5ee..b4399b13c0 100644
--- a/api/build.gradle.kts
+++ b/api/build.gradle.kts
@@ -26,8 +26,6 @@ dependencies {
implementation(libs.commons.lang3)
implementation(libs.commons.collections4)
implementation(libs.guava)
- implementation(libs.jackson.annotations)
- implementation(libs.jackson.databind)
testImplementation(libs.junit.jupiter.api)
testImplementation(libs.junit.jupiter.params)
diff --git a/api/src/main/java/org/apache/gravitino/secret/SecretBinding.java
b/api/src/main/java/org/apache/gravitino/secret/SecretBinding.java
new file mode 100644
index 0000000000..01c6e2be05
--- /dev/null
+++ b/api/src/main/java/org/apache/gravitino/secret/SecretBinding.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.secret;
+
+import com.google.common.base.Preconditions;
+import java.util.Objects;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.gravitino.annotation.Evolving;
+
+/**
+ * Write-through secret binding for create/alter requests: a registered
provider instance name plus
+ * plaintext to store.
+ */
+@Evolving
+public final class SecretBinding {
+
+ private final String provider;
+ private final String plaintext;
+
+ /**
+ * Creates a write-through binding.
+ *
+ * @param provider registered provider instance name
+ * @param plaintext plaintext secret to write through
+ */
+ public SecretBinding(String provider, String plaintext) {
+ Preconditions.checkArgument(StringUtils.isNotBlank(provider), "provider
must not be blank");
+ Preconditions.checkArgument(plaintext != null, "plaintext must not be
null");
+ this.provider = provider;
+ this.plaintext = plaintext;
+ }
+
+ /**
+ * Returns the registered provider instance name.
+ *
+ * @return the provider name
+ */
+ public String provider() {
+ return provider;
+ }
+
+ /**
+ * Returns the plaintext secret to write through.
+ *
+ * @return the plaintext secret
+ */
+ public String plaintext() {
+ return plaintext;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof SecretBinding)) {
+ return false;
+ }
+ SecretBinding that = (SecretBinding) o;
+ return Objects.equals(provider, that.provider) &&
Objects.equals(plaintext, that.plaintext);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(provider, plaintext);
+ }
+
+ @Override
+ public String toString() {
+ return "SecretBinding{provider='" + provider + "', plaintext=***}";
+ }
+}
diff --git a/api/src/main/java/org/apache/gravitino/secret/SecretReference.java
b/api/src/main/java/org/apache/gravitino/secret/SecretReference.java
new file mode 100644
index 0000000000..f0747124da
--- /dev/null
+++ b/api/src/main/java/org/apache/gravitino/secret/SecretReference.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.secret;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import java.util.Objects;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.gravitino.annotation.Evolving;
+
+/**
+ * External secret locator for create/alter requests: a registered provider
instance name plus
+ * provider-specific attributes. The server builds the URN; clients must not
send a raw URN.
+ */
+@Evolving
+public final class SecretReference {
+
+ private final String provider;
+ private final Map<String, String> attributes;
+
+ /**
+ * Creates an external secret reference.
+ *
+ * @param provider registered provider instance name
+ * @param attributes provider-specific locator keys; must be non-null and
non-empty
+ */
+ public SecretReference(String provider, Map<String, String> attributes) {
+ Preconditions.checkArgument(StringUtils.isNotBlank(provider), "provider
must not be blank");
+ Preconditions.checkArgument(
+ attributes != null && !attributes.isEmpty(), "attributes must not be
null or empty");
+ this.provider = provider;
+ this.attributes = ImmutableMap.copyOf(attributes);
+ }
+
+ /**
+ * Returns the registered provider instance name.
+ *
+ * @return the provider name
+ */
+ public String provider() {
+ return provider;
+ }
+
+ /**
+ * Returns provider-specific locator attributes (never {@code null} or
empty).
+ *
+ * @return an unmodifiable attributes map
+ */
+ public Map<String, String> attributes() {
+ return attributes;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof SecretReference)) {
+ return false;
+ }
+ SecretReference that = (SecretReference) o;
+ return Objects.equals(provider, that.provider) &&
Objects.equals(attributes, that.attributes);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(provider, attributes);
+ }
+
+ @Override
+ public String toString() {
+ return "SecretReference{provider='" + provider + "', attributes=" +
attributes + "}";
+ }
+}
diff --git a/api/build.gradle.kts
b/api/src/test/java/org/apache/gravitino/secret/TestSecretBinding.java
similarity index 55%
copy from api/build.gradle.kts
copy to api/src/test/java/org/apache/gravitino/secret/TestSecretBinding.java
index f0fe3ba5ee..497b24dd8c 100644
--- a/api/build.gradle.kts
+++ b/api/src/test/java/org/apache/gravitino/secret/TestSecretBinding.java
@@ -16,24 +16,23 @@
* specific language governing permissions and limitations
* under the License.
*/
-plugins {
- `maven-publish`
- id("java")
- id("idea")
-}
-dependencies {
- implementation(libs.commons.lang3)
- implementation(libs.commons.collections4)
- implementation(libs.guava)
- implementation(libs.jackson.annotations)
- implementation(libs.jackson.databind)
+package org.apache.gravitino.secret;
- testImplementation(libs.junit.jupiter.api)
- testImplementation(libs.junit.jupiter.params)
- testRuntimeOnly(libs.junit.jupiter.engine)
-}
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TestSecretBinding {
+
+ @Test
+ void testSecretBinding() {
+ SecretBinding binding = new SecretBinding("memory", "s3cr3t");
+ Assertions.assertEquals("memory", binding.provider());
+ Assertions.assertEquals("s3cr3t", binding.plaintext());
+ Assertions.assertFalse(binding.toString().contains("s3cr3t"));
-tasks.build {
- dependsOn("javadoc")
+ Assertions.assertThrows(IllegalArgumentException.class, () -> new
SecretBinding("", "x"));
+ Assertions.assertThrows(
+ IllegalArgumentException.class, () -> new SecretBinding("memory",
null));
+ }
}
diff --git a/api/build.gradle.kts
b/api/src/test/java/org/apache/gravitino/secret/TestSecretReference.java
similarity index 53%
copy from api/build.gradle.kts
copy to api/src/test/java/org/apache/gravitino/secret/TestSecretReference.java
index f0fe3ba5ee..35df9f428f 100644
--- a/api/build.gradle.kts
+++ b/api/src/test/java/org/apache/gravitino/secret/TestSecretReference.java
@@ -16,24 +16,25 @@
* specific language governing permissions and limitations
* under the License.
*/
-plugins {
- `maven-publish`
- id("java")
- id("idea")
-}
-dependencies {
- implementation(libs.commons.lang3)
- implementation(libs.commons.collections4)
- implementation(libs.guava)
- implementation(libs.jackson.annotations)
- implementation(libs.jackson.databind)
+package org.apache.gravitino.secret;
- testImplementation(libs.junit.jupiter.api)
- testImplementation(libs.junit.jupiter.params)
- testRuntimeOnly(libs.junit.jupiter.engine)
-}
+import java.util.Map;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TestSecretReference {
+
+ @Test
+ void testSecretReference() {
+ SecretReference reference =
+ new SecretReference("vault", Map.of("path", "secret/data/x", "key",
"password"));
+ Assertions.assertEquals("vault", reference.provider());
+ Assertions.assertEquals("secret/data/x",
reference.attributes().get("path"));
-tasks.build {
- dependsOn("javadoc")
+ Assertions.assertThrows(
+ IllegalArgumentException.class, () -> new SecretReference("vault",
null));
+ Assertions.assertThrows(
+ IllegalArgumentException.class, () -> new SecretReference("vault",
Map.of()));
+ }
}
diff --git
a/common/src/main/java/org/apache/gravitino/secret/SecretProvider.java
b/common/src/main/java/org/apache/gravitino/secret/SecretProvider.java
index 2db4d8421f..462b29dfcf 100644
--- a/common/src/main/java/org/apache/gravitino/secret/SecretProvider.java
+++ b/common/src/main/java/org/apache/gravitino/secret/SecretProvider.java
@@ -74,6 +74,23 @@ public interface SecretProvider {
*/
void deleteSecret(SecretUrn urn);
+ /**
+ * Builds a URN for an external secret reference without writing secret
material.
+ *
+ * <p>Providers that only support write-through must leave the default
implementation, which
+ * rejects external references.
+ *
+ * @param propertyKey the entity property key that will store the URN
+ * @param attributes provider-specific locator attributes
+ * @return the external-reference secret URN (must end with {@code
propertyKey})
+ * @throws UnsupportedOperationException if this provider does not support
external references
+ * @throws IllegalArgumentException if attributes are invalid for this
provider
+ */
+ default SecretUrn buildReferenceUrn(String propertyKey, Map<String, String>
attributes) {
+ throw new UnsupportedOperationException(
+ type() + " does not support external secret references");
+ }
+
/**
* Releases resources owned by this provider.
*
diff --git a/core/src/main/java/org/apache/gravitino/GravitinoEnv.java
b/core/src/main/java/org/apache/gravitino/GravitinoEnv.java
index f2e10a712c..def00e8cbc 100644
--- a/core/src/main/java/org/apache/gravitino/GravitinoEnv.java
+++ b/core/src/main/java/org/apache/gravitino/GravitinoEnv.java
@@ -98,6 +98,7 @@ import org.apache.gravitino.metrics.MetricsSystem;
import org.apache.gravitino.metrics.source.JVMMetricsSource;
import org.apache.gravitino.policy.PolicyDispatcher;
import org.apache.gravitino.policy.PolicyManager;
+import org.apache.gravitino.secret.SecretManager;
import org.apache.gravitino.secret.SecretProviderRegistry;
import org.apache.gravitino.stats.StatisticDispatcher;
import org.apache.gravitino.stats.StatisticManager;
@@ -158,7 +159,7 @@ public class GravitinoEnv {
private KmsClientRegistry kmsClientRegistry;
- private SecretProviderRegistry secretProviderRegistry;
+ private SecretManager secretManager;
private TagDispatcher tagDispatcher;
@@ -439,19 +440,27 @@ public class GravitinoEnv {
return kmsClientRegistry;
}
+ /**
+ * Get the {@link SecretManager} associated with the Gravitino environment.
+ *
+ * @return The SecretManager instance.
+ * @throws IllegalStateException if the environment has not been initialized
+ */
+ public SecretManager secretManager() {
+ Preconditions.checkState(secretManager != null, "GravitinoEnv components
are not initialized.");
+ return secretManager;
+ }
+
/**
* Get the secrets-provider registry associated with the Gravitino
environment.
*
- * <p>The environment owns this registry. Callers may inject it into
dependent components but must
- * not close it.
+ * <p>Owned by {@link #secretManager()}. Callers may use it for discovery
but must not close it.
*
* @return The secrets-provider registry instance.
* @throws IllegalStateException if the environment has not been initialized
*/
public SecretProviderRegistry secretProviderRegistry() {
- Preconditions.checkState(
- secretProviderRegistry != null, "GravitinoEnv components are not
initialized.");
- return secretProviderRegistry;
+ return secretManager().getRegistry();
}
/**
@@ -675,8 +684,8 @@ public class GravitinoEnv {
kmsClientRegistry.close();
}
- if (secretProviderRegistry != null) {
- secretProviderRegistry.close();
+ if (secretManager != null) {
+ secretManager.close();
}
LOG.info("Gravitino Environment is shut down.");
@@ -684,7 +693,7 @@ public class GravitinoEnv {
private void initBaseComponents() {
this.kmsClientRegistry = new KmsClientRegistry(config);
- this.secretProviderRegistry = new SecretProviderRegistry(config);
+ this.secretManager = new SecretManager(config);
this.metricsSystem = new MetricsSystem();
metricsSystem.register(new JVMMetricsSource());
@@ -724,7 +733,7 @@ public class GravitinoEnv {
// CatalogManager
// CatalogManager registers its own change-log listener with the entity
store (when the store
// supports it), so no poller wiring is needed here.
- this.catalogManager = new CatalogManager(config, entityStore, idGenerator);
+ this.catalogManager = new CatalogManager(config, entityStore, idGenerator,
secretManager);
this.internalCatalogDispatcher = catalogManager;
CatalogNormalizeDispatcher catalogNormalizeDispatcher =
new CatalogNormalizeDispatcher(catalogManager);
@@ -734,10 +743,10 @@ public class GravitinoEnv {
this.catalogDispatcher = new CatalogHookDispatcher(catalogEventDispatcher);
this.credentialOperationDispatcher =
- new CredentialOperationDispatcher(catalogManager, entityStore,
idGenerator);
+ new CredentialOperationDispatcher(catalogManager, entityStore,
idGenerator, secretManager);
SchemaOperationDispatcher schemaOperationDispatcher =
- new SchemaOperationDispatcher(catalogManager, entityStore,
idGenerator);
+ new SchemaOperationDispatcher(catalogManager, entityStore,
idGenerator, secretManager);
this.internalSchemaDispatcher = schemaOperationDispatcher;
SchemaNormalizeDispatcher schemaNormalizeDispatcher =
new SchemaNormalizeDispatcher(schemaOperationDispatcher,
catalogManager);
@@ -747,13 +756,17 @@ public class GravitinoEnv {
this.schemaDispatcher = new SchemaHookDispatcher(schemaEventDispatcher);
TableOperationDispatcher tableOperationDispatcher =
- new TableOperationDispatcher(catalogManager, entityStore, idGenerator);
+ new TableOperationDispatcher(catalogManager, entityStore, idGenerator,
secretManager);
this.internalTableDispatcher = tableOperationDispatcher;
TableNormalizeDispatcher tableNormalizeDispatcher =
new TableNormalizeDispatcher(tableOperationDispatcher, catalogManager);
TableOperationDispatcher internalTableOperationDispatcher =
new TableOperationDispatcher(
- catalogManager, entityStore, idGenerator, () ->
internalSchemaDispatcher);
+ catalogManager,
+ entityStore,
+ idGenerator,
+ () -> internalSchemaDispatcher,
+ secretManager);
this.internalTableDispatcher =
new TableNormalizeDispatcher(internalTableOperationDispatcher,
catalogManager);
TableEventDispatcher tableEventDispatcher =
@@ -764,13 +777,13 @@ public class GravitinoEnv {
// TODO: We can install hooks when we need, we only supports ownership
post hook,
// partition doesn't have ownership, so we don't need it now.
PartitionOperationDispatcher partitionOperationDispatcher =
- new PartitionOperationDispatcher(catalogManager, entityStore,
idGenerator);
+ new PartitionOperationDispatcher(catalogManager, entityStore,
idGenerator, secretManager);
PartitionNormalizeDispatcher partitionNormalizeDispatcher =
new PartitionNormalizeDispatcher(partitionOperationDispatcher,
catalogManager);
this.partitionDispatcher = new PartitionEventDispatcher(eventBus,
partitionNormalizeDispatcher);
FilesetOperationDispatcher filesetOperationDispatcher =
- new FilesetOperationDispatcher(catalogManager, entityStore,
idGenerator);
+ new FilesetOperationDispatcher(catalogManager, entityStore,
idGenerator, secretManager);
FilesetNormalizeDispatcher filesetNormalizeDispatcher =
new FilesetNormalizeDispatcher(filesetOperationDispatcher,
catalogManager);
this.internalFilesetDispatcher = filesetNormalizeDispatcher;
@@ -779,7 +792,7 @@ public class GravitinoEnv {
this.filesetDispatcher = new FilesetHookDispatcher(filesetEventDispatcher);
TopicOperationDispatcher topicOperationDispatcher =
- new TopicOperationDispatcher(catalogManager, entityStore, idGenerator);
+ new TopicOperationDispatcher(catalogManager, entityStore, idGenerator,
secretManager);
TopicNormalizeDispatcher topicNormalizeDispatcher =
new TopicNormalizeDispatcher(topicOperationDispatcher, catalogManager);
this.internalTopicDispatcher = topicNormalizeDispatcher;
@@ -788,7 +801,7 @@ public class GravitinoEnv {
this.topicDispatcher = new TopicHookDispatcher(topicEventDispatcher);
ModelOperationDispatcher modelOperationDispatcher =
- new ModelOperationDispatcher(catalogManager, entityStore, idGenerator);
+ new ModelOperationDispatcher(catalogManager, entityStore, idGenerator,
secretManager);
ModelNormalizeDispatcher modelNormalizeDispatcher =
new ModelNormalizeDispatcher(modelOperationDispatcher, catalogManager);
ModelEventDispatcher modelEventDispatcher =
@@ -800,7 +813,7 @@ public class GravitinoEnv {
// FunctionOperationDispatcher
FunctionOperationDispatcher functionOperationDispatcher =
new FunctionOperationDispatcher(
- catalogManager, schemaOperationDispatcher, entityStore,
idGenerator);
+ catalogManager, schemaOperationDispatcher, entityStore,
idGenerator, secretManager);
FunctionNormalizeDispatcher functionNormalizeDispatcher =
new FunctionNormalizeDispatcher(functionOperationDispatcher,
catalogManager);
FunctionEventDispatcher functionEventDispatcher =
@@ -812,13 +825,17 @@ public class GravitinoEnv {
// TODO(#11007): Add ViewHookDispatcher for view ownership and privilege
hooks when view
// privilege support is finalized.
ViewOperationDispatcher viewOperationDispatcher =
- new ViewOperationDispatcher(catalogManager, entityStore, idGenerator);
+ new ViewOperationDispatcher(catalogManager, entityStore, idGenerator,
secretManager);
this.internalViewDispatcher = viewOperationDispatcher;
ViewNormalizeDispatcher viewNormalizeDispatcher =
new ViewNormalizeDispatcher(viewOperationDispatcher, catalogManager);
ViewOperationDispatcher internalViewOperationDispatcher =
new ViewOperationDispatcher(
- catalogManager, entityStore, idGenerator, () ->
internalSchemaDispatcher);
+ catalogManager,
+ entityStore,
+ idGenerator,
+ () -> internalSchemaDispatcher,
+ secretManager);
this.internalViewDispatcher =
new ViewNormalizeDispatcher(internalViewOperationDispatcher,
catalogManager);
ViewEventDispatcher viewEventDispatcher =
diff --git
a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java
b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java
index 17b0d58bcf..3bafc882b8 100644
--- a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java
+++ b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java
@@ -106,6 +106,7 @@ import org.apache.gravitino.rel.SupportsPartitions;
import org.apache.gravitino.rel.Table;
import org.apache.gravitino.rel.TableCatalog;
import org.apache.gravitino.rel.ViewCatalog;
+import org.apache.gravitino.secret.SecretManager;
import org.apache.gravitino.storage.IdGenerator;
import org.apache.gravitino.storage.relational.SupportsEntityChangeLog;
import org.apache.gravitino.utils.ClassLoaderKey;
@@ -358,6 +359,11 @@ public class CatalogManager implements CatalogDispatcher,
Closeable {
@Nullable private final CatalogChangeLogListener catalogChangeLogListener;
private final IdGenerator idGenerator;
+
+ // Held for create-time secret writes; consumed by the entity-secrets create
follow-up.
+ @SuppressWarnings("UnusedVariable")
+ private final SecretManager secretManager;
+
private final List<Consumer<NameIdentifier>> removalListeners =
Lists.newArrayList();
private final ConcurrentHashMap<NameIdentifier, AtomicInteger>
localMutationCounts =
new ConcurrentHashMap<>();
@@ -373,11 +379,14 @@ public class CatalogManager implements CatalogDispatcher,
Closeable {
* @param config The configuration for the manager.
* @param store The entity store to use.
* @param idGenerator The id generator to use.
+ * @param secretManager The secret manager used by catalog operations.
*/
- public CatalogManager(Config config, EntityStore store, IdGenerator
idGenerator) {
+ public CatalogManager(
+ Config config, EntityStore store, IdGenerator idGenerator, SecretManager
secretManager) {
this.config = config;
this.store = store;
this.idGenerator = idGenerator;
+ this.secretManager = secretManager;
this.classLoaderSharingEnabled =
config.get(Configs.CATALOG_CLASSLOADER_SHARING_ENABLED);
long cacheEvictionIntervalInMs =
config.get(Configs.CATALOG_CACHE_EVICTION_INTERVAL_MS);
diff --git
a/core/src/main/java/org/apache/gravitino/catalog/FilesetOperationDispatcher.java
b/core/src/main/java/org/apache/gravitino/catalog/FilesetOperationDispatcher.java
index ee843c5196..b7f887de7c 100644
---
a/core/src/main/java/org/apache/gravitino/catalog/FilesetOperationDispatcher.java
+++
b/core/src/main/java/org/apache/gravitino/catalog/FilesetOperationDispatcher.java
@@ -38,6 +38,7 @@ import org.apache.gravitino.file.Fileset;
import org.apache.gravitino.file.FilesetChange;
import org.apache.gravitino.lock.LockType;
import org.apache.gravitino.lock.TreeLockUtils;
+import org.apache.gravitino.secret.SecretManager;
import org.apache.gravitino.storage.IdGenerator;
public class FilesetOperationDispatcher extends OperationDispatcher implements
FilesetDispatcher {
@@ -47,10 +48,14 @@ public class FilesetOperationDispatcher extends
OperationDispatcher implements F
* @param catalogManager The CatalogManager instance to be used for fileset
operations.
* @param store The EntityStore instance to be used for fileset operations.
* @param idGenerator The IdGenerator instance to be used for fileset
operations.
+ * @param secretManager The SecretManager instance to be used for secret
operations.
*/
public FilesetOperationDispatcher(
- CatalogManager catalogManager, EntityStore store, IdGenerator
idGenerator) {
- super(catalogManager, store, idGenerator);
+ CatalogManager catalogManager,
+ EntityStore store,
+ IdGenerator idGenerator,
+ SecretManager secretManager) {
+ super(catalogManager, store, idGenerator, secretManager);
}
/**
diff --git
a/core/src/main/java/org/apache/gravitino/catalog/FunctionOperationDispatcher.java
b/core/src/main/java/org/apache/gravitino/catalog/FunctionOperationDispatcher.java
index ebc5f0d15d..923b5fb452 100644
---
a/core/src/main/java/org/apache/gravitino/catalog/FunctionOperationDispatcher.java
+++
b/core/src/main/java/org/apache/gravitino/catalog/FunctionOperationDispatcher.java
@@ -34,6 +34,7 @@ import org.apache.gravitino.function.FunctionDefinition;
import org.apache.gravitino.function.FunctionType;
import org.apache.gravitino.lock.LockType;
import org.apache.gravitino.lock.TreeLockUtils;
+import org.apache.gravitino.secret.SecretManager;
import org.apache.gravitino.storage.IdGenerator;
/**
@@ -56,15 +57,18 @@ public class FunctionOperationDispatcher extends
OperationDispatcher implements
* Creates a new FunctionOperationDispatcher instance.
*
* @param catalogManager The CatalogManager instance to be used for function
operations.
+ * @param schemaOps The SchemaOperationDispatcher used to validate schema
existence.
* @param store The EntityStore instance to be used for function operations.
* @param idGenerator The IdGenerator instance to be used for function
operations.
+ * @param secretManager The SecretManager instance to be used for secret
operations.
*/
public FunctionOperationDispatcher(
CatalogManager catalogManager,
SchemaOperationDispatcher schemaOps,
EntityStore store,
- IdGenerator idGenerator) {
- super(catalogManager, store, idGenerator);
+ IdGenerator idGenerator,
+ SecretManager secretManager) {
+ super(catalogManager, store, idGenerator, secretManager);
this.schemaOps = schemaOps;
this.managedFunctionOps = new ManagedFunctionOperations(store,
idGenerator);
}
diff --git
a/core/src/main/java/org/apache/gravitino/catalog/ModelOperationDispatcher.java
b/core/src/main/java/org/apache/gravitino/catalog/ModelOperationDispatcher.java
index f3d3389098..1290fefe60 100644
---
a/core/src/main/java/org/apache/gravitino/catalog/ModelOperationDispatcher.java
+++
b/core/src/main/java/org/apache/gravitino/catalog/ModelOperationDispatcher.java
@@ -44,14 +44,18 @@ import org.apache.gravitino.model.ModelCatalog;
import org.apache.gravitino.model.ModelChange;
import org.apache.gravitino.model.ModelVersion;
import org.apache.gravitino.model.ModelVersionChange;
+import org.apache.gravitino.secret.SecretManager;
import org.apache.gravitino.storage.IdGenerator;
import org.apache.gravitino.utils.ThrowableFunction;
public class ModelOperationDispatcher extends OperationDispatcher implements
ModelDispatcher {
public ModelOperationDispatcher(
- CatalogManager catalogManager, EntityStore store, IdGenerator
idGenerator) {
- super(catalogManager, store, idGenerator);
+ CatalogManager catalogManager,
+ EntityStore store,
+ IdGenerator idGenerator,
+ SecretManager secretManager) {
+ super(catalogManager, store, idGenerator, secretManager);
}
@Override
diff --git
a/core/src/main/java/org/apache/gravitino/catalog/OperationDispatcher.java
b/core/src/main/java/org/apache/gravitino/catalog/OperationDispatcher.java
index 733d411cfd..4fa1808fc6 100644
--- a/core/src/main/java/org/apache/gravitino/catalog/OperationDispatcher.java
+++ b/core/src/main/java/org/apache/gravitino/catalog/OperationDispatcher.java
@@ -40,6 +40,8 @@ import org.apache.gravitino.messaging.TopicChange;
import org.apache.gravitino.rel.SupportsPartitions;
import org.apache.gravitino.rel.TableChange;
import org.apache.gravitino.rel.ViewChange;
+import org.apache.gravitino.secret.SecretManager;
+import org.apache.gravitino.secret.SecretPropertyUtils;
import org.apache.gravitino.storage.IdGenerator;
import org.apache.gravitino.utils.ThrowableFunction;
import org.slf4j.Logger;
@@ -58,18 +60,25 @@ public abstract class OperationDispatcher {
protected final IdGenerator idGenerator;
+ protected final SecretManager secretManager;
+
/**
* Creates a new CatalogOperationDispatcher instance.
*
* @param catalogManager The CatalogManager instance to be used for catalog
operations.
* @param store The EntityStore instance to be used for catalog operations.
* @param idGenerator The IdGenerator instance to be used for catalog
operations.
+ * @param secretManager The SecretManager instance to be used for secret
operations.
*/
protected OperationDispatcher(
- CatalogManager catalogManager, EntityStore store, IdGenerator
idGenerator) {
+ CatalogManager catalogManager,
+ EntityStore store,
+ IdGenerator idGenerator,
+ SecretManager secretManager) {
this.catalogManager = catalogManager;
this.store = store;
this.idGenerator = idGenerator;
+ this.secretManager = secretManager;
}
protected <R, E extends Throwable> R doWithTable(
@@ -140,8 +149,12 @@ public abstract class OperationDispatcher {
c.doWithPropertiesMeta(
p -> {
PropertiesMetadata propertiesMetadata = provider.apply(p);
- return properties.keySet().stream()
- .filter(propertiesMetadata::isHiddenProperty)
+ return properties.entrySet().stream()
+ .filter(
+ e ->
+ propertiesMetadata.isHiddenProperty(e.getKey())
+ ||
SecretPropertyUtils.isSecretProperty(e.getKey(), e.getValue()))
+ .map(Map.Entry::getKey)
.collect(Collectors.toSet());
}),
IllegalArgumentException.class);
diff --git
a/core/src/main/java/org/apache/gravitino/catalog/PartitionOperationDispatcher.java
b/core/src/main/java/org/apache/gravitino/catalog/PartitionOperationDispatcher.java
index 37173b6293..8887dbb78a 100644
---
a/core/src/main/java/org/apache/gravitino/catalog/PartitionOperationDispatcher.java
+++
b/core/src/main/java/org/apache/gravitino/catalog/PartitionOperationDispatcher.java
@@ -27,6 +27,7 @@ import org.apache.gravitino.lock.LockType;
import org.apache.gravitino.lock.TreeLockUtils;
import org.apache.gravitino.rel.SupportsPartitions;
import org.apache.gravitino.rel.partitions.Partition;
+import org.apache.gravitino.secret.SecretManager;
import org.apache.gravitino.storage.IdGenerator;
public class PartitionOperationDispatcher extends OperationDispatcher
@@ -38,10 +39,14 @@ public class PartitionOperationDispatcher extends
OperationDispatcher
* @param catalogManager The CatalogManager instance to be used for
partition operations.
* @param store The EntityStore instance to be used for partition operations.
* @param idGenerator The IdGenerator instance to be used for partition
operations.
+ * @param secretManager The SecretManager instance to be used for secret
operations.
*/
public PartitionOperationDispatcher(
- CatalogManager catalogManager, EntityStore store, IdGenerator
idGenerator) {
- super(catalogManager, store, idGenerator);
+ CatalogManager catalogManager,
+ EntityStore store,
+ IdGenerator idGenerator,
+ SecretManager secretManager) {
+ super(catalogManager, store, idGenerator, secretManager);
}
@Override
diff --git
a/core/src/main/java/org/apache/gravitino/catalog/SchemaOperationDispatcher.java
b/core/src/main/java/org/apache/gravitino/catalog/SchemaOperationDispatcher.java
index a6cd4589f0..31c752b21b 100644
---
a/core/src/main/java/org/apache/gravitino/catalog/SchemaOperationDispatcher.java
+++
b/core/src/main/java/org/apache/gravitino/catalog/SchemaOperationDispatcher.java
@@ -42,6 +42,7 @@ import org.apache.gravitino.lock.LockType;
import org.apache.gravitino.lock.TreeLockUtils;
import org.apache.gravitino.meta.AuditInfo;
import org.apache.gravitino.meta.SchemaEntity;
+import org.apache.gravitino.secret.SecretManager;
import org.apache.gravitino.storage.IdGenerator;
import org.apache.gravitino.utils.PrincipalUtils;
import org.apache.gravitino.utils.SchemaEntityCleaner;
@@ -58,10 +59,14 @@ public class SchemaOperationDispatcher extends
OperationDispatcher implements Sc
* @param catalogManager The CatalogManager instance to be used for schema
operations.
* @param store The EntityStore instance to be used for schema operations.
* @param idGenerator The IdGenerator instance to be used for schema
operations.
+ * @param secretManager The SecretManager instance to be used for secret
operations.
*/
public SchemaOperationDispatcher(
- CatalogManager catalogManager, EntityStore store, IdGenerator
idGenerator) {
- super(catalogManager, store, idGenerator);
+ CatalogManager catalogManager,
+ EntityStore store,
+ IdGenerator idGenerator,
+ SecretManager secretManager) {
+ super(catalogManager, store, idGenerator, secretManager);
}
/**
diff --git
a/core/src/main/java/org/apache/gravitino/catalog/TableOperationDispatcher.java
b/core/src/main/java/org/apache/gravitino/catalog/TableOperationDispatcher.java
index c661e18f9a..e2d7bd9d86 100644
---
a/core/src/main/java/org/apache/gravitino/catalog/TableOperationDispatcher.java
+++
b/core/src/main/java/org/apache/gravitino/catalog/TableOperationDispatcher.java
@@ -64,6 +64,7 @@ import org.apache.gravitino.rel.expressions.sorts.SortOrder;
import org.apache.gravitino.rel.expressions.transforms.Transform;
import org.apache.gravitino.rel.indexes.Index;
import org.apache.gravitino.rel.indexes.Indexes;
+import org.apache.gravitino.secret.SecretManager;
import org.apache.gravitino.storage.IdGenerator;
import org.apache.gravitino.utils.NamespaceUtil;
import org.apache.gravitino.utils.PrincipalUtils;
@@ -82,10 +83,19 @@ public class TableOperationDispatcher extends
OperationDispatcher implements Tab
* @param catalogManager The CatalogManager instance to be used for table
operations.
* @param store The EntityStore instance to be used for table operations.
* @param idGenerator The IdGenerator instance to be used for table
operations.
+ * @param secretManager The SecretManager instance to be used for secret
operations.
*/
public TableOperationDispatcher(
- CatalogManager catalogManager, EntityStore store, IdGenerator
idGenerator) {
- this(catalogManager, store, idGenerator, () ->
GravitinoEnv.getInstance().schemaDispatcher());
+ CatalogManager catalogManager,
+ EntityStore store,
+ IdGenerator idGenerator,
+ SecretManager secretManager) {
+ this(
+ catalogManager,
+ store,
+ idGenerator,
+ () -> GravitinoEnv.getInstance().schemaDispatcher(),
+ secretManager);
}
/**
@@ -95,13 +105,15 @@ public class TableOperationDispatcher extends
OperationDispatcher implements Tab
* @param store The EntityStore instance to be used for table operations.
* @param idGenerator The IdGenerator instance to be used for table
operations.
* @param schemaDispatcherSupplier The SchemaDispatcher supplier to ensure
schemas are imported.
+ * @param secretManager The SecretManager instance to be used for secret
operations.
*/
public TableOperationDispatcher(
CatalogManager catalogManager,
EntityStore store,
IdGenerator idGenerator,
- Supplier<SchemaDispatcher> schemaDispatcherSupplier) {
- super(catalogManager, store, idGenerator);
+ Supplier<SchemaDispatcher> schemaDispatcherSupplier,
+ SecretManager secretManager) {
+ super(catalogManager, store, idGenerator, secretManager);
this.schemaDispatcherSupplier =
Preconditions.checkNotNull(
schemaDispatcherSupplier, "schemaDispatcherSupplier must not be
null");
diff --git
a/core/src/main/java/org/apache/gravitino/catalog/TopicOperationDispatcher.java
b/core/src/main/java/org/apache/gravitino/catalog/TopicOperationDispatcher.java
index 3c43860701..e2f03b1743 100644
---
a/core/src/main/java/org/apache/gravitino/catalog/TopicOperationDispatcher.java
+++
b/core/src/main/java/org/apache/gravitino/catalog/TopicOperationDispatcher.java
@@ -44,6 +44,7 @@ import org.apache.gravitino.messaging.Topic;
import org.apache.gravitino.messaging.TopicChange;
import org.apache.gravitino.meta.AuditInfo;
import org.apache.gravitino.meta.TopicEntity;
+import org.apache.gravitino.secret.SecretManager;
import org.apache.gravitino.storage.IdGenerator;
import org.apache.gravitino.utils.PrincipalUtils;
import org.slf4j.Logger;
@@ -58,10 +59,14 @@ public class TopicOperationDispatcher extends
OperationDispatcher implements Top
* @param catalogManager The CatalogManager instance to be used for catalog
operations.
* @param store The EntityStore instance to be used for catalog operations.
* @param idGenerator The IdGenerator instance to be used for catalog
operations.
+ * @param secretManager The SecretManager instance to be used for secret
operations.
*/
public TopicOperationDispatcher(
- CatalogManager catalogManager, EntityStore store, IdGenerator
idGenerator) {
- super(catalogManager, store, idGenerator);
+ CatalogManager catalogManager,
+ EntityStore store,
+ IdGenerator idGenerator,
+ SecretManager secretManager) {
+ super(catalogManager, store, idGenerator, secretManager);
}
/**
diff --git
a/core/src/main/java/org/apache/gravitino/catalog/ViewOperationDispatcher.java
b/core/src/main/java/org/apache/gravitino/catalog/ViewOperationDispatcher.java
index 2bb02e9917..731639dca9 100644
---
a/core/src/main/java/org/apache/gravitino/catalog/ViewOperationDispatcher.java
+++
b/core/src/main/java/org/apache/gravitino/catalog/ViewOperationDispatcher.java
@@ -51,6 +51,7 @@ import org.apache.gravitino.rel.Column;
import org.apache.gravitino.rel.Representation;
import org.apache.gravitino.rel.View;
import org.apache.gravitino.rel.ViewChange;
+import org.apache.gravitino.secret.SecretManager;
import org.apache.gravitino.storage.IdGenerator;
import org.apache.gravitino.utils.PrincipalUtils;
import org.slf4j.Logger;
@@ -69,10 +70,19 @@ public class ViewOperationDispatcher extends
OperationDispatcher implements View
* @param catalogManager The CatalogManager instance to be used for view
operations.
* @param store The EntityStore instance to be used for view operations.
* @param idGenerator The IdGenerator instance to be used for view
operations.
+ * @param secretManager The SecretManager instance to be used for secret
operations.
*/
public ViewOperationDispatcher(
- CatalogManager catalogManager, EntityStore store, IdGenerator
idGenerator) {
- this(catalogManager, store, idGenerator, () ->
GravitinoEnv.getInstance().schemaDispatcher());
+ CatalogManager catalogManager,
+ EntityStore store,
+ IdGenerator idGenerator,
+ SecretManager secretManager) {
+ this(
+ catalogManager,
+ store,
+ idGenerator,
+ () -> GravitinoEnv.getInstance().schemaDispatcher(),
+ secretManager);
}
/**
@@ -82,13 +92,15 @@ public class ViewOperationDispatcher extends
OperationDispatcher implements View
* @param store The EntityStore instance to be used for view operations.
* @param idGenerator The IdGenerator instance to be used for view
operations.
* @param schemaDispatcherSupplier The SchemaDispatcher supplier to ensure
schemas are imported.
+ * @param secretManager The SecretManager instance to be used for secret
operations.
*/
public ViewOperationDispatcher(
CatalogManager catalogManager,
EntityStore store,
IdGenerator idGenerator,
- Supplier<SchemaDispatcher> schemaDispatcherSupplier) {
- super(catalogManager, store, idGenerator);
+ Supplier<SchemaDispatcher> schemaDispatcherSupplier,
+ SecretManager secretManager) {
+ super(catalogManager, store, idGenerator, secretManager);
this.schemaDispatcherSupplier =
Preconditions.checkNotNull(
schemaDispatcherSupplier, "schemaDispatcherSupplier must not be
null");
diff --git
a/core/src/main/java/org/apache/gravitino/credential/CredentialOperationDispatcher.java
b/core/src/main/java/org/apache/gravitino/credential/CredentialOperationDispatcher.java
index 599e7805dd..857f41b88c 100644
---
a/core/src/main/java/org/apache/gravitino/credential/CredentialOperationDispatcher.java
+++
b/core/src/main/java/org/apache/gravitino/credential/CredentialOperationDispatcher.java
@@ -37,6 +37,7 @@ import org.apache.gravitino.connector.BaseCatalog;
import org.apache.gravitino.connector.credential.PathContext;
import org.apache.gravitino.connector.credential.SupportsPathBasedCredentials;
import org.apache.gravitino.exceptions.NoSuchCatalogException;
+import org.apache.gravitino.secret.SecretManager;
import org.apache.gravitino.storage.IdGenerator;
import org.apache.gravitino.utils.NameIdentifierUtil;
import org.apache.gravitino.utils.PrincipalUtils;
@@ -45,8 +46,11 @@ import org.apache.gravitino.utils.PrincipalUtils;
public class CredentialOperationDispatcher extends OperationDispatcher {
public CredentialOperationDispatcher(
- CatalogManager catalogManager, EntityStore store, IdGenerator
idGenerator) {
- super(catalogManager, store, idGenerator);
+ CatalogManager catalogManager,
+ EntityStore store,
+ IdGenerator idGenerator,
+ SecretManager secretManager) {
+ super(catalogManager, store, idGenerator, secretManager);
}
public List<Credential> getCredentials(NameIdentifier identifier,
CredentialPrivilege privilege) {
diff --git a/core/src/main/java/org/apache/gravitino/secret/SecretManager.java
b/core/src/main/java/org/apache/gravitino/secret/SecretManager.java
new file mode 100644
index 0000000000..a404cc8e94
--- /dev/null
+++ b/core/src/main/java/org/apache/gravitino/secret/SecretManager.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.secret;
+
+import static org.apache.gravitino.secret.SecretConstants.ATTR_ENTITY_ID;
+import static org.apache.gravitino.secret.SecretConstants.ATTR_ENTITY_TYPE;
+import static org.apache.gravitino.secret.SecretConstants.ATTR_PROPERTY_KEY;
+import static
org.apache.gravitino.secret.SecretPropertyUtils.validateSecretBindings;
+import static
org.apache.gravitino.secret.SecretPropertyUtils.validateSecretReferences;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import java.io.Closeable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.gravitino.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * SecretManager owns write-through / external-reference secret lifecycle
against configured
+ * providers, and rolls back write-through secrets when entity create fails.
+ */
+public class SecretManager implements Closeable {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(SecretManager.class);
+
+ private final SecretProviderRegistry registry;
+
+ public SecretManager(Config config) {
+ this.registry = new SecretProviderRegistry(config);
+ }
+
+ public SecretManager(SecretProviderRegistry registry) {
+ this.registry = registry;
+ }
+
+ /**
+ * Returns the secrets-provider registry owned by this manager.
+ *
+ * @return the registry
+ */
+ public SecretProviderRegistry getRegistry() {
+ return registry;
+ }
+
+ /**
+ * Builds external-reference URNs from {@code secretReferences} without
writing secret material.
+ *
+ * <p>Callers must put the returned URN strings into properties themselves
(e.g. via {@link
+ * SecretPropertyUtils#applySecretUrns}). External-ref URNs are owned
outside Gravitino and must
+ * not be passed to {@link #rollbackWritten}.
+ *
+ * @param secretReferences property key → secret locator
+ * @return external-reference URNs (insertion order)
+ */
+ public List<SecretUrn> getSecretReferenceUrns(Map<String, SecretReference>
secretReferences) {
+ Preconditions.checkArgument(
+ secretReferences != null && !secretReferences.isEmpty(),
+ "secretReferences must not be null or empty");
+ validateSecretReferences(secretReferences);
+
+ List<SecretUrn> urns = new ArrayList<>(secretReferences.size());
+ for (Map.Entry<String, SecretReference> entry :
secretReferences.entrySet()) {
+ String key = entry.getKey();
+ SecretReference locator = entry.getValue();
+ String providerName = locator.provider();
+ SecretProvider provider = registry.getProvider(providerName);
+ try {
+ SecretUrn urn = provider.buildReferenceUrn(key, locator.attributes());
+ validateUrnEndsWithPropertyKey(urn, key);
+ urns.add(urn);
+ } catch (UnsupportedOperationException e) {
+ throw new IllegalArgumentException(
+ "Provider \""
+ + providerName
+ + "\" does not support secretReferences for key \""
+ + key
+ + "\"",
+ e);
+ }
+ }
+ return List.copyOf(urns);
+ }
+
+ /**
+ * Builds write-through URNs from {@code secretBindings} without writing
secret material.
+ *
+ * <p>Callers should pass the returned URNs to {@link #writeSecrets} to
persist plaintext from
+ * each binding's value, then put URNs into properties (e.g. via {@link
+ * SecretPropertyUtils#applySecretUrns}).
+ *
+ * @param entityType {@code catalog}, {@code schema}, or {@code fileset}
+ * @param entityId stable numeric entity id
+ * @param secretBindings property key → write-through binding
+ * @return write-through URNs (insertion order)
+ */
+ public List<SecretUrn> getSecretBindingUrns(
+ String entityType, long entityId, Map<String, SecretBinding>
secretBindings) {
+ Preconditions.checkArgument(StringUtils.isNotBlank(entityType),
"entityType must not be blank");
+ Preconditions.checkArgument(
+ secretBindings != null && !secretBindings.isEmpty(),
+ "secretBindings must not be null or empty");
+ validateSecretBindings(secretBindings);
+
+ List<SecretUrn> urns = new ArrayList<>(secretBindings.size());
+ for (Map.Entry<String, SecretBinding> entry : secretBindings.entrySet()) {
+ String key = entry.getKey();
+ String providerName = entry.getValue().provider();
+ // Ensure the provider is registered before building the URN.
+ registry.getProvider(providerName);
+ Map<String, String> attributes =
+ ImmutableMap.of(
+ ATTR_ENTITY_TYPE, entityType,
+ ATTR_ENTITY_ID, String.valueOf(entityId),
+ ATTR_PROPERTY_KEY, key);
+ SecretUrn urn = SecretUrn.buildWriteThrough(providerName, attributes);
+ validateUrnEndsWithPropertyKey(urn, key);
+ urns.add(urn);
+ }
+ return List.copyOf(urns);
+ }
+
+ /**
+ * Writes plaintext secrets from {@code secretBindings} values into the
write-through providers
+ * for {@code secretUrns} (e.g. Vault).
+ *
+ * <p>{@code secretUrns} must come from {@link #getSecretBindingUrns}. On
failure, already-written
+ * URNs are rolled back. Callers must put URN strings into properties
themselves (e.g. via {@link
+ * SecretPropertyUtils#applySecretUrns}).
+ *
+ * @param secretBindings property key → write-through binding
+ * @param secretUrns write-through URNs from {@link #getSecretBindingUrns}
+ */
+ public void writeSecrets(Map<String, SecretBinding> secretBindings,
List<SecretUrn> secretUrns) {
+ Preconditions.checkArgument(
+ secretBindings != null && !secretBindings.isEmpty(),
+ "secretBindings must not be null or empty");
+ Preconditions.checkArgument(
+ secretUrns != null && !secretUrns.isEmpty(), "secretUrns must not be
null or empty");
+ validateSecretBindings(secretBindings);
+
+ List<SecretUrn> written = new ArrayList<>(secretUrns.size());
+ try {
+ for (SecretUrn urn : secretUrns) {
+ List<String> segments = urn.identifierSegments();
+ Preconditions.checkArgument(
+ segments.size() == 3,
+ "Write-through secret URN must have entityType, entityId,
propertyKey segments: %s",
+ urn);
+ String entityType = segments.get(0);
+ String entityId = segments.get(1);
+ String propertyKey = segments.get(2);
+ SecretBinding binding = secretBindings.get(propertyKey);
+ Preconditions.checkArgument(
+ binding != null, "No secretBindings entry for property key
\"%s\"", propertyKey);
+ String plaintext = binding.plaintext();
+ Map<String, String> attributes =
+ ImmutableMap.of(
+ ATTR_ENTITY_TYPE, entityType,
+ ATTR_ENTITY_ID, entityId,
+ ATTR_PROPERTY_KEY, propertyKey);
+ SecretUrn writtenUrn =
+ registry.getProvider(urn.providerName()).writeSecret(plaintext,
attributes);
+ Preconditions.checkArgument(
+ urn.equals(writtenUrn),
+ "Provider returned unexpected URN: expected %s, got %s",
+ urn,
+ writtenUrn);
+ written.add(writtenUrn);
+ }
+ } catch (RuntimeException e) {
+ rollbackWritten(written);
+ throw e;
+ }
+ }
+
+ /**
+ * Best-effort delete of write-through secrets after a failed create.
+ *
+ * <p>Only write-through URNs that were persisted by {@link #writeSecrets}
may be passed. Do not
+ * roll back external reference URNs from {@link #getSecretReferenceUrns}.
+ *
+ * @param secretUrns write-through URNs from {@link #getSecretBindingUrns}
+ */
+ public void rollbackWritten(@Nullable List<SecretUrn> secretUrns) {
+ if (secretUrns == null || secretUrns.isEmpty()) {
+ return;
+ }
+ for (SecretUrn urn : secretUrns) {
+ try {
+ registry.getProvider(urn.providerName()).deleteSecret(urn);
+ } catch (Exception e) {
+ LOG.warn("Failed to roll back written secret {}", urn, e);
+ }
+ }
+ }
+
+ @Override
+ public void close() {
+ registry.close();
+ }
+
+ private static void validateUrnEndsWithPropertyKey(SecretUrn urn, String
propertyKey) {
+ Preconditions.checkArgument(
+ urn.toString().endsWith(propertyKey),
+ "Built secret URN must end with property key \"%s\": %s",
+ propertyKey,
+ urn);
+ }
+}
diff --git
a/core/src/main/java/org/apache/gravitino/secret/SecretPropertyUtils.java
b/core/src/main/java/org/apache/gravitino/secret/SecretPropertyUtils.java
new file mode 100644
index 0000000000..334773efb3
--- /dev/null
+++ b/core/src/main/java/org/apache/gravitino/secret/SecretPropertyUtils.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.secret;
+
+import static org.apache.gravitino.secret.SecretConstants.URN_PREFIX;
+
+import com.google.common.base.Preconditions;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.annotation.Nullable;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * Helpers for secret-related entity property handling and request validation.
+ *
+ * <p>This is intentionally separate from {@link SecretManager}, which owns
secret lifecycle
+ * (build/write/rollback) rather than property assembly and request-shape
checks.
+ */
+public final class SecretPropertyUtils {
+
+ private SecretPropertyUtils() {}
+
+ /**
+ * Returns whether a property value is a Gravitino secret URN for the given
key.
+ *
+ * @param key the property key
+ * @param value the property value
+ * @return true when value starts with the secret URN prefix and ends with
the key
+ */
+ public static boolean isSecretProperty(@Nullable String key, @Nullable
String value) {
+ return key != null && value != null && value.startsWith(URN_PREFIX) &&
value.endsWith(key);
+ }
+
+ /**
+ * Rejects property keys that appear in both {@code secretBindings} and
{@code secretReferences}.
+ *
+ * @param secretBindings property key → write-through binding (may be null)
+ * @param secretReferences property key → secret locator (may be null)
+ */
+ public static void checkNoOverlap(
+ @Nullable Map<String, SecretBinding> secretBindings,
+ @Nullable Map<String, SecretReference> secretReferences) {
+ Set<String> bindingKeys = secretBindings == null ? Set.of() :
secretBindings.keySet();
+ Set<String> referenceKeys = secretReferences == null ? Set.of() :
secretReferences.keySet();
+ Set<String> overlap = new HashSet<>(bindingKeys);
+ overlap.retainAll(referenceKeys);
+ Preconditions.checkArgument(
+ overlap.isEmpty(),
+ "Property keys cannot appear in both secretBindings and
secretReferences: %s",
+ overlap);
+ }
+
+ /**
+ * Applies each URN string into {@code properties} under the property key
encoded in the URN (last
+ * identifier segment).
+ *
+ * @param properties mutable entity properties
+ * @param secretUrns secret URNs whose last identifier segment is the
property key
+ */
+ public static void applySecretUrns(Map<String, String> properties,
List<SecretUrn> secretUrns) {
+ Preconditions.checkArgument(properties != null, "properties must not be
null");
+ Preconditions.checkArgument(secretUrns != null, "secretUrns must not be
null");
+ for (SecretUrn urn : secretUrns) {
+ List<String> segments = urn.identifierSegments();
+ Preconditions.checkArgument(
+ !segments.isEmpty(), "Secret URN must contain at least one
identifier segment: %s", urn);
+ properties.put(segments.get(segments.size() - 1), urn.toString());
+ }
+ }
+
+ /**
+ * Validates create-time {@code secretBindings} request shape.
+ *
+ * @param bindings property key → write-through binding
+ */
+ static void validateSecretBindings(Map<String, SecretBinding> bindings) {
+ for (Map.Entry<String, SecretBinding> entry : bindings.entrySet()) {
+ String key = entry.getKey();
+ Preconditions.checkArgument(
+ StringUtils.isNotBlank(key), "secretBindings keys must not be
blank");
+ Preconditions.checkArgument(
+ entry.getValue() != null, "secretBindings[%s] must not be null",
key);
+ }
+ }
+
+ /**
+ * Validates create-time {@code secretReferences} request shape.
+ *
+ * @param references property key → secret locator
+ */
+ static void validateSecretReferences(Map<String, SecretReference>
references) {
+ for (Map.Entry<String, SecretReference> entry : references.entrySet()) {
+ String key = entry.getKey();
+ Preconditions.checkArgument(
+ StringUtils.isNotBlank(key), "secretReferences keys must not be
blank");
+ Preconditions.checkArgument(
+ entry.getValue() != null, "secretReferences[%s] must not be null",
key);
+ }
+ }
+}
diff --git
a/core/src/test/java/org/apache/gravitino/TestGravitinoEnvSecretProviderRegistry.java
b/core/src/test/java/org/apache/gravitino/TestGravitinoEnvSecretProviderRegistry.java
index 370802d327..7345c23923 100644
---
a/core/src/test/java/org/apache/gravitino/TestGravitinoEnvSecretProviderRegistry.java
+++
b/core/src/test/java/org/apache/gravitino/TestGravitinoEnvSecretProviderRegistry.java
@@ -7,10 +7,10 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
@@ -20,6 +20,7 @@ package org.apache.gravitino;
import java.util.Properties;
import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.gravitino.secret.SecretManager;
import org.apache.gravitino.secret.SecretProviderRegistry;
import org.apache.gravitino.secret.memory.InMemorySecretsProvider;
import org.junit.jupiter.api.Assertions;
@@ -31,17 +32,20 @@ public class TestGravitinoEnvSecretProviderRegistry {
void testEmptyRegistryIsOptionalAndClosedWithEnvironment() throws
IllegalAccessException {
TestGravitinoEnv env = new TestGravitinoEnv();
Assertions.assertThrows(IllegalStateException.class,
env::secretProviderRegistry);
+ Assertions.assertThrows(IllegalStateException.class, env::secretManager);
- SecretProviderRegistry registry = new SecretProviderRegistry(new
Config(false) {});
- FieldUtils.writeField(env, "secretProviderRegistry", registry, true);
+ SecretManager secretManager = new SecretManager(new Config(false) {});
+ FieldUtils.writeField(env, "secretManager", secretManager, true);
- Assertions.assertSame(registry, env.secretProviderRegistry());
- Assertions.assertTrue(registry.listProviders().isEmpty());
+ Assertions.assertSame(secretManager, env.secretManager());
+ Assertions.assertSame(secretManager.getRegistry(),
env.secretProviderRegistry());
+
Assertions.assertTrue(secretManager.getRegistry().listProviders().isEmpty());
env.shutdown();
- Assertions.assertSame(registry, env.secretProviderRegistry());
- Assertions.assertThrows(IllegalStateException.class,
registry::listProviders);
+ Assertions.assertSame(secretManager, env.secretManager());
+ Assertions.assertThrows(
+ IllegalStateException.class, () ->
secretManager.getRegistry().listProviders());
}
@Test
@@ -58,13 +62,15 @@ public class TestGravitinoEnvSecretProviderRegistry {
config.loadFromProperties(properties);
env.initializeBaseComponents(config);
+ SecretManager secretManager = env.secretManager();
SecretProviderRegistry registry = env.secretProviderRegistry();
+ Assertions.assertSame(secretManager.getRegistry(), registry);
Assertions.assertEquals(1, registry.listProviders().size());
Assertions.assertEquals("memory", registry.getProvider("memory").type());
env.shutdown();
- Assertions.assertSame(registry, env.secretProviderRegistry());
+ Assertions.assertSame(secretManager, env.secretManager());
Assertions.assertThrows(IllegalStateException.class,
registry::listProviders);
}
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestCatalogManager.java
b/core/src/test/java/org/apache/gravitino/catalog/TestCatalogManager.java
index ddc57cc556..c55c404469 100644
--- a/core/src/test/java/org/apache/gravitino/catalog/TestCatalogManager.java
+++ b/core/src/test/java/org/apache/gravitino/catalog/TestCatalogManager.java
@@ -65,6 +65,7 @@ import org.apache.gravitino.meta.BaseMetalake;
import org.apache.gravitino.meta.CatalogEntity;
import org.apache.gravitino.meta.SchemaEntity;
import org.apache.gravitino.meta.SchemaVersion;
+import org.apache.gravitino.secret.SecretManager;
import org.apache.gravitino.storage.RandomIdGenerator;
import org.apache.gravitino.storage.memory.TestMemoryEntityStore;
import
org.apache.gravitino.storage.memory.TestMemoryEntityStore.InMemoryEntityStore;
@@ -113,7 +114,8 @@ public class TestCatalogManager {
entityStore.put(metalakeEntity, true);
- catalogManager = new CatalogManager(config, entityStore, new
RandomIdGenerator());
+ catalogManager =
+ new CatalogManager(config, entityStore, new RandomIdGenerator(), new
SecretManager(config));
FieldUtils.writeField(GravitinoEnv.getInstance(), "lockManager", new
LockManager(config), true);
catalogManager = Mockito.spy(catalogManager);
}
@@ -793,7 +795,8 @@ public class TestCatalogManager {
@Test
void testCloseUnregistersCatalogChangeLogListener() {
ChangeLogAwareEntityStore store = new ChangeLogAwareEntityStore();
- CatalogManager manager = new CatalogManager(config, store, new
RandomIdGenerator());
+ CatalogManager manager =
+ new CatalogManager(config, store, new RandomIdGenerator(), new
SecretManager(config));
EntityChangeLogListener registeredListener = store.listener.get();
Assertions.assertNotNull(registeredListener);
@@ -809,7 +812,8 @@ public class TestCatalogManager {
store.initialize(config);
store.put(metalakeEntity, true);
- CatalogManager manager = new CatalogManager(config, store, new
RandomIdGenerator());
+ CatalogManager manager =
+ new CatalogManager(config, store, new RandomIdGenerator(), new
SecretManager(config));
NameIdentifier ident = NameIdentifier.of("metalake",
"delete_returns_false");
Map<String, String> props =
ImmutableMap.of(
@@ -852,7 +856,8 @@ public class TestCatalogManager {
store.initialize(config);
store.put(metalakeEntity, true);
- CatalogManager manager = new CatalogManager(config, store, new
RandomIdGenerator());
+ CatalogManager manager =
+ new CatalogManager(config, store, new RandomIdGenerator(), new
SecretManager(config));
NameIdentifier ident = NameIdentifier.of("metalake",
"failed_create_cleanup");
// A creation that fails validation (key1 is required but missing) stores
the entity and then
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestCatalogNormalizeDispatcher.java
b/core/src/test/java/org/apache/gravitino/catalog/TestCatalogNormalizeDispatcher.java
index 9babbe7ce6..a299beb1b7 100644
---
a/core/src/test/java/org/apache/gravitino/catalog/TestCatalogNormalizeDispatcher.java
+++
b/core/src/test/java/org/apache/gravitino/catalog/TestCatalogNormalizeDispatcher.java
@@ -27,15 +27,19 @@ import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import java.time.Instant;
import java.util.Map;
+import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.gravitino.Catalog;
import org.apache.gravitino.Config;
import org.apache.gravitino.Configs;
import org.apache.gravitino.EntityStore;
+import org.apache.gravitino.GravitinoEnv;
import org.apache.gravitino.MetadataObjects;
import org.apache.gravitino.NameIdentifier;
+import org.apache.gravitino.lock.LockManager;
import org.apache.gravitino.meta.AuditInfo;
import org.apache.gravitino.meta.BaseMetalake;
import org.apache.gravitino.meta.SchemaVersion;
+import org.apache.gravitino.secret.SecretManager;
import org.apache.gravitino.storage.RandomIdGenerator;
import org.apache.gravitino.storage.memory.TestMemoryEntityStore;
import org.junit.jupiter.api.AfterAll;
@@ -61,7 +65,7 @@ public class TestCatalogNormalizeDispatcher {
.build();
@BeforeAll
- public static void setUp() throws IOException {
+ public static void setUp() throws IOException, IllegalAccessException {
Config config = new Config(false) {};
config.set(Configs.CATALOG_LOAD_ISOLATED, false);
@@ -70,7 +74,9 @@ public class TestCatalogNormalizeDispatcher {
entityStore.put(metalakeEntity, true);
- catalogManager = new CatalogManager(config, entityStore, new
RandomIdGenerator());
+ catalogManager =
+ new CatalogManager(config, entityStore, new RandomIdGenerator(), new
SecretManager(config));
+ FieldUtils.writeField(GravitinoEnv.getInstance(), "lockManager", new
LockManager(config), true);
catalogManager = Mockito.spy(catalogManager);
catalogNormalizeDispatcher = new
CatalogNormalizeDispatcher(catalogManager);
}
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestClassLoaderPoolIntegration.java
b/core/src/test/java/org/apache/gravitino/catalog/TestClassLoaderPoolIntegration.java
index 6e59027004..5589d41d53 100644
---
a/core/src/test/java/org/apache/gravitino/catalog/TestClassLoaderPoolIntegration.java
+++
b/core/src/test/java/org/apache/gravitino/catalog/TestClassLoaderPoolIntegration.java
@@ -34,6 +34,7 @@ import org.apache.gravitino.lock.LockManager;
import org.apache.gravitino.meta.AuditInfo;
import org.apache.gravitino.meta.BaseMetalake;
import org.apache.gravitino.meta.SchemaVersion;
+import org.apache.gravitino.secret.SecretManager;
import org.apache.gravitino.storage.RandomIdGenerator;
import org.apache.gravitino.storage.memory.TestMemoryEntityStore;
import
org.apache.gravitino.storage.memory.TestMemoryEntityStore.InMemoryEntityStore;
@@ -90,7 +91,8 @@ public class TestClassLoaderPoolIntegration {
@BeforeEach
public void beforeEach() throws IOException {
- catalogManager = new CatalogManager(config, entityStore, new
RandomIdGenerator());
+ catalogManager =
+ new CatalogManager(config, entityStore, new RandomIdGenerator(), new
SecretManager(config));
}
@AfterEach
@@ -243,7 +245,11 @@ public class TestClassLoaderPoolIntegration {
noSharingConfig.set(Configs.CATALOG_CLASSLOADER_SHARING_ENABLED, false);
CatalogManager noSharingManager =
- new CatalogManager(noSharingConfig, entityStore, new
RandomIdGenerator());
+ new CatalogManager(
+ noSharingConfig,
+ entityStore,
+ new RandomIdGenerator(),
+ new SecretManager(noSharingConfig));
try {
Map<String, String> props =
ImmutableMap.of("key1", "value1", "key2", "value2", "key5-1",
"value3");
@@ -292,7 +298,11 @@ public class TestClassLoaderPoolIntegration {
noSharingConfig.set(Configs.CATALOG_CLASSLOADER_SHARING_ENABLED, false);
CatalogManager noSharingManager =
- new CatalogManager(noSharingConfig, entityStore, new
RandomIdGenerator());
+ new CatalogManager(
+ noSharingConfig,
+ entityStore,
+ new RandomIdGenerator(),
+ new SecretManager(noSharingConfig));
try {
Map<String, String> props =
ImmutableMap.of("key1", "value1", "key2", "value2", "key5-1",
"value3");
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java
b/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java
index 9dd411b28b..3e26888264 100644
---
a/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java
+++
b/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java
@@ -45,9 +45,9 @@ public class TestFilesetOperationDispatcher extends
TestOperationDispatcher {
@BeforeAll
public static void initialize() throws IOException {
schemaOperationDispatcher =
- new SchemaOperationDispatcher(catalogManager, entityStore,
idGenerator);
+ new SchemaOperationDispatcher(catalogManager, entityStore,
idGenerator, secretManager);
filesetOperationDispatcher =
- new FilesetOperationDispatcher(catalogManager, entityStore,
idGenerator);
+ new FilesetOperationDispatcher(catalogManager, entityStore,
idGenerator, secretManager);
}
public static FilesetOperationDispatcher getFilesetOperationDispatcher() {
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestFunctionOperationDispatcher.java
b/core/src/test/java/org/apache/gravitino/catalog/TestFunctionOperationDispatcher.java
index cac62bf4f2..41caa1303f 100644
---
a/core/src/test/java/org/apache/gravitino/catalog/TestFunctionOperationDispatcher.java
+++
b/core/src/test/java/org/apache/gravitino/catalog/TestFunctionOperationDispatcher.java
@@ -28,6 +28,7 @@ import org.apache.gravitino.connector.BaseCatalog;
import org.apache.gravitino.function.FunctionChange;
import org.apache.gravitino.function.FunctionDefinition;
import org.apache.gravitino.function.FunctionType;
+import org.apache.gravitino.secret.SecretManager;
import org.apache.gravitino.storage.IdGenerator;
import org.apache.gravitino.storage.RandomIdGenerator;
import org.junit.jupiter.api.Assertions;
@@ -61,7 +62,9 @@ public class TestFunctionOperationDispatcher {
when(catalogManager.loadCatalogAndWrap(NameIdentifier.of(METALAKE,
HIVE_CATALOG)))
.thenReturn(hiveWrapper);
- dispatcher = new FunctionOperationDispatcher(catalogManager, schemaOps,
store, idGenerator);
+ dispatcher =
+ new FunctionOperationDispatcher(
+ catalogManager, schemaOps, store, idGenerator,
mock(SecretManager.class));
}
@Test
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestModelOperationDispatcher.java
b/core/src/test/java/org/apache/gravitino/catalog/TestModelOperationDispatcher.java
index c9f2cb069a..da5675094e 100644
---
a/core/src/test/java/org/apache/gravitino/catalog/TestModelOperationDispatcher.java
+++
b/core/src/test/java/org/apache/gravitino/catalog/TestModelOperationDispatcher.java
@@ -65,9 +65,9 @@ public class TestModelOperationDispatcher extends
TestOperationDispatcher {
FieldUtils.writeField(GravitinoEnv.getInstance(), "lockManager", new
LockManager(config), true);
modelOperationDispatcher =
- new ModelOperationDispatcher(catalogManager, entityStore, idGenerator);
+ new ModelOperationDispatcher(catalogManager, entityStore, idGenerator,
secretManager);
schemaOperationDispatcher =
- new SchemaOperationDispatcher(catalogManager, entityStore,
idGenerator);
+ new SchemaOperationDispatcher(catalogManager, entityStore,
idGenerator, secretManager);
}
@Test
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestOperationDispatcher.java
b/core/src/test/java/org/apache/gravitino/catalog/TestOperationDispatcher.java
index 73f8a146af..a72673ac21 100644
---
a/core/src/test/java/org/apache/gravitino/catalog/TestOperationDispatcher.java
+++
b/core/src/test/java/org/apache/gravitino/catalog/TestOperationDispatcher.java
@@ -53,6 +53,7 @@ import org.apache.gravitino.lock.LockManager;
import org.apache.gravitino.meta.AuditInfo;
import org.apache.gravitino.meta.BaseMetalake;
import org.apache.gravitino.meta.SchemaVersion;
+import org.apache.gravitino.secret.SecretManager;
import org.apache.gravitino.storage.IdGenerator;
import org.apache.gravitino.storage.RandomIdGenerator;
import org.apache.gravitino.storage.memory.TestMemoryEntityStore;
@@ -77,6 +78,8 @@ public abstract class TestOperationDispatcher {
protected static CatalogManager catalogManager;
+ protected static SecretManager secretManager;
+
private static Config config;
@BeforeAll
@@ -97,7 +100,8 @@ public abstract class TestOperationDispatcher {
.build();
entityStore.put(metalakeEntity, true);
- catalogManager = new CatalogManager(config, entityStore, idGenerator);
+ secretManager = new SecretManager(config);
+ catalogManager = new CatalogManager(config, entityStore, idGenerator,
secretManager);
FieldUtils.writeField(GravitinoEnv.getInstance(), "catalogManager",
catalogManager, true);
Config config = mock(Config.class);
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestPartitionOperationDispatcher.java
b/core/src/test/java/org/apache/gravitino/catalog/TestPartitionOperationDispatcher.java
index 9ddc3b1d30..bf3d116240 100644
---
a/core/src/test/java/org/apache/gravitino/catalog/TestPartitionOperationDispatcher.java
+++
b/core/src/test/java/org/apache/gravitino/catalog/TestPartitionOperationDispatcher.java
@@ -78,11 +78,11 @@ public class TestPartitionOperationDispatcher extends
TestOperationDispatcher {
protected static void prepareTable() throws IllegalAccessException {
schemaOperationDispatcher =
- new SchemaOperationDispatcher(catalogManager, entityStore,
idGenerator);
+ new SchemaOperationDispatcher(catalogManager, entityStore,
idGenerator, secretManager);
tableOperationDispatcher =
- new TableOperationDispatcher(catalogManager, entityStore, idGenerator);
+ new TableOperationDispatcher(catalogManager, entityStore, idGenerator,
secretManager);
partitionOperationDispatcher =
- new PartitionOperationDispatcher(catalogManager, entityStore,
idGenerator);
+ new PartitionOperationDispatcher(catalogManager, entityStore,
idGenerator, secretManager);
Config config = mock(Config.class);
doReturn(100000L).when(config).get(TREE_LOCK_MAX_NODE_IN_MEMORY);
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestSchemaOperationDispatcher.java
b/core/src/test/java/org/apache/gravitino/catalog/TestSchemaOperationDispatcher.java
index 455e428c34..bb2b0d0576 100644
---
a/core/src/test/java/org/apache/gravitino/catalog/TestSchemaOperationDispatcher.java
+++
b/core/src/test/java/org/apache/gravitino/catalog/TestSchemaOperationDispatcher.java
@@ -61,7 +61,8 @@ public class TestSchemaOperationDispatcher extends
TestOperationDispatcher {
@BeforeAll
public static void initialize() throws IOException, IllegalAccessException {
- dispatcher = new SchemaOperationDispatcher(catalogManager, entityStore,
idGenerator);
+ dispatcher =
+ new SchemaOperationDispatcher(catalogManager, entityStore,
idGenerator, secretManager);
Config config = mock(Config.class);
doReturn(100000L).when(config).get(Configs.TREE_LOCK_MAX_NODE_IN_MEMORY);
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestTableOperationDispatcher.java
b/core/src/test/java/org/apache/gravitino/catalog/TestTableOperationDispatcher.java
index 260d23a4c0..a834398c71 100644
---
a/core/src/test/java/org/apache/gravitino/catalog/TestTableOperationDispatcher.java
+++
b/core/src/test/java/org/apache/gravitino/catalog/TestTableOperationDispatcher.java
@@ -80,10 +80,14 @@ public class TestTableOperationDispatcher extends
TestOperationDispatcher {
@BeforeAll
public static void initialize() throws IOException, IllegalAccessException {
schemaOperationDispatcher =
- new SchemaOperationDispatcher(catalogManager, entityStore,
idGenerator);
+ new SchemaOperationDispatcher(catalogManager, entityStore,
idGenerator, secretManager);
tableOperationDispatcher =
new TableOperationDispatcher(
- catalogManager, entityStore, idGenerator, () ->
schemaOperationDispatcher);
+ catalogManager,
+ entityStore,
+ idGenerator,
+ () -> schemaOperationDispatcher,
+ secretManager);
Config config = mock(Config.class);
doReturn(100000L).when(config).get(TREE_LOCK_MAX_NODE_IN_MEMORY);
@@ -265,7 +269,9 @@ public class TestTableOperationDispatcher extends
TestOperationDispatcher {
public void
testTableOperationDispatcherRejectsNullSchemaDispatcherSupplier() {
Assertions.assertThrows(
NullPointerException.class,
- () -> new TableOperationDispatcher(catalogManager, entityStore,
idGenerator, null));
+ () ->
+ new TableOperationDispatcher(
+ catalogManager, entityStore, idGenerator, null,
secretManager));
}
@Test
@@ -277,7 +283,7 @@ public class TestTableOperationDispatcher extends
TestOperationDispatcher {
Supplier<SchemaDispatcher> nullSchemaDispatcherSupplier = () -> null;
TableOperationDispatcher dispatcher =
new TableOperationDispatcher(
- catalogManager, entityStore, idGenerator,
nullSchemaDispatcherSupplier);
+ catalogManager, entityStore, idGenerator,
nullSchemaDispatcherSupplier, secretManager);
NameIdentifier tableIdent = NameIdentifier.of(tableNs,
"table_null_dispatcher");
Column[] columns =
new Column[] {
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestTopicOperationDispatcher.java
b/core/src/test/java/org/apache/gravitino/catalog/TestTopicOperationDispatcher.java
index 50055ef183..084f12132b 100644
---
a/core/src/test/java/org/apache/gravitino/catalog/TestTopicOperationDispatcher.java
+++
b/core/src/test/java/org/apache/gravitino/catalog/TestTopicOperationDispatcher.java
@@ -66,9 +66,9 @@ public class TestTopicOperationDispatcher extends
TestOperationDispatcher {
@BeforeAll
public static void initialize() throws IOException, IllegalAccessException {
schemaOperationDispatcher =
- new SchemaOperationDispatcher(catalogManager, entityStore,
idGenerator);
+ new SchemaOperationDispatcher(catalogManager, entityStore,
idGenerator, secretManager);
topicOperationDispatcher =
- new TopicOperationDispatcher(catalogManager, entityStore, idGenerator);
+ new TopicOperationDispatcher(catalogManager, entityStore, idGenerator,
secretManager);
Config config = mock(Config.class);
doReturn(100000L).when(config).get(TREE_LOCK_MAX_NODE_IN_MEMORY);
diff --git
a/core/src/test/java/org/apache/gravitino/catalog/TestViewOperationDispatcher.java
b/core/src/test/java/org/apache/gravitino/catalog/TestViewOperationDispatcher.java
index 8f5b0403f1..26372f662e 100644
---
a/core/src/test/java/org/apache/gravitino/catalog/TestViewOperationDispatcher.java
+++
b/core/src/test/java/org/apache/gravitino/catalog/TestViewOperationDispatcher.java
@@ -72,10 +72,14 @@ public class TestViewOperationDispatcher extends
TestOperationDispatcher {
@BeforeAll
public static void initialize() throws IOException, IllegalAccessException {
schemaOperationDispatcher =
- new SchemaOperationDispatcher(catalogManager, entityStore,
idGenerator);
+ new SchemaOperationDispatcher(catalogManager, entityStore,
idGenerator, secretManager);
viewOperationDispatcher =
new ViewOperationDispatcher(
- catalogManager, entityStore, idGenerator, () ->
schemaOperationDispatcher);
+ catalogManager,
+ entityStore,
+ idGenerator,
+ () -> schemaOperationDispatcher,
+ secretManager);
Config config = mock(Config.class);
doReturn(100000L).when(config).get(TREE_LOCK_MAX_NODE_IN_MEMORY);
@@ -207,7 +211,9 @@ public class TestViewOperationDispatcher extends
TestOperationDispatcher {
public void testViewOperationDispatcherRejectsNullSchemaDispatcherSupplier()
{
Assertions.assertThrows(
NullPointerException.class,
- () -> new ViewOperationDispatcher(catalogManager, entityStore,
idGenerator, null));
+ () ->
+ new ViewOperationDispatcher(
+ catalogManager, entityStore, idGenerator, null,
secretManager));
}
@Test
@@ -219,7 +225,7 @@ public class TestViewOperationDispatcher extends
TestOperationDispatcher {
Supplier<SchemaDispatcher> nullSchemaDispatcherSupplier = () -> null;
ViewOperationDispatcher dispatcher =
new ViewOperationDispatcher(
- catalogManager, entityStore, idGenerator,
nullSchemaDispatcherSupplier);
+ catalogManager, entityStore, idGenerator,
nullSchemaDispatcherSupplier, secretManager);
NameIdentifier viewIdent = NameIdentifier.of(viewNs,
"view_null_dispatcher");
Representation[] representations =
new Representation[] {
diff --git
a/core/src/test/java/org/apache/gravitino/credential/TestCredentialOperationDispatcher.java
b/core/src/test/java/org/apache/gravitino/credential/TestCredentialOperationDispatcher.java
index 761918215c..60648b2c14 100644
---
a/core/src/test/java/org/apache/gravitino/credential/TestCredentialOperationDispatcher.java
+++
b/core/src/test/java/org/apache/gravitino/credential/TestCredentialOperationDispatcher.java
@@ -143,7 +143,7 @@ public class TestCredentialOperationDispatcher extends
TestOperationDispatcher {
Mockito.when(baseCatalog.ops()).thenReturn(ops);
CredentialOperationDispatcher dispatcher =
- new CredentialOperationDispatcher(catalogManager, entityStore,
idGenerator);
+ new CredentialOperationDispatcher(catalogManager, entityStore,
idGenerator, secretManager);
List<Credential> credentials =
dispatcher.getCredentials(
diff --git
a/core/src/test/java/org/apache/gravitino/secret/TestSecretManager.java
b/core/src/test/java/org/apache/gravitino/secret/TestSecretManager.java
new file mode 100644
index 0000000000..ef6c2b08a6
--- /dev/null
+++ b/core/src/test/java/org/apache/gravitino/secret/TestSecretManager.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.secret;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import org.apache.gravitino.Config;
+import org.apache.gravitino.secret.memory.InMemorySecretsProvider;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TestSecretManager {
+
+ @Test
+ void testWriteSecrets() {
+ try (SecretManager secretManager = memorySecretManager()) {
+ Map<String, String> properties = new HashMap<>(Map.of("jdbc-user",
"root"));
+ Map<String, SecretBinding> secretBindings =
+ Map.of("jdbc-password", new SecretBinding("memory", "s3cr3t"));
+ List<SecretUrn> secretUrns =
+ secretManager.getSecretBindingUrns("catalog", 42L, secretBindings);
+ secretManager.writeSecrets(secretBindings, secretUrns);
+ SecretPropertyUtils.applySecretUrns(properties, secretUrns);
+
+ String urn = properties.get("jdbc-password");
+
Assertions.assertTrue(SecretPropertyUtils.isSecretProperty("jdbc-password",
urn));
+ Assertions.assertEquals("root", properties.get("jdbc-user"));
+ Assertions.assertEquals(1, secretUrns.size());
+ Assertions.assertEquals(
+ "s3cr3t",
+
secretManager.getRegistry().getProvider("memory").readSecret(secretUrns.get(0)));
+ }
+ }
+
+ @Test
+ void testGetSecretReferenceUrnsRejectedByMemoryProvider() {
+ try (SecretManager secretManager = memorySecretManager()) {
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ secretManager.getSecretReferenceUrns(
+ Map.of(
+ "jdbc-password",
+ new SecretReference("memory", Map.of("path",
"secret/data/x")))));
+ }
+ }
+
+ @Test
+ void testRejectOverlapAndInvalidBinding() {
+ try (SecretManager secretManager = memorySecretManager()) {
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ SecretPropertyUtils.checkNoOverlap(
+ Map.of("jdbc-password", new SecretBinding("memory",
"s3cr3t")),
+ Map.of(
+ "jdbc-password",
+ new SecretReference("memory", Map.of("path",
"secret/data/x")))));
+
+ Assertions.assertThrows(
+ IllegalArgumentException.class, () -> new SecretBinding(" ",
"s3cr3t"));
+
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ secretManager.getSecretReferenceUrns(
+ Map.of(
+ "jdbc-password",
+ new SecretReference("memory", Map.of("path",
"secret/data/x")))));
+ }
+ }
+
+ @Test
+ void testSecretBindingToStringRedactsPlaintext() {
+ SecretBinding binding = new SecretBinding("memory", "s3cr3t");
+ Assertions.assertFalse(binding.toString().contains("s3cr3t"));
+ Assertions.assertTrue(binding.toString().contains("***"));
+ }
+
+ private static SecretManager memorySecretManager() {
+ Config config = new Config(false) {};
+ Properties properties = new Properties();
+ properties.setProperty(SecretProviderRegistry.GRAVITINO_SECRET_PROVIDERS,
"memory");
+ properties.setProperty(
+ SecretProviderRegistry.GRAVITINO_SECRET_PROVIDER_PREFIX
+ + "memory."
+ + SecretProviderRegistry.CLASS_NAME,
+ InMemorySecretsProvider.class.getName());
+ config.loadFromProperties(properties);
+ return new SecretManager(config);
+ }
+}
