This is an automated email from the ASF dual-hosted git repository.
fanjia pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 36fe1134fa [Improve][Core][Starter] Support encryption and decryption
for transform config (#9932)
36fe1134fa is described below
commit 36fe1134fa36cc6e013e7242609cbfce34213fc6
Author: loupipalien <[email protected]>
AuthorDate: Mon Oct 13 21:08:11 2025 +0800
[Improve][Core][Starter] Support encryption and decryption for transform
config (#9932)
---
.../core/starter/utils/ConfigShadeUtils.java | 10 ++++
.../core/starter/utils/ConfigShadeTest.java | 36 +++++++++++
.../resources/config.shade_with_transform.json | 70 ++++++++++++++++++++++
3 files changed, 116 insertions(+)
diff --git
a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java
b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java
index 27a6e753c9..459f4c4030 100644
---
a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java
+++
b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java
@@ -177,6 +177,9 @@ public final class ConfigShadeUtils {
(ArrayList<Map<String, Object>>)
configMap.get(Constants.SOURCE);
List<Map<String, Object>> sinks =
(ArrayList<Map<String, Object>>) configMap.get(Constants.SINK);
+ List<Map<String, Object>> transforms =
+ (ArrayList<Map<String, Object>>)
+ configMap.getOrDefault(Constants.TRANSFORM, new
ArrayList<>());
Preconditions.checkArgument(
!sources.isEmpty(), "Miss <Source> config! Please check the
config file.");
Preconditions.checkArgument(
@@ -193,8 +196,15 @@ public final class ConfigShadeUtils {
sink.computeIfPresent(sensitiveOption,
processFunction);
}
});
+ transforms.forEach(
+ transform -> {
+ for (String sensitiveOption : sensitiveOptions) {
+ transform.computeIfPresent(sensitiveOption,
processFunction);
+ }
+ });
configMap.put(Constants.SOURCE, sources);
configMap.put(Constants.SINK, sinks);
+ configMap.put(Constants.TRANSFORM, transforms);
return ConfigFactory.parseMap(configMap);
}
diff --git
a/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java
b/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java
index 770170c762..8e1a1f48d1 100644
---
a/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java
+++
b/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java
@@ -367,6 +367,42 @@ public class ConfigShadeTest {
encryptedConfig.getConfigList("source").get(0).getString("password"));
}
+ @Test
+ public void testDecryptWithTransform() throws URISyntaxException {
+ URL resource =
ConfigShadeTest.class.getResource("/config.shade_with_transform.json");
+ Assertions.assertNotNull(resource);
+ Config decryptedProps = ConfigBuilder.of(Paths.get(resource.toURI()),
Lists.newArrayList());
+
+ Assertions.assertEquals(
+ "access_key",
+
decryptedProps.getConfigList("source").get(0).getString("access_key"));
+ Assertions.assertEquals(
+ "secret_key",
+
decryptedProps.getConfigList("source").get(0).getString("secret_key"));
+ Assertions.assertEquals(
+ "api_key",
decryptedProps.getConfigList("transform").get(0).getString("api_key"));
+ Assertions.assertEquals(
+ "api_key",
decryptedProps.getConfigList("transform").get(1).getString("api_key"));
+ Assertions.assertEquals(
+ "token",
decryptedProps.getConfigList("sink").get(0).getString("token"));
+
+ String accessKey = ConfigShadeUtils.encryptOption("base64",
"access_key");
+ String secretKey = ConfigShadeUtils.encryptOption("base64",
"secret_key");
+ String apiKey = ConfigShadeUtils.encryptOption("base64", "api_key");
+ String token = ConfigShadeUtils.encryptOption("base64", "token");
+ Config encryptedConfig =
ConfigShadeUtils.encryptConfig(decryptedProps);
+ Assertions.assertEquals(
+ accessKey,
encryptedConfig.getConfigList("source").get(0).getString("access_key"));
+ Assertions.assertEquals(
+ secretKey,
encryptedConfig.getConfigList("source").get(0).getString("secret_key"));
+ Assertions.assertEquals(
+ apiKey,
encryptedConfig.getConfigList("transform").get(0).getString("api_key"));
+ Assertions.assertEquals(
+ apiKey,
encryptedConfig.getConfigList("transform").get(1).getString("api_key"));
+ Assertions.assertEquals(
+ token,
encryptedConfig.getConfigList("sink").get(0).getString("token"));
+ }
+
public static class ConfigShadeWithProps implements ConfigShade {
private String suffix;
diff --git
a/seatunnel-core/seatunnel-core-starter/src/test/resources/config.shade_with_transform.json
b/seatunnel-core/seatunnel-core-starter/src/test/resources/config.shade_with_transform.json
new file mode 100644
index 0000000000..3c99b55711
--- /dev/null
+++
b/seatunnel-core/seatunnel-core-starter/src/test/resources/config.shade_with_transform.json
@@ -0,0 +1,70 @@
+{
+ "env" : {
+ "shade.identifier" : "base64",
+ "parallelism" : 1,
+ "shade.options" : ["api_key"]
+ },
+ "source" : [
+ {
+ "plugin_name": "S3File",
+ "schema": {
+ "fields": {
+ "id": "int",
+ "age": "int",
+ "name": "string"
+ },
+ "primaryKey": {
+ "name": "id",
+ "columnNames": [
+ "id"
+ ]
+ }
+ },
+ "path": "/test_json_data.json",
+ "bucket": "seatunnel",
+ "access_key": "YWNjZXNzX2tleQ==",
+ "secret_key": "c2VjcmV0X2tleQ==",
+ "file_format_type": "json",
+ "fs.s3a.endpoint": "xxx.seatunnel.s3.com",
+ "fs.s3a.aws.credentials.provider":
"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider",
+ "plugin_output": "s3file_output"
+ }
+ ],
+ "transform" : [
+ {
+ "plugin_name": "LLM",
+ "api_key": "YXBpX2tleQ==",
+ "model_provider": "DOUBAO",
+ "inference_columns": ["name"],
+ "model": "doubao-1-5-thinking-pro-250415",
+ "prompt": "Inferring male or female based on name",
+ "plugin_input": "s3file_output",
+ "plugin_output": "llm_output"
+ },
+ {
+ "plugin_name": "Embedding",
+ "model": "doubao-embedding-text-240715",
+ "api_key": "YXBpX2tleQ==",
+ "model_provider": "DOUBAO",
+ "vectorization_fields": {
+ "name_vector": "name"
+ },
+ "plugin_input": "llm_output",
+ "plugin_output": "embedding_output"
+ }
+ ],
+ "sink" : [
+ {
+ "plugin_name": "Milvus",
+ "enable_auto_id": true,
+ "batch_size": 1000,
+ "database": "default",
+ "schema_save_mode": "RECREATE_SCHEMA",
+ "url": "https://milvus.com:19530";,
+ "token": "dG9rZW4=",
+ "create_index": true,
+ "load_collection": true,
+ "plugin_input": "embedding_output"
+ }
+ ]
+}
