This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 4f18ec50ed [rest] Merge RESTToken with catalog options (#6629)
4f18ec50ed is described below
commit 4f18ec50ede4bd20fca1f4b4a557f1dff0ce4ed7
Author: Zouxxyy <[email protected]>
AuthorDate: Wed Nov 19 10:55:57 2025 +0800
[rest] Merge RESTToken with catalog options (#6629)
---
.../org/apache/paimon/rest/RESTTokenFileIO.java | 39 ++----
.../apache/paimon/rest/RESTTokenFileIOTest.java | 144 ---------------------
.../apache/paimon/rest/MockRESTCatalogTest.java | 15 +++
.../org/apache/paimon/rest/RESTCatalogTest.java | 23 ++++
4 files changed, 49 insertions(+), 172 deletions(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java
b/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java
index 819624d2fd..469ce08d2b 100644
--- a/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java
+++ b/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java
@@ -171,10 +171,7 @@ public class RESTTokenFileIO implements FileIO {
}
Options options = catalogContext.options();
- options =
- new Options(
- RESTTokenFileIO.mergeTokenWithDlfEndpointHandling(
- options.toMap(), token.token()));
+ options = new Options(RESTUtil.merge(options.toMap(),
token.token()));
options.set(FILE_IO_ALLOW_CACHE, false);
CatalogContext context =
CatalogContext.create(
@@ -227,34 +224,20 @@ public class RESTTokenFileIO implements FileIO {
identifier,
response.getExpiresAtMillis());
- token = new RESTToken(response.getToken(),
response.getExpiresAtMillis());
+ token =
+ new RESTToken(
+ mergeTokenWithCatalogOptions(response.getToken()),
+ response.getExpiresAtMillis());
}
- /**
- * Merges token properties with catalog properties and handles DLF OSS
endpoint configuration.
- *
- * <p>This method performs the same merge logic as {@link
RESTUtil#merge(Map, Map)} but also
- * handles the special case where the DLF OSS endpoint should override the
standard OSS
- * endpoint. When 'dlf.oss-endpoint' is present in the merged properties,
it will be used to set
- * 'fs.oss.endpoint' for OSS file system configuration.
- *
- * @param restTokenProperties the properties from the REST token
- * @param catalogProperties the catalog properties to merge with
- * @return merged properties with DLF OSS endpoint handling applied
- */
- public static Map<String, String> mergeTokenWithDlfEndpointHandling(
- Map<String, String> catalogProperties, Map<String, String>
restTokenProperties) {
- // Use RESTUtil.merge for the basic merge logic
- Map<String, String> result =
- Maps.newLinkedHashMap(RESTUtil.merge(catalogProperties,
restTokenProperties));
-
- // Handle special case: dlf.oss-endpoint should override
fs.oss.endpoint
- String dlfOssEndpoint = result.get(DLF_OSS_ENDPOINT.key());
+ private Map<String, String> mergeTokenWithCatalogOptions(Map<String,
String> token) {
+ Map<String, String> newToken = Maps.newLinkedHashMap(token);
+ // DLF OSS endpoint should override the standard OSS endpoint.
+ String dlfOssEndpoint =
catalogContext.options().get(DLF_OSS_ENDPOINT.key());
if (dlfOssEndpoint != null && !dlfOssEndpoint.isEmpty()) {
- result.put("fs.oss.endpoint", dlfOssEndpoint);
+ newToken.put("fs.oss.endpoint", dlfOssEndpoint);
}
-
- return ImmutableMap.copyOf(result);
+ return ImmutableMap.copyOf(newToken);
}
/**
diff --git
a/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java
b/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java
deleted file mode 100644
index a5133546e2..0000000000
---
a/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.rest;
-
-import org.junit.jupiter.api.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.paimon.rest.RESTCatalogOptions.DLF_OSS_ENDPOINT;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-/** Test for {@link RESTTokenFileIO}. */
-public class RESTTokenFileIOTest {
- @Test
- public void testMergeTokenWithDlfEndpointHandling() {
- // Test case 1: dlf.oss-endpoint overrides fs.oss.endpoint when
present and non-empty
- {
- Map<String, String> catalogProperties = new HashMap<>();
- catalogProperties.put("fs.oss.endpoint", "original-endpoint");
- catalogProperties.put("other.config", "value1");
- Map<String, String> restTokenProperties = new HashMap<>();
- restTokenProperties.put(DLF_OSS_ENDPOINT.key(),
"new-oss-endpoint");
- restTokenProperties.put("other.config", "value2"); // This should
override
- Map<String, String> result =
- RESTTokenFileIO.mergeTokenWithDlfEndpointHandling(
- catalogProperties, restTokenProperties);
- assertEquals("new-oss-endpoint", result.get("fs.oss.endpoint"));
- assertEquals(
- "value2", result.get("other.config")); //
restTokenProperties takes precedence
- assertEquals("new-oss-endpoint",
result.get(DLF_OSS_ENDPOINT.key()));
- assertEquals(3, result.size());
- }
-
- // Test case 2: dlf.oss-endpoint adds fs.oss.endpoint when not present
in catalogProperties
- {
- Map<String, String> catalogProperties = new HashMap<>();
- catalogProperties.put("other.config", "value1");
- Map<String, String> restTokenProperties = new HashMap<>();
- restTokenProperties.put(DLF_OSS_ENDPOINT.key(),
"new-oss-endpoint");
- Map<String, String> result =
- RESTTokenFileIO.mergeTokenWithDlfEndpointHandling(
- catalogProperties, restTokenProperties);
- assertEquals("new-oss-endpoint", result.get("fs.oss.endpoint"));
- assertEquals("value1", result.get("other.config"));
- assertEquals("new-oss-endpoint",
result.get(DLF_OSS_ENDPOINT.key()));
- assertEquals(3, result.size());
- }
-
- // Test case 3: Empty dlf.oss-endpoint should not override
- {
- Map<String, String> catalogProperties = new HashMap<>();
- catalogProperties.put("fs.oss.endpoint", "original-endpoint");
- Map<String, String> restTokenProperties = new HashMap<>();
- restTokenProperties.put(DLF_OSS_ENDPOINT.key(), "");
- Map<String, String> result =
- RESTTokenFileIO.mergeTokenWithDlfEndpointHandling(
- catalogProperties, restTokenProperties);
- assertEquals("original-endpoint", result.get("fs.oss.endpoint"));
- assertEquals(
- 2, result.size()); // fs.oss.endpoint and dlf.oss-endpoint
(empty string is not
- // null)
- }
-
- // Test case 4: Null dlf.oss-endpoint should not override
- {
- Map<String, String> catalogProperties = new HashMap<>();
- catalogProperties.put("fs.oss.endpoint", "original-endpoint");
- Map<String, String> restTokenProperties = new HashMap<>();
- restTokenProperties.put(DLF_OSS_ENDPOINT.key(), null);
- Map<String, String> result =
- RESTTokenFileIO.mergeTokenWithDlfEndpointHandling(
- catalogProperties, restTokenProperties);
- assertEquals("original-endpoint", result.get("fs.oss.endpoint"));
- assertEquals(1, result.size()); // Only fs.oss.endpoint (null
values are filtered out)
- }
-
- // Test case 5: No dlf.oss-endpoint in restTokenProperties,
fs.oss.endpoint unchanged
- {
- Map<String, String> catalogProperties = new HashMap<>();
- catalogProperties.put("fs.oss.endpoint", "original-endpoint");
- Map<String, String> restTokenProperties = new HashMap<>();
- restTokenProperties.put("other.config", "value1");
- Map<String, String> result =
- RESTTokenFileIO.mergeTokenWithDlfEndpointHandling(
- catalogProperties, restTokenProperties);
- assertEquals("original-endpoint", result.get("fs.oss.endpoint"));
- assertEquals("value1", result.get("other.config"));
- assertEquals(2, result.size());
- }
-
- // Test case 6: Both empty maps
- {
- Map<String, String> catalogProperties = new HashMap<>();
- Map<String, String> restTokenProperties = new HashMap<>();
- Map<String, String> result =
- RESTTokenFileIO.mergeTokenWithDlfEndpointHandling(
- catalogProperties, restTokenProperties);
- assertEquals(0, result.size());
- }
-
- // Test case 7: Both null maps
- {
- Map<String, String> result =
- RESTTokenFileIO.mergeTokenWithDlfEndpointHandling(null,
null);
- assertEquals(0, result.size());
- }
-
- // Test case 8: restTokenProperties adds new keys that don't exist in
catalogProperties
- {
- Map<String, String> catalogProperties = new HashMap<>();
- catalogProperties.put("key1", "catalog1");
- Map<String, String> restTokenProperties = new HashMap<>();
- restTokenProperties.put("key2", "token2");
- restTokenProperties.put("key3", "token3");
- restTokenProperties.put(DLF_OSS_ENDPOINT.key(), "dlf-endpoint");
- Map<String, String> result =
- RESTTokenFileIO.mergeTokenWithDlfEndpointHandling(
- catalogProperties, restTokenProperties);
- assertEquals("catalog1", result.get("key1"));
- assertEquals("token2", result.get("key2"));
- assertEquals("token3", result.get("key3"));
- assertEquals("dlf-endpoint", result.get(DLF_OSS_ENDPOINT.key()));
- assertEquals("dlf-endpoint", result.get("fs.oss.endpoint")); //
DLF endpoint handling
- assertEquals(5, result.size());
- }
- }
-}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
index 0fc0f83394..f6080fb312 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
@@ -47,6 +47,7 @@ import org.junit.jupiter.api.Test;
import java.io.File;
import java.io.IOException;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -273,6 +274,12 @@ class MockRESTCatalogTest extends RESTCatalogTest {
return initCatalog(true);
}
+ @Override
+ protected Catalog newRestCatalogWithDataToken(Map<String, String>
extraOptions)
+ throws IOException {
+ return initCatalog(true, extraOptions);
+ }
+
@Override
protected void revokeTablePermission(Identifier identifier) {
restCatalogServer.addNoPermissionTable(identifier);
@@ -322,6 +329,11 @@ class MockRESTCatalogTest extends RESTCatalogTest {
}
private RESTCatalog initCatalog(boolean enableDataToken) throws
IOException {
+ return initCatalog(enableDataToken, Collections.emptyMap());
+ }
+
+ private RESTCatalog initCatalog(boolean enableDataToken, Map<String,
String> extraOptions)
+ throws IOException {
String restWarehouse = UUID.randomUUID().toString();
this.config =
new ConfigResponse(
@@ -348,6 +360,9 @@ class MockRESTCatalogTest extends RESTCatalogTest {
? dataPath.replaceFirst("file",
RESTFileIOTestLoader.SCHEME)
: dataPath;
options.set(RESTTestFileIO.DATA_PATH_CONF_KEY, path);
+ for (Map.Entry<String, String> entry : extraOptions.entrySet()) {
+ options.set(entry.getKey(), entry.getValue());
+ }
return new RESTCatalog(CatalogContext.create(options));
}
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
index 2a7c1945bf..9520f75c1f 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
@@ -105,6 +105,7 @@ import static org.apache.paimon.CoreOptions.TYPE;
import static org.apache.paimon.TableType.OBJECT_TABLE;
import static org.apache.paimon.catalog.Catalog.SYSTEM_DATABASE_NAME;
import static org.apache.paimon.rest.RESTApi.PAGE_TOKEN;
+import static org.apache.paimon.rest.RESTCatalogOptions.DLF_OSS_ENDPOINT;
import static org.apache.paimon.rest.auth.DLFToken.TOKEN_DATE_FORMATTER;
import static
org.apache.paimon.utils.SnapshotManagerTest.createSnapshotWithMillis;
import static org.assertj.core.api.Assertions.assertThat;
@@ -1597,6 +1598,25 @@ public abstract class RESTCatalogTest extends
CatalogTestBase {
}
}
+ @Test
+ void testValidToken() throws Exception {
+ Map<String, String> options = new HashMap<>();
+ options.put(DLF_OSS_ENDPOINT.key(), "test-endpoint");
+ this.catalog = newRestCatalogWithDataToken(options);
+ Identifier identifier =
+ Identifier.create("test_data_token",
"table_for_testing_valid_token");
+ RESTToken expiredDataToken =
+ new RESTToken(
+ ImmutableMap.of("akId", "akId", "akSecret",
UUID.randomUUID().toString()),
+ System.currentTimeMillis() + 3600_000L);
+ setDataTokenToRestServerForMock(identifier, expiredDataToken);
+ createTable(identifier, Maps.newHashMap(), Lists.newArrayList("col1"));
+ FileStoreTable fileStoreTable = (FileStoreTable)
catalog.getTable(identifier);
+ RESTTokenFileIO fileIO = (RESTTokenFileIO) fileStoreTable.fileIO();
+ RESTToken fileDataToken = fileIO.validToken();
+ assertEquals("test-endpoint",
fileDataToken.token().get("fs.oss.endpoint"));
+ }
+
@Test
void testRefreshFileIOWhenExpired() throws Exception {
this.catalog = newRestCatalogWithDataToken();
@@ -2802,6 +2822,9 @@ public abstract class RESTCatalogTest extends
CatalogTestBase {
protected abstract Catalog newRestCatalogWithDataToken() throws
IOException;
+ protected abstract Catalog newRestCatalogWithDataToken(Map<String, String>
extraOptions)
+ throws IOException;
+
protected abstract void revokeTablePermission(Identifier identifier);
protected abstract void authTableColumns(Identifier identifier,
List<String> columns);
