This is an automated email from the ASF dual-hosted git repository.
jerry-024 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new de872c4df6 [filesystem] Isolate Jindo OSS presigning dependencies
(#9856)
de872c4df6 is described below
commit de872c4df69f2c03394e2758c3c55659e8f5888c
Author: Junrui Lee <[email protected]>
AuthorDate: Wed Sep 16 15:20:09 2026 +0800
[filesystem] Isolate Jindo OSS presigning dependencies (#9856)
---
paimon-filesystems/paimon-jindo/pom.xml | 127 ++++-
.../apache/paimon/jindo/JindoBlobPresigner.java | 85 ++++
.../java/org/apache/paimon/jindo/JindoFileIO.java | 98 ++--
.../src/main/resources/META-INF/NOTICE | 32 ++
.../org/apache/paimon/jindo/JindoFileIOTest.java | 16 +-
.../org/apache/paimon/jindo/JindoPluginITCase.java | 522 +++++++++++++++++++++
6 files changed, 824 insertions(+), 56 deletions(-)
diff --git a/paimon-filesystems/paimon-jindo/pom.xml
b/paimon-filesystems/paimon-jindo/pom.xml
index e96a5ff63b..f70fe56d91 100644
--- a/paimon-filesystems/paimon-jindo/pom.xml
+++ b/paimon-filesystems/paimon-jindo/pom.xml
@@ -34,6 +34,7 @@
<properties>
<fs.oss.sdk.version>3.17.4</fs.oss.sdk.version>
<jindodata.version>6.9.1</jindodata.version>
+ <oss.plugin.directory>paimon-plugin-jindo-oss</oss.plugin.directory>
</properties>
<repositories>
@@ -55,6 +56,12 @@
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-oss-impl</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
@@ -160,6 +167,9 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
+ <excludes>
+
<exclude>org/apache/paimon/jindo/JindoBlobPresigner*.class</exclude>
+ </excludes>
<archive>
<manifestEntries>
<Multi-Release>true</Multi-Release>
@@ -168,34 +178,121 @@
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-oss-presigner-implementation</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+
<outputDirectory>${project.build.outputDirectory}/${oss.plugin.directory}</outputDirectory>
+ <resources>
+ <resource>
+
<directory>${project.build.outputDirectory}</directory>
+ <includes>
+
<include>org/apache/paimon/jindo/JindoBlobPresigner*.class</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>unpack-oss-runtime</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>unpack-dependencies</goal>
+ </goals>
+ <configuration>
+ <includeScope>runtime</includeScope>
+
<excludeArtifactIds>paimon-oss-impl,slf4j-api,jsr305</excludeArtifactIds>
+
<outputDirectory>${project.build.outputDirectory}/${oss.plugin.directory}</outputDirectory>
+
<excludes>META-INF/*.SF,META-INF/*.DSA,META-INF/*.RSA,META-INF/maven/**</excludes>
+ </configuration>
+ </execution>
+ <execution>
+ <id>unpack-oss-presigner</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>unpack</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-oss-impl</artifactId>
+ <version>${project.version}</version>
+ <overWrite>true</overWrite>
+
<outputDirectory>${project.build.outputDirectory}/${oss.plugin.directory}</outputDirectory>
+
<includes>org/apache/paimon/oss/OSSBlobPresigner*.class</includes>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>integration-tests</id>
+ <configuration>
+ <systemPropertyVariables>
+
<jindo.plugin.jar>${project.build.directory}/${project.build.finalName}.jar</jindo.plugin.jar>
+ </systemPropertyVariables>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<configuration>
<artifactSet>
<includes combine.children="append">
-
<include>org.apache.paimon:paimon-oss-impl</include>
- <include>com.aliyun.oss:aliyun-sdk-oss</include>
- <include>com.aliyun:*</include>
- <include>com.google.code.gson:gson</include>
- <include>commons-codec:commons-codec</include>
- <include>commons-logging:commons-logging</include>
- <include>io.opentracing:*</include>
- <include>org.apache.httpcomponents:*</include>
- <include>org.codehaus.jettison:jettison</include>
- <include>org.ini4j:ini4j</include>
- <include>org.jdom:jdom2</include>
- <include>stax:stax-api</include>
+ <include>*:*</include>
</includes>
+ <excludes>
+ <exclude>org.slf4j:slf4j-api</exclude>
+ <exclude>com.google.code.findbugs:jsr305</exclude>
+ </excludes>
</artifactSet>
<filters>
<filter>
-
<artifact>org.apache.paimon:paimon-oss-impl</artifact>
+ <artifact>*:*</artifact>
<includes>
-
<include>org/apache/paimon/oss/OSSBlobPresigner.class</include>
- <include>META-INF/versions/11/**</include>
+ <include>org/apache/paimon/jindo/**</include>
+ <include>${oss.plugin.directory}/**</include>
+ <include>META-INF/MANIFEST.MF</include>
+ <include>META-INF/LICENSE*</include>
+
<include>META-INF/licenses/LICENSE.jdom</include>
+
<include>META-INF/licenses/LICENSE.jaxb</include>
+ <include>META-INF/NOTICE*</include>
+ <include>META-INF/DEPENDENCIES</include>
+ <include>META-INF/maven/**</include>
+
<include>META-INF/services/org.apache.paimon.fs.FileIOLoader</include>
+
<include>META-INF/versions/11/javax/xml/bind/**</include>
</includes>
</filter>
+ <filter>
+
<artifact>org.apache.paimon:paimon-oss-impl</artifact>
+ <excludes>
+ <exclude>META-INF/NOTICE*</exclude>
+ </excludes>
+ </filter>
</filters>
</configuration>
</plugin>
diff --git
a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoBlobPresigner.java
b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoBlobPresigner.java
new file mode 100644
index 0000000000..57e399e9d5
--- /dev/null
+++
b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoBlobPresigner.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.jindo;
+
+import org.apache.paimon.data.BlobDescriptor;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.oss.OSSBlobPresigner;
+import org.apache.paimon.utils.StringUtils;
+
+import com.aliyun.oss.OSSClient;
+import com.aliyun.oss.OSSClientBuilder;
+
+import java.io.IOException;
+import java.time.Duration;
+
+/** OSS presigning implementation loaded from the private plugin directory. */
+public class JindoBlobPresigner implements JindoFileIO.BlobPresigner {
+
+ private OSSClient client;
+
+ public JindoBlobPresigner() {}
+
+ JindoBlobPresigner(OSSClient client) {
+ this.client = client;
+ }
+
+ @Override
+ public void configure(Options options) {
+ client = createBlobClient(options);
+ }
+
+ @Override
+ public String create(Path tableRoot, BlobDescriptor descriptor, Duration
validity)
+ throws IOException {
+ return OSSBlobPresigner.create(client, tableRoot, descriptor,
validity);
+ }
+
+ @Override
+ public void close() {
+ client.shutdown();
+ }
+
+ static OSSClient createBlobClient(Options options) {
+ String endpoint = options.get("fs.oss.endpoint");
+ if (!endpoint.contains("://")) {
+ endpoint = "https://"; + endpoint;
+ }
+ String securityToken = options.get("fs.oss.securityToken");
+ OSSClientBuilder builder = new OSSClientBuilder();
+ OSSClient client =
+ (OSSClient)
+ (StringUtils.isNullOrWhitespaceOnly(securityToken)
+ ? builder.build(
+ endpoint,
+ options.get("fs.oss.accessKeyId"),
+ options.get("fs.oss.accessKeySecret"))
+ : builder.build(
+ endpoint,
+ options.get("fs.oss.accessKeyId"),
+ options.get("fs.oss.accessKeySecret"),
+ securityToken));
+ String region = options.get("fs.oss.region");
+ if (!StringUtils.isNullOrWhitespaceOnly(region)) {
+ client.setRegion(region);
+ }
+ return client;
+ }
+}
diff --git
a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java
b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java
index 1e4646d4b9..31e0d440d9 100644
---
a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java
+++
b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java
@@ -25,7 +25,7 @@ import org.apache.paimon.fs.HadoopOptionsProvider;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.TwoPhaseOutputStream;
import org.apache.paimon.options.Options;
-import org.apache.paimon.oss.OSSBlobPresigner;
+import org.apache.paimon.plugin.PluginLoader;
import org.apache.paimon.utils.IOUtils;
import org.apache.paimon.utils.Pair;
import org.apache.paimon.utils.SensitiveConfigUtils;
@@ -36,8 +36,6 @@ import com.aliyun.jindodata.dls.JindoDlsFileSystem;
import com.aliyun.jindodata.oss.JindoOssFileSystem;
import com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider;
import com.aliyun.jindodata.store.JindoMpuStore;
-import com.aliyun.oss.OSSClient;
-import com.aliyun.oss.OSSClientBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;
@@ -70,11 +68,9 @@ public class JindoFileIO extends HadoopCompliantFileIO
implements HadoopOptionsP
*/
private static final String[] CONFIG_PREFIXES = {"fs."};
- private static final String OSS_ENDPOINT = "fs.oss.endpoint";
private static final String OSS_ACCESS_KEY_ID = "fs.oss.accessKeyId";
private static final String OSS_ACCESS_KEY_SECRET =
"fs.oss.accessKeySecret";
private static final String OSS_SECURITY_TOKEN = "fs.oss.securityToken";
- private static final String OSS_REGION = "fs.oss.region";
private static final String OSS_USER_AGENT_EXTENDED =
"fs.oss.user.agent.extended";
private static final String OSS_SHOW_DIR_TIMESTAMP =
"fs.oss.show-dir-timestamp";
private static final String DLF_ACCESS_TRACKING_EXTENDED_INFO =
@@ -99,12 +95,12 @@ public class JindoFileIO extends HadoopCompliantFileIO
implements HadoopOptionsP
private Options hadoopOptions;
private Options hadoopOptionsWithCache;
private boolean allowCache = true;
- private transient OSSClient blobClient;
+ private transient BlobPresigner blobPresigner;
public JindoFileIO() {}
- JindoFileIO(OSSClient blobClient) {
- this.blobClient = blobClient;
+ JindoFileIO(BlobPresigner blobPresigner) {
+ this.blobPresigner = blobPresigner;
}
@Override
@@ -229,40 +225,45 @@ public class JindoFileIO extends HadoopCompliantFileIO
implements HadoopOptionsP
@Override
public String createBlobPresignedUrl(
Path tableRoot, BlobDescriptor descriptor, Duration validity)
throws IOException {
- return OSSBlobPresigner.create(blobClient(), tableRoot, descriptor,
validity);
+ BlobPresigner presigner = blobPresigner();
+ Thread thread = Thread.currentThread();
+ ClassLoader previous = thread.getContextClassLoader();
+ try {
+
thread.setContextClassLoader(presigner.getClass().getClassLoader());
+ return presigner.create(tableRoot, descriptor, validity);
+ } finally {
+ thread.setContextClassLoader(previous);
+ }
}
- private synchronized OSSClient blobClient() {
- if (blobClient == null) {
- blobClient = createBlobClient(hadoopOptions);
+ private synchronized BlobPresigner blobPresigner() {
+ if (blobPresigner == null) {
+ PluginLoader loader = BlobPlugin.getLoader();
+ Thread thread = Thread.currentThread();
+ ClassLoader previous = thread.getContextClassLoader();
+ try {
+ thread.setContextClassLoader(loader.submoduleClassLoader());
+ BlobPresigner presigner =
+
loader.newInstance("org.apache.paimon.jindo.JindoBlobPresigner");
+ presigner.configure(hadoopOptions);
+ blobPresigner = presigner;
+ } finally {
+ thread.setContextClassLoader(previous);
+ }
}
- return blobClient;
+ return blobPresigner;
}
- static OSSClient createBlobClient(Options options) {
- String endpoint = options.get(OSS_ENDPOINT);
- if (!endpoint.contains("://")) {
- endpoint = "https://"; + endpoint;
- }
- String securityToken = options.get(OSS_SECURITY_TOKEN);
- OSSClientBuilder builder = new OSSClientBuilder();
- OSSClient client =
- (OSSClient)
- (StringUtils.isNullOrWhitespaceOnly(securityToken)
- ? builder.build(
- endpoint,
- options.get(OSS_ACCESS_KEY_ID),
- options.get(OSS_ACCESS_KEY_SECRET))
- : builder.build(
- endpoint,
- options.get(OSS_ACCESS_KEY_ID),
- options.get(OSS_ACCESS_KEY_SECRET),
- securityToken));
- String region = options.get(OSS_REGION);
- if (!StringUtils.isNullOrWhitespaceOnly(region)) {
- client.setRegion(region);
+ private static class BlobPlugin {
+
+ private static PluginLoader loader;
+
+ private static synchronized PluginLoader getLoader() {
+ if (loader == null) {
+ loader = new PluginLoader("paimon-plugin-jindo-oss");
+ }
+ return loader;
}
- return client;
}
@Override
@@ -314,9 +315,16 @@ public class JindoFileIO extends HadoopCompliantFileIO
implements HadoopOptionsP
@Override
public synchronized void close() {
- if (blobClient != null) {
- blobClient.shutdown();
- blobClient = null;
+ if (blobPresigner != null) {
+ Thread thread = Thread.currentThread();
+ ClassLoader previous = thread.getContextClassLoader();
+ try {
+
thread.setContextClassLoader(blobPresigner.getClass().getClassLoader());
+ blobPresigner.close();
+ blobPresigner = null;
+ } finally {
+ thread.setContextClassLoader(previous);
+ }
}
if (!allowCache) {
fsMap.values().stream().map(Pair::getKey).forEach(IOUtils::closeQuietly);
@@ -324,6 +332,18 @@ public class JindoFileIO extends HadoopCompliantFileIO
implements HadoopOptionsP
}
}
+ /** Contract shared with the isolated OSS implementation. */
+ public interface BlobPresigner extends AutoCloseable {
+
+ void configure(Options options);
+
+ String create(Path tableRoot, BlobDescriptor descriptor, Duration
validity)
+ throws IOException;
+
+ @Override
+ void close();
+ }
+
private static class CacheKey {
private final Options options;
diff --git a/paimon-filesystems/paimon-jindo/src/main/resources/META-INF/NOTICE
b/paimon-filesystems/paimon-jindo/src/main/resources/META-INF/NOTICE
new file mode 100644
index 0000000000..e5d564724e
--- /dev/null
+++ b/paimon-filesystems/paimon-jindo/src/main/resources/META-INF/NOTICE
@@ -0,0 +1,32 @@
+paimon-jindo
+Copyright 2023-2026 The Apache Software Foundation
+
+This project includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This project bundles the following dependencies under the Apache Software
License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt)
+
+- com.aliyun.oss:aliyun-sdk-oss:3.17.4
+- com.aliyun:aliyun-java-sdk-core:4.5.10
+- com.aliyun:aliyun-java-sdk-kms:2.11.0
+- com.aliyun:aliyun-java-sdk-ram:3.1.0
+- com.google.code.gson:gson:2.8.6
+- commons-codec:commons-codec:1.11
+- commons-logging:commons-logging:1.1.3
+- io.opentracing:opentracing-api:0.33.0
+- io.opentracing:opentracing-noop:0.33.0
+- io.opentracing:opentracing-util:0.33.0
+- org.apache.httpcomponents:httpclient:4.5.13
+- org.apache.httpcomponents:httpcore:4.4.13
+- org.codehaus.jettison:jettison:1.5.4
+- org.ini4j:ini4j:0.5.4
+
+This project bundles the following dependencies under the JDOM license.
+You find it under licenses/LICENSE.jdom.
+
+- org.jdom:jdom2:2.0.6.1
+
+This project bundles the following dependencies under the CDDL 1.1 license.
+You find it under licenses/LICENSE.jaxb.
+
+- javax.xml.bind:jaxb-api:2.3.1
diff --git
a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoFileIOTest.java
b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoFileIOTest.java
index 91406badb4..06b430a2a2 100644
---
a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoFileIOTest.java
+++
b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoFileIOTest.java
@@ -37,6 +37,7 @@ import java.net.URI;
import java.net.URL;
import java.security.MessageDigest;
import java.time.Duration;
+import java.util.Date;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
@@ -85,7 +86,7 @@ public class JindoFileIOTest {
options.set("fs.oss.accessKeySecret", "access-secret");
options.set("fs.oss.securityToken", "security-token");
- OSSClient client = JindoFileIO.createBlobClient(options);
+ OSSClient client = JindoBlobPresigner.createBlobClient(options);
try {
assertThat(client.getEndpoint()).isEqualTo(URI.create("https://oss.example.com";));
assertThat(client.getObjectOperation().getRegion()).isEqualTo("cn-hangzhou");
@@ -95,6 +96,17 @@ public class JindoFileIOTest {
.isEqualTo("access-secret");
assertThat(client.getCredentialsProvider().getCredentials().getSecurityToken())
.isEqualTo("security-token");
+ URL signedUrl =
+ client.generatePresignedUrl(
+ "bucket", "object", new
Date(System.currentTimeMillis() + 60_000));
+ assertThat(signedUrl.getProtocol()).isEqualTo("https");
+
assertThat(signedUrl.getHost()).isEqualTo("bucket.oss.example.com");
+ assertThat(signedUrl.getPath()).isEqualTo("/object");
+ assertThat(signedUrl.getQuery())
+ .contains(
+ "OSSAccessKeyId=access-key",
+ "Signature=",
+ "security-token=security-token");
} finally {
client.shutdown();
}
@@ -119,7 +131,7 @@ public class JindoFileIOTest {
"https://bucket.oss.example.com/";
+ invocation.getArgument(1)));
- JindoFileIO fileIO = new JindoFileIO(client);
+ JindoFileIO fileIO = new JindoFileIO(new JindoBlobPresigner(client));
assertThat(fileIO.createBlobPresignedUrl(tableRoot, descriptor,
Duration.ofHours(1)))
.startsWith("https://bucket.oss.example.com/table/data/_bloburl_";);
diff --git
a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoPluginITCase.java
b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoPluginITCase.java
new file mode 100644
index 0000000000..7f4e2451ae
--- /dev/null
+++
b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoPluginITCase.java
@@ -0,0 +1,522 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.jindo;
+
+import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.data.BlobDescriptor;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.options.Options;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.URI;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
+import java.util.ServiceLoader;
+import java.util.Set;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.stream.Collectors;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Packaged-artifact tests for {@link JindoFileIO}. */
+public class JindoPluginITCase {
+
+ private static final String PLUGIN_DIRECTORY = "paimon-plugin-jindo-oss/";
+
+ private static final String PRESIGNER_IMPLEMENTATION =
+ "org.apache.paimon.jindo.JindoBlobPresigner";
+
+ @TempDir java.nio.file.Path tempDir;
+
+ @Test
+ public void testPluginJarContents() throws Exception {
+ try (JarFile jar = new JarFile(pluginJar())) {
+ Set<String> entries =
+ jar.stream()
+ .map(JarEntry::getName)
+ .map(JindoPluginITCase::normalizeEntry)
+ .collect(Collectors.toSet());
+ for (String originalPackage :
+ Arrays.asList(
+ "com/google/gson/",
+ "com/aliyun/oss/",
+ "com/aliyuncs/",
+ "org/apache/http/",
+ "org/apache/commons/codec/",
+ "org/apache/commons/logging/",
+ "org/jdom2/",
+ "org/codehaus/jettison/",
+ "io/opentracing/",
+ "org/ini4j/",
+ "javax/xml/stream/")) {
+ assertThat(entries)
+ .as("Private package %s at the root of %s",
originalPackage, jar.getName())
+ .noneMatch(entry -> entry.startsWith(originalPackage));
+ }
+ for (String representativeClass :
+ Arrays.asList(
+ PRESIGNER_IMPLEMENTATION.replace('.', '/'),
+ "org/apache/paimon/oss/OSSBlobPresigner",
+ "com/google/gson/Gson",
+ "com/aliyun/oss/OSSClient",
+ "com/aliyuncs/DefaultAcsClient",
+ "org/apache/http/impl/client/HttpClients",
+ "org/apache/commons/codec/binary/Base64",
+ "org/apache/commons/logging/LogFactory",
+ "org/jdom2/Document",
+ "org/codehaus/jettison/json/JSONObject",
+ "io/opentracing/Tracer",
+ "org/ini4j/Ini")) {
+ assertThat(entries)
+ .contains(PLUGIN_DIRECTORY + representativeClass +
".class")
+ .doesNotContain(representativeClass + ".class");
+ }
+ for (String rootEntry :
+ Arrays.asList(
+ "org/apache/paimon/jindo/JindoFileIO.class",
+
"org/apache/paimon/jindo/JindoFileIO$BlobPresigner.class",
+ "org/apache/paimon/jindo/JindoLoader.class",
+
"META-INF/services/org.apache.paimon.fs.FileIOLoader")) {
+ assertThat(entries)
+ .contains(rootEntry)
+ .doesNotContain(PLUGIN_DIRECTORY + rootEntry);
+ }
+ for (String resource :
+ Arrays.asList(
+ "versioninfo.properties", "common.properties",
"oss.properties")) {
+ assertThat(entries).contains(PLUGIN_DIRECTORY +
resource).doesNotContain(resource);
+ }
+ for (String providedPackage :
+ Arrays.asList("com/aliyun/jindodata/",
"org/apache/hadoop/", "org/slf4j/")) {
+ assertThat(entries)
+ .noneMatch(
+ entry ->
+ entry.startsWith(providedPackage)
+ || entry.startsWith(
+ PLUGIN_DIRECTORY +
providedPackage));
+ }
+ assertPrivatePaimonClasses(entries);
+ }
+ }
+
+ @Test
+ public void testPluginWithMinimalHostClasspath() throws Exception {
+ assertPluginClassLoading(false, false);
+ }
+
+ @Test
+ public void testPluginBeforeHostDependencies() throws Exception {
+ assertPluginClassLoading(true, true);
+ }
+
+ @Test
+ public void testPluginAfterHostDependencies() throws Exception {
+ assertPluginClassLoading(false, true);
+ }
+
+ @Test
+ public void testPrivatePaimonClassesAllowNestedImplementations() {
+ Set<String> entries =
+ new HashSet<>(
+ Arrays.asList(
+ PLUGIN_DIRECTORY
+ +
"org/apache/paimon/jindo/JindoBlobPresigner.class",
+ PLUGIN_DIRECTORY +
"org/apache/paimon/oss/OSSBlobPresigner.class",
+ PLUGIN_DIRECTORY
+ +
"org/apache/paimon/jindo/JindoBlobPresigner$1.class",
+ PLUGIN_DIRECTORY
+ +
"META-INF/versions/11/org/apache/paimon/oss/OSSBlobPresigner$Helper.class"));
+ assertPrivatePaimonClasses(entries);
+ }
+
+ @Test
+ public void testPrivatePaimonClassesRejectHostClasses() {
+ for (String unexpectedClass :
+ Arrays.asList(
+ PLUGIN_DIRECTORY + "org/apache/paimon/fs/Path.class",
+ PLUGIN_DIRECTORY +
"META-INF/versions/11/org/apache/paimon/fs/Path.class",
+ "META-INF/versions/11/"
+ + PLUGIN_DIRECTORY
+ + "org/apache/paimon/fs/Path.class",
+ PLUGIN_DIRECTORY
+ +
"org/apache/paimon/jindo/JindoFileIO$BlobPresigner.class",
+ PLUGIN_DIRECTORY +
"org/apache/paimon/oss/OSSBlobPresignerOther.class")) {
+ Set<String> entries =
+ new HashSet<>(
+ Arrays.asList(
+ PLUGIN_DIRECTORY
+ +
"org/apache/paimon/jindo/JindoBlobPresigner.class",
+ PLUGIN_DIRECTORY
+ +
"org/apache/paimon/oss/OSSBlobPresigner.class",
+ unexpectedClass));
+ assertThatThrownBy(() -> assertPrivatePaimonClasses(entries))
+ .as("Reject private host class %s", unexpectedClass)
+ .isInstanceOf(AssertionError.class);
+ }
+ }
+
+ private static String normalizeEntry(String entry) {
+ return entry.replaceFirst("^META-INF/versions/[0-9]+/", "")
+ .replaceFirst(
+ "^" + PLUGIN_DIRECTORY + "META-INF/versions/[0-9]+/",
PLUGIN_DIRECTORY);
+ }
+
+ private static void assertPrivatePaimonClasses(Set<String> entries) {
+ assertThat(entries.stream().map(JindoPluginITCase::normalizeEntry))
+ .filteredOn(
+ entry ->
+ entry.startsWith(PLUGIN_DIRECTORY +
"org/apache/paimon/")
+ && entry.endsWith(".class"))
+ .allMatch(
+ entry ->
+ entry.matches(
+ PLUGIN_DIRECTORY
+ +
"org/apache/paimon/(jindo/JindoBlobPresigner|oss/OSSBlobPresigner)(\\$[^/]+)?\\.class"));
+ }
+
+ private static File pluginJar() throws Exception {
+ File jar = new
File(System.getProperty("jindo.plugin.jar")).getCanonicalFile();
+ assertThat(jar).isFile();
+ return jar;
+ }
+
+ private static File classLocation(Class<?> type) throws Exception {
+ return new
File(type.getProtectionDomain().getCodeSource().getLocation().toURI())
+ .getCanonicalFile();
+ }
+
+ private static Class<?> loadClassFrom(
+ ClassLoader classLoader, String className, File expectedJar)
throws Exception {
+ Class<?> type = Class.forName(className, true, classLoader);
+ assertThat(type.getClassLoader()).as("Loader of %s",
className).isSameAs(classLoader);
+ assertThat(classLocation(type)).as("Source of %s",
className).isEqualTo(expectedJar);
+ return type;
+ }
+
+ private void assertPluginClassLoading(boolean jarFirst, boolean
conflictingDependencies)
+ throws Exception {
+ File jar = pluginJar();
+ List<URL> urls = new ArrayList<>();
+ urls.add(tempDir.toUri().toURL());
+ List<String> hostClasses =
+ new ArrayList<>(
+ Arrays.asList(
+ "org.apache.paimon.plugin.PluginLoader",
+ "org.apache.paimon.options.Options",
+
"org.apache.paimon.shade.guava30.com.google.common.collect.Iterators",
+ "com.aliyun.jindodata.Version",
+
"com.aliyun.jindodata.common.JindoHadoopSystem"));
+ for (String artifact : Arrays.asList("hadoop-common", "slf4j-api")) {
+ urls.add(hostDependency(artifact).toURI().toURL());
+ }
+ if (conflictingDependencies) {
+ urls.add(hostDependency("commons-logging").toURI().toURL());
+ hostClasses.addAll(
+ Arrays.asList(
+ "com.google.gson.Gson",
+ "org.apache.http.client.methods.HttpGet",
+ "org.apache.http.HttpVersion",
+ "org.apache.paimon.oss.OSSBlobPresigner",
+ "com.aliyun.oss.OSSClient"));
+ }
+ for (String className : hostClasses) {
+ URL location = classLocation(testClass(className)).toURI().toURL();
+ if (!urls.contains(location)) {
+ urls.add(location);
+ }
+ }
+ Files.write(
+ tempDir.resolve("versioninfo.properties"),
+ "version=host-version\n".getBytes(StandardCharsets.UTF_8));
+ Files.write(
+ tempDir.resolve("common.properties"),
+
"ConnectionError=host-error\n".getBytes(StandardCharsets.UTF_8));
+ urls.add(jarFirst ? 0 : urls.size(), jar.toURI().toURL());
+ Thread thread = Thread.currentThread();
+ ClassLoader contextClassLoader = thread.getContextClassLoader();
+ try (URLClassLoader host =
+ new URLClassLoader(
+ urls.toArray(new URL[0]),
ClassLoader.getSystemClassLoader().getParent())) {
+ thread.setContextClassLoader(host);
+ assertThatThrownBy(() -> host.loadClass(PRESIGNER_IMPLEMENTATION))
+ .isInstanceOf(ClassNotFoundException.class);
+ assertThat(host.getResource(PRESIGNER_IMPLEMENTATION.replace('.',
'/') + ".class"))
+ .isNull();
+ Class<?> optionsClass = host.loadClass(Options.class.getName());
+ Map<String, String> settings = new HashMap<>();
+ settings.put("fs.oss.endpoint", "oss.example.com");
+ settings.put("fs.oss.region", "cn-hangzhou");
+ settings.put("fs.oss.accessKeyId", "access-key");
+ settings.put("fs.oss.accessKeySecret", "access-secret");
+ settings.put("fs.oss.securityToken", "security-token");
+ Object options =
optionsClass.getConstructor(Map.class).newInstance(settings);
+ Class<?> implementation = assertPublicEntryPoint(host, jar,
options);
+ try (URLClassLoader plugin = (URLClassLoader)
implementation.getClassLoader()) {
+
assertThat(implementation.getSimpleName()).isEqualTo("JindoBlobPresigner");
+ assertThat(implementation.getEnclosingClass()).isNull();
+
assertThat(implementation.getClassLoader()).isSameAs(plugin).isNotSameAs(host);
+ assertThat(implementation.getInterfaces())
+
.contains(host.loadClass(JindoFileIO.BlobPresigner.class.getName()));
+ Class<?> logFactory =
plugin.loadClass("org.apache.commons.logging.LogFactory");
+ if (conflictingDependencies) {
+ assertThat(logFactory)
+
.isSameAs(host.loadClass("org.apache.commons.logging.LogFactory"));
+ } else {
+ assertThat(logFactory.getClassLoader()).isSameAs(plugin);
+ assertThatThrownBy(
+ () ->
host.loadClass("org.apache.commons.logging.LogFactory"))
+ .isInstanceOf(ClassNotFoundException.class);
+ }
+ for (String className :
+ Arrays.asList(
+ "com.google.gson.Gson",
+ "org.apache.http.client.methods.HttpGet",
+ "org.apache.paimon.oss.OSSBlobPresigner",
+ "com.aliyun.oss.OSSClient")) {
+ Class<?> privateClass = Class.forName(className, true,
plugin);
+ assertThat(privateClass.getClassLoader()).isSameAs(plugin);
+
assertThat(privateClass.getProtectionDomain().getCodeSource().getLocation())
+ .isEqualTo(host.getResource(PLUGIN_DIRECTORY));
+ if (conflictingDependencies) {
+ assertThat(
+ loadClassFrom(
+ host,
+ className,
+
classLocation(testClass(className))))
+ .isNotSameAs(privateClass);
+ } else {
+ assertThatThrownBy(() -> host.loadClass(className))
+ .isInstanceOf(ClassNotFoundException.class);
+ assertThat(host.getResource(className.replace('.',
'/') + ".class"))
+ .isNull();
+ }
+ }
+ assertGson(plugin);
+ if (conflictingDependencies) {
+ assertGson(host);
+ }
+ assertPluginResources(host, plugin);
+ assertBlobClient(implementation, optionsClass, options,
plugin);
+ assertThat(thread.getContextClassLoader()).isSameAs(host);
+ }
+ Class<?> serviceClass =
host.loadClass("org.apache.paimon.fs.FileIOLoader");
+ boolean found = false;
+ for (Object provider : ServiceLoader.load(serviceClass, host)) {
+ if
(provider.getClass().getName().equals("org.apache.paimon.jindo.JindoLoader")) {
+ loadClassFrom(host, provider.getClass().getName(), jar);
+
assertThat(serviceClass.getMethod("getScheme").invoke(provider))
+ .isEqualTo("oss");
+ found = true;
+ break;
+ }
+ }
+ assertThat(found).as("Jindo FileIOLoader SPI from plugin
artifact").isTrue();
+ } finally {
+ thread.setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ private static Class<?> testClass(String name) throws
ClassNotFoundException {
+ return Class.forName(name, false,
JindoPluginITCase.class.getClassLoader());
+ }
+
+ private static File hostDependency(String artifact) throws IOException {
+ String classPath =
+ System.getProperty(
+ "surefire.test.class.path",
System.getProperty("java.class.path"));
+ for (String entry : classPath.split(File.pathSeparator)) {
+ File dependency = new File(entry);
+ if (dependency.isFile()
+ && dependency.getName().startsWith(artifact + "-")
+ && dependency.getName().endsWith(".jar")) {
+ return dependency.getCanonicalFile();
+ }
+ }
+ throw new AssertionError("Missing host dependency: " + artifact);
+ }
+
+ private static Class<?> assertPublicEntryPoint(ClassLoader host, File jar,
Object options)
+ throws Exception {
+ Class<?> fileIOClass = loadClassFrom(host,
"org.apache.paimon.jindo.JindoFileIO", jar);
+ assertThat(fileIOClass.getDeclaredClasses())
+
.contains(host.loadClass(JindoFileIO.BlobPresigner.class.getName()))
+ .allSatisfy(nestedClass ->
assertThat(nestedClass.getClassLoader()).isSameAs(host));
+ Object fileIO = fileIOClass.getConstructor().newInstance();
+ Field presignerField = fileIOClass.getDeclaredField("blobPresigner");
+ presignerField.setAccessible(true);
+ try {
+ assertThat(presignerField.get(fileIO)).as("Presigner is
lazy").isNull();
+ Class<?> contextClass =
host.loadClass(CatalogContext.class.getName());
+ Object context =
+ contextClass.getMethod("create",
options.getClass()).invoke(null, options);
+ fileIOClass.getMethod("configure", contextClass).invoke(fileIO,
context);
+ assertThat(presignerField.get(fileIO))
+ .as("Configuration does not load the plugin")
+ .isNull();
+ Class<?> pathClass = host.loadClass(Path.class.getName());
+ Class<?> descriptorClass =
host.loadClass(BlobDescriptor.class.getName());
+ Object tableRoot =
+
pathClass.getConstructor(String.class).newInstance("oss://bucket/table");
+ Object descriptor =
+ descriptorClass
+ .getConstructor(String.class, long.class,
long.class)
+ .newInstance("oss://bucket/table/data/file", 10L,
20L);
+ Method presign =
+ fileIOClass.getMethod(
+ "createBlobPresignedUrl", pathClass,
descriptorClass, Duration.class);
+ for (int attempt = 0; attempt < 2; attempt++) {
+ assertThatThrownBy(
+ () -> presign.invoke(fileIO, tableRoot,
descriptor, Duration.ZERO))
+ .isInstanceOf(InvocationTargetException.class)
+ .hasCauseInstanceOf(IOException.class)
+ .hasStackTraceContaining("positive whole seconds")
+ .hasStackTraceContaining(PRESIGNER_IMPLEMENTATION +
".create")
+
.hasStackTraceContaining("org.apache.paimon.oss.OSSBlobPresigner.create");
+
assertThat(Thread.currentThread().getContextClassLoader()).isSameAs(host);
+ }
+ Class<?> implementation = presignerField.get(fileIO).getClass();
+
assertThat(implementation.getName()).isEqualTo(PRESIGNER_IMPLEMENTATION);
+ return implementation;
+ } finally {
+ fileIOClass.getMethod("close").invoke(fileIO);
+
assertThat(Thread.currentThread().getContextClassLoader()).isSameAs(host);
+ }
+ }
+
+ private static void assertGson(ClassLoader loader) throws Exception {
+ Class<?> gsonClass = loader.loadClass("com.google.gson.Gson");
+ Object gson = gsonClass.getConstructor().newInstance();
+ assertThat(
+ gsonClass
+ .getMethod("toJson", Object.class)
+ .invoke(gson,
Collections.singletonMap("plugin", 1)))
+ .isEqualTo("{\"plugin\":1}");
+ }
+
+ private static void assertBlobClient(
+ Class<?> implementation, Class<?> optionsClass, Object options,
ClassLoader plugin)
+ throws Exception {
+ Method createClient =
implementation.getDeclaredMethod("createBlobClient", optionsClass);
+ createClient.setAccessible(true);
+ Class<?> clientClass = plugin.loadClass("com.aliyun.oss.OSSClient");
+ assertThat(clientClass.getClassLoader()).isSameAs(plugin);
+ Object client = createClient.invoke(null, options);
+ try {
+ assertThat(client.getClass()).isSameAs(clientClass);
+ assertThat(clientClass.getMethod("getEndpoint").invoke(client))
+ .isEqualTo(URI.create("https://oss.example.com";));
+ Object operation =
clientClass.getMethod("getObjectOperation").invoke(client);
+
assertThat(operation.getClass().getMethod("getRegion").invoke(operation))
+ .isEqualTo("cn-hangzhou");
+ URL signedUrl =
+ (URL)
+ clientClass
+ .getMethod(
+ "generatePresignedUrl",
+ String.class,
+ String.class,
+ Date.class)
+ .invoke(
+ client,
+ "bucket",
+ "object",
+ new
Date(System.currentTimeMillis() + 60_000));
+ assertThat(signedUrl.getProtocol()).isEqualTo("https");
+
assertThat(signedUrl.getHost()).isEqualTo("bucket.oss.example.com");
+ assertThat(signedUrl.getPath()).isEqualTo("/object");
+ assertThat(signedUrl.getQuery())
+ .contains(
+ "OSSAccessKeyId=access-key",
+ "Signature=",
+ "security-token=security-token");
+ } finally {
+ client.getClass().getMethod("shutdown").invoke(client);
+ }
+ }
+
+ private static void assertPluginResources(ClassLoader host, ClassLoader
plugin)
+ throws Exception {
+ for (String resource : Arrays.asList("versioninfo.properties",
"common.properties")) {
+ URL pluginResource = plugin.getResource(resource);
+ assertThat(pluginResource).isNotNull();
+ assertThat(pluginResource.toExternalForm())
+ .contains("!/" + PLUGIN_DIRECTORY + resource);
+ assertThat(Collections.list(plugin.getResources(resource)))
+ .containsExactly(pluginResource);
+ }
+ assertThat(resourceProperties(host,
"versioninfo.properties").getProperty("version"))
+ .isEqualTo("host-version");
+ assertThat(resourceProperties(host,
"common.properties").getProperty("ConnectionError"))
+ .isEqualTo("host-error");
+ String version =
+ resourceProperties(plugin,
"versioninfo.properties").getProperty("version");
+ assertThat(version).isNotBlank().isNotIn("host-version",
"unknown-version");
+ assertThat(
+
plugin.loadClass("com.aliyun.oss.common.utils.VersionInfoUtils")
+ .getMethod("getVersion")
+ .invoke(null))
+ .isEqualTo(version);
+ Class<?> managerClass =
plugin.loadClass("com.aliyun.oss.common.utils.ResourceManager");
+ Object manager =
+ managerClass
+ .getMethod("getInstance", String.class, Locale.class)
+ .invoke(null, "common", Locale.ROOT);
+ String message =
+ resourceProperties(plugin,
"common.properties").getProperty("ConnectionError");
+ assertThat(message).isNotBlank().isNotEqualTo("host-error");
+ assertThat(
+ managerClass
+ .getMethod("getString", String.class)
+ .invoke(manager, "ConnectionError"))
+ .isEqualTo(message);
+ }
+
+ private static Properties resourceProperties(ClassLoader loader, String
resource)
+ throws IOException {
+ Properties properties = new Properties();
+ try (InputStream stream = loader.getResourceAsStream(resource)) {
+ assertThat(stream).as(resource).isNotNull();
+ properties.load(stream);
+ }
+ return properties;
+ }
+}
