This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new e396992da6 [oss] Fix blob presigned URL validation when
fs.oss.sld.enabled is set (#10230)
e396992da6 is described below
commit e396992da6d8c0112a014fb5249cbd0399d91f85
Author: Eunbin Son <[email protected]>
AuthorDate: Sat Oct 3 18:05:54 2026 +0900
[oss] Fix blob presigned URL validation when fs.oss.sld.enabled is set
(#10230)
---
.../org/apache/paimon/jindo/JindoFileIOTest.java | 2 +
.../org/apache/paimon/oss/OSSBlobPresigner.java | 8 +-
.../java/org/apache/paimon/oss/OSSFileIOTest.java | 104 +++++++++++++++++++++
3 files changed, 112 insertions(+), 2 deletions(-)
diff --git
a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoFileIOTest.java
b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoFileIOTest.java
index 06b430a2a2..17ee5e6d5b 100644
---
a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoFileIOTest.java
+++
b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoFileIOTest.java
@@ -27,6 +27,7 @@ import org.apache.paimon.options.Options;
import org.apache.paimon.utils.Pair;
import com.aliyun.jindodata.common.JindoHadoopSystem;
+import com.aliyun.oss.ClientConfiguration;
import com.aliyun.oss.HttpMethod;
import com.aliyun.oss.OSSClient;
import com.aliyun.oss.model.ObjectMetadata;
@@ -124,6 +125,7 @@ public class JindoFileIOTest {
"paimon-blob-descriptor-sha256",
sha256Hex(descriptor.serialize()));
when(client.headObject(eq("bucket"),
contains("_bloburl_"))).thenReturn(metadata);
when(client.getEndpoint()).thenReturn(URI.create("https://oss.example.com";));
+ when(client.getClientConfiguration()).thenReturn(new
ClientConfiguration());
when(client.generatePresignedUrl(eq("bucket"), anyString(), any(),
eq(HttpMethod.GET)))
.thenAnswer(
invocation ->
diff --git
a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSBlobPresigner.java
b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSBlobPresigner.java
index 55f9089e32..c47e7de875 100644
---
a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSBlobPresigner.java
+++
b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSBlobPresigner.java
@@ -219,11 +219,15 @@ public final class OSSBlobPresigner {
private static void validatePresignedUrl(
OSSClient client, URL url, String bucket, String targetKey) throws
Exception {
URI endpoint = client.getEndpoint();
- String expectedHost = bucket + "." + endpoint.getHost();
+ // With fs.oss.sld.enabled the client signs path-style URLs: the host
is the endpoint and
+ // the bucket becomes the first path segment, so bucket and key are
still both checked.
+ boolean pathStyle = client.getClientConfiguration().isSLDEnabled();
+ String expectedHost = pathStyle ? endpoint.getHost() : bucket + "." +
endpoint.getHost();
+ String expectedPath = pathStyle ? "/" + bucket + "/" + targetKey : "/"
+ targetKey;
if (!"https".equalsIgnoreCase(endpoint.getScheme())
|| !"https".equalsIgnoreCase(url.getProtocol())
|| !expectedHost.equalsIgnoreCase(url.getHost())
- || !("/" + targetKey).equals(url.toURI().getPath())) {
+ || !expectedPath.equals(url.toURI().getPath())) {
throw new IOException("OSS client generated a presigned URL for an
invalid target.");
}
}
diff --git
a/paimon-filesystems/paimon-oss-impl/src/test/java/org/apache/paimon/oss/OSSFileIOTest.java
b/paimon-filesystems/paimon-oss-impl/src/test/java/org/apache/paimon/oss/OSSFileIOTest.java
index e801d25f81..629b14cdfc 100644
---
a/paimon-filesystems/paimon-oss-impl/src/test/java/org/apache/paimon/oss/OSSFileIOTest.java
+++
b/paimon-filesystems/paimon-oss-impl/src/test/java/org/apache/paimon/oss/OSSFileIOTest.java
@@ -93,6 +93,7 @@ public class OSSFileIOTest {
copied.setPartNumber(1);
copied.setETag("etag");
when(client.uploadPartCopy(any())).thenReturn(copied);
+ when(client.getClientConfiguration()).thenReturn(new
ClientConfiguration());
when(client.getEndpoint()).thenReturn(URI.create("https://oss-cn-hangzhou.aliyuncs.com";));
when(client.generatePresignedUrl(eq("bucket"), anyString(), any(),
eq(HttpMethod.GET)))
.thenAnswer(
@@ -158,6 +159,7 @@ public class OSSFileIOTest {
new
BlobDescriptor("oss://bucket/table/-internal.aliyuncs.com/source.blob", 0, 1);
when(client.headObject(eq("bucket"), contains("_bloburl_")))
.thenReturn(matchingMetadata(descriptor));
+ when(client.getClientConfiguration()).thenReturn(new
ClientConfiguration());
when(client.getEndpoint())
.thenReturn(URI.create("https://oss-cn-hangzhou-internal.aliyuncs.com";));
when(client.generatePresignedUrl(eq("bucket"), anyString(), any(),
eq(HttpMethod.GET)))
@@ -337,11 +339,112 @@ public class OSSFileIOTest {
"https://bucket.oss-cn-hangzhou.aliyuncs.com/table/_bloburl_hash";);
}
+ @Test
+ public void testCreateBlobPresignedUrlAcceptsPathStyleWhenSldEnabled()
throws Exception {
+ BlobDescriptor descriptor = new
BlobDescriptor("oss://bucket/table/source.blob", 0, 1);
+ String key = "table/_bloburl_" + sha256Hex(descriptor.serialize());
+ for (String endpoint :
+ new String[] {
+ "https://oss-cn-hangzhou.aliyuncs.com";,
+ "https://oss-cn-hangzhou-internal.aliyuncs.com";,
+ "https://ep-abc.oss.cn-hangzhou.privatelink.aliyuncs.com";
+ }) {
+ String signed = endpoint + "/bucket/" + key + "?Signature=test";
+ assertThat(presignWithSldMock(endpoint, descriptor,
signed)).isEqualTo(signed);
+ }
+
+ // The bucket is still verified, now as the first path segment.
+ assertThatThrownBy(
+ () ->
+ presignWithSldMock(
+ "https://oss-cn-hangzhou.aliyuncs.com";,
+ descriptor,
+
"https://oss-cn-hangzhou.aliyuncs.com/other/"; + key))
+ .isInstanceOf(java.io.IOException.class)
+ .hasMessageContaining("invalid target");
+ }
+
+ @Test
+ public void
testCreateBlobPresignedUrlAcceptsSdkPathStyleUrlWithEncodedKey() throws
Exception {
+ BlobDescriptor descriptor =
+ new BlobDescriptor("oss://bucket/table/a
b/\uD55C/source.blob", 0, 1);
+
+ URL url = new
URL(presignWithSldClient("https://oss-cn-hangzhou.aliyuncs.com";, descriptor));
+
+ assertThat(url.getHost()).isEqualTo("oss-cn-hangzhou.aliyuncs.com");
+ assertThat(url.toURI().getPath())
+ .isEqualTo(
+ "/bucket/table/a b/\uD55C/_bloburl_" +
sha256Hex(descriptor.serialize()));
+ }
+
+ @Test
+ public void
testCreateBlobPresignedUrlAcceptsSdkPathStyleUrlWithEndpointPort()
+ throws Exception {
+ BlobDescriptor descriptor = new
BlobDescriptor("oss://bucket/table/source.blob", 0, 1);
+
+ URL url =
+ new URL(
+ presignWithSldClient(
+ "https://oss-cn-hangzhou.aliyuncs.com:8443";,
descriptor));
+
+ assertThat(url.getHost()).isEqualTo("oss-cn-hangzhou.aliyuncs.com");
+ assertThat(url.getPort()).isEqualTo(8443);
+ assertThat(url.getPath())
+ .isEqualTo("/bucket/table/_bloburl_" +
sha256Hex(descriptor.serialize()));
+ }
+
+ private static String presignWithSldMock(
+ String endpoint, BlobDescriptor descriptor, String generatedUrl)
throws Exception {
+ OSSClient client = mock(OSSClient.class);
+ when(client.headObject(eq("bucket"), contains("_bloburl_")))
+ .thenReturn(matchingMetadata(descriptor));
+ ClientConfiguration configuration = new ClientConfiguration();
+ configuration.setSLDEnabled(true);
+ when(client.getClientConfiguration()).thenReturn(configuration);
+ when(client.getEndpoint()).thenReturn(URI.create(endpoint));
+ when(client.generatePresignedUrl(eq("bucket"), anyString(), any(),
eq(HttpMethod.GET)))
+ .thenReturn(presignedUrl(generatedUrl));
+ return new TestOSSFileIO(client)
+ .createBlobPresignedUrl(
+ new Path("oss://bucket/table"), descriptor,
Duration.ofMinutes(5));
+ }
+
+ /**
+ * Presigns with the URL the SDK itself signs once {@code
fs.oss.sld.enabled} has turned on
+ * second-level domain mode, so the validator is checked against the real
path-style output.
+ */
+ private static String presignWithSldClient(String endpoint, BlobDescriptor
descriptor)
+ throws Exception {
+ OSSClient signer = hadoopStyleClient(endpoint);
+ try {
+ signer.getClientConfiguration().setSLDEnabled(true);
+ OSSClient client = mock(OSSClient.class);
+ when(client.headObject(eq("bucket"), contains("_bloburl_")))
+ .thenReturn(matchingMetadata(descriptor));
+
when(client.getClientConfiguration()).thenReturn(signer.getClientConfiguration());
+ when(client.getEndpoint()).thenReturn(signer.getEndpoint());
+ when(client.generatePresignedUrl(eq("bucket"), anyString(), any(),
eq(HttpMethod.GET)))
+ .thenAnswer(
+ invocation ->
+ signer.generatePresignedUrl(
+ "bucket",
+ invocation.getArgument(1),
+ invocation.getArgument(2),
+ HttpMethod.GET));
+ return new TestOSSFileIO(client)
+ .createBlobPresignedUrl(
+ new Path("oss://bucket/table"), descriptor,
Duration.ofMinutes(5));
+ } finally {
+ signer.shutdown();
+ }
+ }
+
private static void assertInvalidPresignedUrl(String endpoint, String
generatedUrl) {
OSSClient client = mock(OSSClient.class);
BlobDescriptor descriptor = new
BlobDescriptor("oss://bucket/table/source.blob", 0, 1);
when(client.headObject(eq("bucket"), contains("_bloburl_")))
.thenReturn(matchingMetadata(descriptor));
+ when(client.getClientConfiguration()).thenReturn(new
ClientConfiguration());
when(client.getEndpoint()).thenReturn(URI.create(endpoint));
when(client.generatePresignedUrl(eq("bucket"), anyString(), any(),
eq(HttpMethod.GET)))
.thenReturn(presignedUrl(generatedUrl));
@@ -407,6 +510,7 @@ public class OSSFileIOTest {
}
private static void stubPresigning(OSSClient client) {
+ when(client.getClientConfiguration()).thenReturn(new
ClientConfiguration());
when(client.getEndpoint()).thenReturn(URI.create("https://oss-cn-hangzhou.aliyuncs.com";));
when(client.generatePresignedUrl(eq("bucket"), anyString(), any(),
eq(HttpMethod.GET)))
.thenAnswer(
