This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push: new c39e88dcf0 [HUDI-4495] Fix handling of S3 paths incompatible with java URI standards (#6237) c39e88dcf0 is described below commit c39e88dcf0c196bca08aced6e16d47fc72f5e83e Author: Udit Mehrotra <udi...@amazon.com> AuthorDate: Thu Jul 28 20:04:14 2022 -0700 [HUDI-4495] Fix handling of S3 paths incompatible with java URI standards (#6237) --- .../apache/hudi/common/fs/HoodieWrapperFileSystem.java | 7 +++++-- .../org/apache/hudi/common/fs/TestStorageSchemes.java | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieWrapperFileSystem.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieWrapperFileSystem.java index bceea8e367..2979696be7 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieWrapperFileSystem.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieWrapperFileSystem.java @@ -145,8 +145,11 @@ public class HoodieWrapperFileSystem extends FileSystem { URI oldURI = oldPath.toUri(); URI newURI; try { - newURI = new URI(newScheme, oldURI.getUserInfo(), oldURI.getHost(), oldURI.getPort(), oldURI.getPath(), - oldURI.getQuery(), oldURI.getFragment()); + newURI = new URI(newScheme, + oldURI.getAuthority(), + oldURI.getPath(), + oldURI.getQuery(), + oldURI.getFragment()); return new CachingPath(newURI); } catch (URISyntaxException e) { // TODO - Better Exception handling diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestStorageSchemes.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestStorageSchemes.java index 85f3ce65ec..9b173254ac 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestStorageSchemes.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestStorageSchemes.java @@ -18,8 +18,10 @@ package org.apache.hudi.common.fs; +import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -54,4 +56,19 @@ public class TestStorageSchemes { StorageSchemes.isAppendSupported("s2"); }, "Should throw exception for unsupported schemes"); } + + @Test + public void testConversionToNewSchema() { + Path s3TablePath1 = new Path("s3://test.1234/table1"); + assertEquals(s3TablePath1, HoodieWrapperFileSystem.convertPathWithScheme(s3TablePath1, "s3")); + + Path s3TablePath2 = new Path("s3://1234.test/table1"); + assertEquals(s3TablePath2, HoodieWrapperFileSystem.convertPathWithScheme(s3TablePath2, "s3")); + + Path s3TablePath3 = new Path("s3://test1234/table1"); + assertEquals(s3TablePath3, HoodieWrapperFileSystem.convertPathWithScheme(s3TablePath3, "s3")); + + Path hdfsTablePath = new Path("hdfs://sandbox.foo.com:8020/test.1234/table1"); + System.out.println(HoodieWrapperFileSystem.convertPathWithScheme(hdfsTablePath, "hdfs")); + } }