This is an automated email from the ASF dual-hosted git repository. ayushsaxena pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 756a8fc HIVE-25960: Fix S3a recursive listing logic. (#3031). (Ayush Saxena reviewed by Laszlo Bodor) 756a8fc is described below commit 756a8fce9f75e139fd7c5cdc45ff0eb629ad9504 Author: Ayush Saxena <ayushsax...@apache.org> AuthorDate: Tue Feb 22 19:44:02 2022 +0530 HIVE-25960: Fix S3a recursive listing logic. (#3031). (Ayush Saxena reviewed by Laszlo Bodor) --- .../java/org/apache/hadoop/hive/common/FileUtils.java | 16 +++++++++++++++- .../org/apache/hadoop/hive/common/TestFileUtils.java | 17 +++++++++++++++++ .../apache/hadoop/hive/metastore/utils/FileUtils.java | 16 +++++++++++++++- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java index b617623..e92b700 100644 --- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java @@ -361,13 +361,27 @@ public final class FileUtils { RemoteIterator<LocatedFileStatus> remoteIterator = fs.listFiles(base.getPath(), true); while (remoteIterator.hasNext()) { LocatedFileStatus each = remoteIterator.next(); - Path relativePath = new Path(each.getPath().toString().replace(base.toString(), "")); + Path relativePath = makeRelative(base.getPath(), each.getPath()); if (org.apache.hadoop.hive.metastore.utils.FileUtils.RemoteIteratorWithFilter.HIDDEN_FILES_FULL_PATH_FILTER.accept(relativePath)) { results.add(each); } } } + /** + * Returns a relative path wrt the parent path. + * @param parentPath the parent path. + * @param childPath the child path. + * @return childPath relative to parent path. + */ + public static Path makeRelative(Path parentPath, Path childPath) { + String parentString = + parentPath.toString().endsWith(Path.SEPARATOR) ? parentPath.toString() : parentPath.toString() + Path.SEPARATOR; + String childString = + childPath.toString().endsWith(Path.SEPARATOR) ? childPath.toString() : childPath.toString() + Path.SEPARATOR; + return new Path(childString.replaceFirst(parentString, "")); + } + public static boolean isS3a(FileSystem fs) { try { return "s3a".equalsIgnoreCase(fs.getScheme()); diff --git a/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java b/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java index 15e74db..4f4d604 100644 --- a/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java +++ b/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java @@ -264,4 +264,21 @@ public class TestFileUtils { equalsIgnoreCase("Distcp is called with doAsUser and delete source set as true")); } } + + @Test + public void testMakeRelative() { + Path parentPath = new Path("/user/hive/database"); + Path childPath = new Path(parentPath, "table/dir/subdir"); + Path relativePath = FileUtils.makeRelative(parentPath, childPath); + assertEquals("table/dir/subdir", relativePath.toString()); + + // try with parent as Root. + relativePath = FileUtils.makeRelative(new Path(Path.SEPARATOR), childPath); + assertEquals("user/hive/database/table/dir/subdir", relativePath.toString()); + + // try with non child path, it should return the child path as is. + childPath = new Path("/user/hive/database1/table/dir/subdir"); + relativePath = FileUtils.makeRelative(parentPath, childPath); + assertEquals(childPath.toString(), relativePath.toString()); + } } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java index dad71b7..b8ab28b 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java @@ -404,13 +404,27 @@ public class FileUtils { RemoteIterator<LocatedFileStatus> remoteIterator = fs.listFiles(base, true); while (remoteIterator.hasNext()) { LocatedFileStatus each = remoteIterator.next(); - Path relativePath = new Path(each.getPath().toString().replace(base.toString(), "")); + Path relativePath = makeRelative(base, each.getPath()); if (RemoteIteratorWithFilter.HIDDEN_FILES_FULL_PATH_FILTER.accept(relativePath)) { results.add(each); } } } + /** + * Returns a relative path wrt the parent path. + * @param parentPath the parent path. + * @param childPath the child path. + * @return childPath relative to parent path. + */ + public static Path makeRelative(Path parentPath, Path childPath) { + String parentString = + parentPath.toString().endsWith(Path.SEPARATOR) ? parentPath.toString() : parentPath.toString() + Path.SEPARATOR; + String childString = + childPath.toString().endsWith(Path.SEPARATOR) ? childPath.toString() : childPath.toString() + Path.SEPARATOR; + return new Path(childString.replaceFirst(parentString, "")); + } + public static boolean isS3a(FileSystem fs) { try { return "s3a".equalsIgnoreCase(fs.getScheme());