This is an automated email from the ASF dual-hosted git repository. sankarh pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3.1 by this push: new ce5e13d HIVE-25600: Compaction job creates redundant base/delta folder within base/delta folder (Nikhil Gupta, reviewed by Sankar Hariappan) ce5e13d is described below commit ce5e13da3554df8538d46e10dce87b4ef04c3119 Author: guptanikhil007 <gupta.nikhil0...@gmail.com> AuthorDate: Wed Oct 27 13:36:10 2021 +0530 HIVE-25600: Compaction job creates redundant base/delta folder within base/delta folder (Nikhil Gupta, reviewed by Sankar Hariappan) Signed-off-by: Sankar Hariappan <sank...@apache.org> Closes (#2705) --- .../org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java index 95870ad..474f6c5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java @@ -1184,6 +1184,16 @@ public class CompactorMR { Path tmpPath = fileStatus.getPath(); //newPath is the base/delta dir Path newPath = new Path(finalLocation, tmpPath.getName()); + /* rename(A, B) has "interesting" behavior if A and B are directories. If B doesn't exist, + * it does the expected operation and everything that was in A is now in B. If B exists, + * it will make A a child of B. + * This issue can happen if the previous MR job succeeded but HMS was unable to persist compaction result. + * We will delete the directory B if it exists to avoid the above issue + */ + if (fs.exists(newPath)) { + LOG.info(String.format("Final path %s already exists. Deleting the path to avoid redundant base creation", newPath.toString())); + fs.delete(newPath, true); + } /* Create the markers in the tmp location and rename everything in the end to prevent race condition between * marker creation and split read. */ AcidUtils.OrcAcidVersion.writeVersionFile(tmpPath, fs); @@ -1192,6 +1202,7 @@ public class CompactorMR { } fs.delete(tmpLocation, true); } + private void createCompactorMarker(JobConf conf, Path finalLocation, FileSystem fs) throws IOException { if(conf.getBoolean(IS_MAJOR, false)) {