Repository: hive Updated Branches: refs/heads/master a10bd8cae -> db21c3ff6
HIVE-16943 : MoveTask should separate src FileSystem from dest FileSystem (Fei Hui via Ashutosh Chauhan) Signed-off-by: Ashutosh Chauhan <hashut...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/db21c3ff Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/db21c3ff Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/db21c3ff Branch: refs/heads/master Commit: db21c3ff6c09ca920b9406efe95694d110219483 Parents: a10bd8c Author: Fei Hui <feihui dot ustc at gmail dot com> Authored: Thu Jun 22 19:22:00 2017 -0800 Committer: Ashutosh Chauhan <hashut...@apache.org> Committed: Mon Jun 26 07:50:11 2017 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/exec/MoveTask.java | 27 +++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/db21c3ff/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java index f329b51..53bc9fe 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java @@ -94,7 +94,7 @@ public class MoveTask extends Task<MoveWork> implements Serializable { FileSystem fs = sourcePath.getFileSystem(conf); if (isDfsDir) { - moveFileInDfs (sourcePath, targetPath, fs); + moveFileInDfs (sourcePath, targetPath, conf); } else { // This is a local file FileSystem dstFs = FileSystem.getLocal(conf); @@ -106,21 +106,36 @@ public class MoveTask extends Task<MoveWork> implements Serializable { } } - private void moveFileInDfs (Path sourcePath, Path targetPath, FileSystem fs) + private void moveFileInDfs (Path sourcePath, Path targetPath, HiveConf conf) throws HiveException, IOException { + + final FileSystem srcFs, tgtFs; + try { + tgtFs = targetPath.getFileSystem(conf); + } catch (IOException e) { + LOG.error("Failed to get dest fs", e); + throw new HiveException(e.getMessage(), e); + } + try { + srcFs = sourcePath.getFileSystem(conf); + } catch (IOException e) { + LOG.error("Failed to get src fs", e); + throw new HiveException(e.getMessage(), e); + } + // if source exists, rename. Otherwise, create a empty directory - if (fs.exists(sourcePath)) { + if (srcFs.exists(sourcePath)) { Path deletePath = null; // If it multiple level of folder are there fs.rename is failing so first // create the targetpath.getParent() if it not exist if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_INSERT_INTO_MULTILEVEL_DIRS)) { - deletePath = createTargetPath(targetPath, fs); + deletePath = createTargetPath(targetPath, tgtFs); } Hive.clearDestForSubDirSrc(conf, targetPath, sourcePath, false); if (!Hive.moveFile(conf, sourcePath, targetPath, true, false)) { try { if (deletePath != null) { - fs.delete(deletePath, true); + tgtFs.delete(deletePath, true); } } catch (IOException e) { LOG.info("Unable to delete the path created for facilitating rename" @@ -129,7 +144,7 @@ public class MoveTask extends Task<MoveWork> implements Serializable { throw new HiveException("Unable to rename: " + sourcePath + " to: " + targetPath); } - } else if (!fs.mkdirs(targetPath)) { + } else if (!tgtFs.mkdirs(targetPath)) { throw new HiveException("Unable to make directory: " + targetPath); } }
