HIVE-11194 - Exchange partition on external tables should fail with error message when target folder already exists (Aihua Xu, reviewd by Chao Sun)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8121b9ab Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8121b9ab Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8121b9ab Branch: refs/heads/beeline-cli Commit: 8121b9ab644cbe477df477827dd82a9859a7791b Parents: 65e9fcf Author: Chao Sun <sunc...@apache.org> Authored: Mon Jul 13 09:36:22 2015 -0700 Committer: Chao Sun <sunc...@apache.org> Committed: Mon Jul 13 09:36:22 2015 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/common/FileUtils.java | 9 +++- .../hadoop/hive/metastore/HiveMetaStore.java | 2 +- .../queries/clientnegative/exchange_partition.q | 19 +++++++ .../clientnegative/exchange_partition.q.out | 54 ++++++++++++++++++++ 4 files changed, 82 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/8121b9ab/common/src/java/org/apache/hadoop/hive/common/FileUtils.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java index c2c54bc..7e4f386 100644 --- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java @@ -30,7 +30,6 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.DefaultFileAccess; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; @@ -635,6 +634,14 @@ public final class FileUtils { Path destPath, boolean inheritPerms, Configuration conf) throws IOException { LOG.info("Renaming " + sourcePath + " to " + destPath); + + // If destPath directory exists, rename call will move the sourcePath + // into destPath without failing. So check it before renaming. + if (fs.exists(destPath)) { + throw new IOException("Cannot rename the source path. The destination " + + "path already exists."); + } + if (!inheritPerms) { //just rename the directory return fs.rename(sourcePath, destPath); http://git-wip-us.apache.org/repos/asf/hive/blob/8121b9ab/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 4c9cd79..920e762 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -2581,7 +2581,7 @@ public class HiveMetaStore extends ThriftHiveMetastore { pathCreated = wh.renameDir(sourcePath, destPath); success = ms.commitTransaction(); } finally { - if (!success) { + if (!success || !pathCreated) { ms.rollbackTransaction(); if (pathCreated) { wh.renameDir(destPath, sourcePath); http://git-wip-us.apache.org/repos/asf/hive/blob/8121b9ab/ql/src/test/queries/clientnegative/exchange_partition.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/exchange_partition.q b/ql/src/test/queries/clientnegative/exchange_partition.q new file mode 100644 index 0000000..7dc4f57 --- /dev/null +++ b/ql/src/test/queries/clientnegative/exchange_partition.q @@ -0,0 +1,19 @@ +dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/ex_table1; +dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/ex_table1/part=part1; +CREATE EXTERNAL TABLE ex_table1 ( key INT, value STRING) + PARTITIONED BY (part STRING) + STORED AS textfile + LOCATION 'file:${system:test.tmp.dir}/ex_table1'; + +dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/ex_table2; + +CREATE EXTERNAL TABLE ex_table2 ( key INT, value STRING) + PARTITIONED BY (part STRING) + STORED AS textfile + LOCATION 'file:${system:test.tmp.dir}/ex_table2'; + +INSERT OVERWRITE TABLE ex_table2 PARTITION (part='part1') +SELECT key, value FROM src WHERE key < 10; +SHOW PARTITIONS ex_table2; + +ALTER TABLE ex_table1 EXCHANGE PARTITION (part='part1') WITH TABLE ex_table2; http://git-wip-us.apache.org/repos/asf/hive/blob/8121b9ab/ql/src/test/results/clientnegative/exchange_partition.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/exchange_partition.q.out b/ql/src/test/results/clientnegative/exchange_partition.q.out new file mode 100644 index 0000000..b81fb99 --- /dev/null +++ b/ql/src/test/results/clientnegative/exchange_partition.q.out @@ -0,0 +1,54 @@ +PREHOOK: query: CREATE EXTERNAL TABLE ex_table1 ( key INT, value STRING) + PARTITIONED BY (part STRING) + STORED AS textfile +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@ex_table1 +POSTHOOK: query: CREATE EXTERNAL TABLE ex_table1 ( key INT, value STRING) + PARTITIONED BY (part STRING) + STORED AS textfile +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ex_table1 +PREHOOK: query: CREATE EXTERNAL TABLE ex_table2 ( key INT, value STRING) + PARTITIONED BY (part STRING) + STORED AS textfile +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@ex_table2 +POSTHOOK: query: CREATE EXTERNAL TABLE ex_table2 ( key INT, value STRING) + PARTITIONED BY (part STRING) + STORED AS textfile +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ex_table2 +PREHOOK: query: INSERT OVERWRITE TABLE ex_table2 PARTITION (part='part1') +SELECT key, value FROM src WHERE key < 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ex_table2@part=part1 +POSTHOOK: query: INSERT OVERWRITE TABLE ex_table2 PARTITION (part='part1') +SELECT key, value FROM src WHERE key < 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ex_table2@part=part1 +POSTHOOK: Lineage: ex_table2 PARTITION(part=part1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ex_table2 PARTITION(part=part1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SHOW PARTITIONS ex_table2 +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@ex_table2 +POSTHOOK: query: SHOW PARTITIONS ex_table2 +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@ex_table2 +part=part1 +PREHOOK: query: ALTER TABLE ex_table1 EXCHANGE PARTITION (part='part1') WITH TABLE ex_table2 +PREHOOK: type: null +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Got exception: java.io.IOException Cannot rename the source path. The destination path already exists.)