Repository: tajo Updated Branches: refs/heads/branch-0.8.0 6824aab67 -> f6b086c74
TAJO-563: INSERT OVERWRITE should not remove data before query success. (hyunsik) Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/f6b086c7 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/f6b086c7 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/f6b086c7 Branch: refs/heads/branch-0.8.0 Commit: f6b086c74422ae669169e35976921b7fb2eeef13 Parents: 6824aab Author: Hyunsik Choi <[email protected]> Authored: Thu Apr 10 16:32:23 2014 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Thu Apr 10 16:33:13 2014 +0900 ---------------------------------------------------------------------- CHANGES.txt | 3 ++ .../java/org/apache/tajo/TajoConstants.java | 1 + .../apache/tajo/master/querymaster/Query.java | 30 ++++++++++++++++++-- .../master/querymaster/QueryMasterTask.java | 12 +------- 4 files changed, 33 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/f6b086c7/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index f2ff384..f3b41be 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -312,6 +312,9 @@ Release 0.8.0 - unreleased BUG FIXES + TAJO-563: INSERT OVERWRITE should not remove data before query success. + (hyunsik) + TAJO-738: NPE occur when failed in QueryMaster's GlobalPlanner.build(). (hyoungjunkim via hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/f6b086c7/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java b/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java index 61eecd1..028ca81 100644 --- a/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java +++ b/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java @@ -26,6 +26,7 @@ public class TajoConstants { public static final String WAREHOUSE_DIR_NAME = "warehouse"; public static final String SYSTEM_RESOURCE_DIR_NAME = "resource"; public static final String RESULT_DIR_NAME="RESULT"; + public static final String INSERT_OVERWIRTE_OLD_TABLE_NAME ="OLD_TABLE"; public static final String DEFAULT_TABLESPACE_NAME = "default"; public static final String DEFAULT_DATABASE_NAME = "default"; http://git-wip-us.apache.org/repos/asf/tajo/blob/f6b086c7/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java index 3a4df7b..a8f5b31 100644 --- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java @@ -396,8 +396,34 @@ public class Query implements EventHandler<QueryEvent> { finalOutputDir = queryContext.getOutputPath(); try { FileSystem fs = stagingResultDir.getFileSystem(query.systemConf); - fs.rename(stagingResultDir, finalOutputDir); - LOG.info("Moved from the staging dir to the output directory '" + finalOutputDir); + + if (queryContext.isOutputOverwrite()) { // INSERT OVERWRITE INTO + + // it moves the original table into the temporary location. + // Then it moves the new result table into the original table location. + // Upon failed, it recovers the original table if possible. + boolean movedToOldTable = false; + boolean committed = false; + Path oldTableDir = new Path(queryContext.getStagingDir(), TajoConstants.INSERT_OVERWIRTE_OLD_TABLE_NAME); + try { + if (fs.exists(finalOutputDir)) { + fs.rename(finalOutputDir, oldTableDir); + movedToOldTable = fs.exists(oldTableDir); + } else { // if the parent does not exist, make its parent directory. + fs.mkdirs(finalOutputDir.getParent()); + } + fs.rename(stagingResultDir, finalOutputDir); + committed = fs.exists(finalOutputDir); + } catch (IOException ioe) { + // recover the old table + if (movedToOldTable && !committed) { + fs.rename(oldTableDir, finalOutputDir); + } + } + } else { + fs.rename(stagingResultDir, finalOutputDir); + LOG.info("Moved from the staging dir to the output directory '" + finalOutputDir); + } } catch (IOException e) { e.printStackTrace(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/f6b086c7/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java index 271eaf9..8b3403f 100644 --- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java @@ -407,17 +407,7 @@ public class QueryMasterTask extends CompositeService { ///////////////////////////////////////////////// if (queryContext.hasOutputPath()) { outputDir = queryContext.getOutputPath(); - if (queryContext.isOutputOverwrite()) { - if (defaultFS.exists(outputDir.getParent())) { - if (defaultFS.exists(outputDir)) { - defaultFS.delete(outputDir, true); - LOG.info("The output directory '" + outputDir + "' is cleaned."); - } - } else { - defaultFS.mkdirs(outputDir.getParent()); - LOG.info("The output directory's parent '" + outputDir.getParent() + "' is created."); - } - } else { + if (!queryContext.isOutputOverwrite()) { if (defaultFS.exists(outputDir)) { throw new IOException("The output directory '" + outputDir + " already exists."); }
