Repository: hive Updated Branches: refs/heads/branch-2.3 9194cae42 -> 066e67fa0
HIVE-16542 make merge that targets acid 2.0 table fail-fast (Eugene Koifman, reviwed by Wei Zheng) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/066e67fa Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/066e67fa Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/066e67fa Branch: refs/heads/branch-2.3 Commit: 066e67fa06ac4dcdf7c8392deab596f5a4df6b71 Parents: 9194cae Author: Eugene Koifman <ekoif...@hortonworks.com> Authored: Fri Apr 28 09:04:38 2017 -0700 Committer: Eugene Koifman <ekoif...@hortonworks.com> Committed: Fri Apr 28 09:04:38 2017 -0700 ---------------------------------------------------------------------- .../ql/parse/UpdateDeleteSemanticAnalyzer.java | 14 +++++++ .../apache/hadoop/hive/ql/TestTxnCommands.java | 1 - .../apache/hadoop/hive/ql/TestTxnCommands2.java | 3 +- .../ql/TestTxnCommands2WithSplitUpdate.java | 41 ++++++++++++++++++++ 4 files changed, 56 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/066e67fa/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java index 64f1bdd..0541a40 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java @@ -32,15 +32,18 @@ import org.antlr.runtime.TokenRewriteStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.TransactionalValidationListener; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.hooks.Entity; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; +import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveUtils; @@ -662,9 +665,11 @@ public class UpdateDeleteSemanticAnalyzer extends SemanticAnalyzer { */ String extraPredicate = null; int numWhenMatchedUpdateClauses = 0, numWhenMatchedDeleteClauses = 0; + int numInsertClauses = 0; for(ASTNode whenClause : whenClauses) { switch (getWhenClauseOperation(whenClause).getType()) { case HiveParser.TOK_INSERT: + numInsertClauses++; handleInsert(whenClause, rewrittenQueryStr, target, onClause, targetTable, targetName, onClauseAsText); break; case HiveParser.TOK_UPDATE: @@ -691,6 +696,15 @@ public class UpdateDeleteSemanticAnalyzer extends SemanticAnalyzer { if(numWhenMatchedUpdateClauses > 1) { throw new SemanticException(ErrorMsg.MERGE_TOO_MANY_UPDATE, ctx.getCmd()); } + assert numInsertClauses < 2; + if(numInsertClauses == 1 && numWhenMatchedUpdateClauses == 1) { + if(AcidUtils.getAcidOperationalProperties(targetTable).isSplitUpdate()) { + throw new IllegalStateException("Tables with " + + hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES + "=" + + TransactionalValidationListener.DEFAULT_TRANSACTIONAL_PROPERTY + " currently do not " + + "support MERGE with both Insert and Update clauses."); + } + } } if(numWhenMatchedDeleteClauses + numWhenMatchedUpdateClauses == 2 && extraPredicate == null) { throw new SemanticException(ErrorMsg.MERGE_PREDIACTE_REQUIRED, ctx.getCmd()); http://git-wip-us.apache.org/repos/asf/hive/blob/066e67fa/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java index 6dae7ba..c8bc119 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java @@ -103,7 +103,6 @@ public class TestTxnCommands { hiveConf = new HiveConf(this.getClass()); hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, ""); - hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR); hiveConf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict"); hiveConf http://git-wip-us.apache.org/repos/asf/hive/blob/066e67fa/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java index 6718ae9..e2db5b7 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java @@ -113,7 +113,6 @@ public class TestTxnCommands2 { hiveConf = new HiveConf(this.getClass()); hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, ""); - hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR); hiveConf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict"); hiveConf.setVar(HiveConf.ConfVars.HIVEINPUTFORMAT, HiveInputFormat.class.getName()); @@ -1676,7 +1675,7 @@ public class TestTxnCommands2 { return 0; } } - private String makeValuesClause(int[][] rows) { + String makeValuesClause(int[][] rows) { assert rows.length > 0; StringBuilder sb = new StringBuilder("values"); for(int[] row : rows) { http://git-wip-us.apache.org/repos/asf/hive/blob/066e67fa/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java index 67e917c..e516f18 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java @@ -540,4 +540,45 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2 { resultCount = 2; Assert.assertEquals(resultCount, Integer.parseInt(rs.get(0))); } + @Ignore("HIVE-14947") + @Test + @Override + public void testDynamicPartitionsMerge() throws Exception {} + @Ignore("HIVE-14947") + @Test + @Override + public void testDynamicPartitionsMerge2() throws Exception {} + @Ignore("HIVE-14947") + @Test + @Override + public void testMerge() throws Exception {} + + /** + * todo: remove this test once HIVE-14947 is done (parent class has a better version) + */ + @Test + @Override + public void testMerge2() throws Exception { + int[][] baseValsOdd = {{5,5},{11,11}}; + int[][] baseValsEven = {{2,2},{4,44}}; + runStatementOnDriver("insert into " + Table.NONACIDPART2 + " PARTITION(p2='odd') " + makeValuesClause(baseValsOdd)); + runStatementOnDriver("insert into " + Table.NONACIDPART2 + " PARTITION(p2='even') " + makeValuesClause(baseValsEven)); + int[][] vals = {{2,1},{4,3},{5,6},{7,8}}; + runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(vals)); + List<String> r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); + Assert.assertEquals(stringifyValues(vals), r); + String query = "merge into " + Table.ACIDTBL + + " using " + Table.NONACIDPART2 + " source ON " + Table.ACIDTBL + ".a = source.a2 " + + "WHEN MATCHED THEN UPDATE set b = source.b2 "; + r = runStatementOnDriver(query); + + r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); + int[][] rExpected = {{2,2},{4,44},{5,5},{7,8}}; + Assert.assertEquals(stringifyValues(rExpected), r); + + } + @Ignore("HIVE-14947") + @Test + @Override + public void testMergeWithPredicate() throws Exception {} }
