subject:"hive git commit\: HIVE\-12439 \: CompactionTxnHandler.markCleaned\(\) and TxnHandler.openTxns\(\) misc improvements \(Wei Zheng, reviewed by Eugene Koifman\)"

hive git commit: HIVE-12439 : CompactionTxnHandler.markCleaned() and TxnHandler.openTxns() misc improvements (Wei Zheng, reviewed by Eugene Koifman)

2016-03-21 Thread weiz

Repository: hive
Updated Branches:
  refs/heads/branch-1 505c5585c -> 0aaddb7d7


HIVE-12439 : CompactionTxnHandler.markCleaned() and TxnHandler.openTxns() misc 
improvements (Wei Zheng, reviewed by Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0aaddb7d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0aaddb7d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0aaddb7d

Branch: refs/heads/branch-1
Commit: 0aaddb7d753a2936c973d9ab99e6edb2554f94ae
Parents: 505c558
Author: Wei Zheng 
Authored: Mon Mar 21 14:50:12 2016 -0700
Committer: Wei Zheng 
Committed: Mon Mar 21 14:50:12 2016 -0700

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |   7 +
 .../metastore/txn/CompactionTxnHandler.java | 120 +
 .../hadoop/hive/metastore/txn/TxnDbUtil.java|   4 +-
 .../hadoop/hive/metastore/txn/TxnHandler.java   | 245 ++-
 .../hive/metastore/txn/TestTxnHandler.java  |  83 ++-
 5 files changed, 333 insertions(+), 126 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/0aaddb7d/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 4a575b3..b78bea2 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -573,6 +573,13 @@ public class HiveConf extends Configuration {
 "select query has incorrect syntax or something similar inside a 
transaction, the\n" +
 "entire transaction will fail and fall-back to DataNucleus will not be 
possible. You\n" +
 "should disable the usage of direct SQL inside transactions if that 
happens in your case."),
+METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH("hive.direct.sql.max.query.length", 
100, "The maximum\n" +
+" size of a query string (in KB)."),
+
METASTORE_DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE("hive.direct.sql.max.elements.in.clause",
 1000,
+"The maximum number of values in a IN clause. Once exceeded, it will 
be broken into\n" +
+" multiple OR separated IN clauses."),
+
METASTORE_DIRECT_SQL_MAX_ELEMENTS_VALUES_CLAUSE("hive.direct.sql.max.elements.values.clause",
+1000, "The maximum number of values in a VALUES clause for INSERT 
statement."),
 
METASTORE_ORM_RETRIEVE_MAPNULLS_AS_EMPTY_STRINGS("hive.metastore.orm.retrieveMapNullsAsEmptyStrings",false,
 "Thrift does not support nulls in maps, so any nulls present in maps 
retrieved from ORM must " +
 "either be pruned or converted to empty strings. Some backing dbs such 
as Oracle persist empty strings " +

http://git-wip-us.apache.org/repos/asf/hive/blob/0aaddb7d/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
--
diff --git 
a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
 
b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
index 4d736b9..28e06ed 100644
--- 
a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
+++ 
b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
@@ -364,36 +364,38 @@ public class CompactionTxnHandler extends TxnHandler {
 rs = stmt.executeQuery(s);
 List txnids = new ArrayList<>();
 while (rs.next()) txnids.add(rs.getLong(1));
+// Remove entries from txn_components, as there may be aborted txn 
components
 if (txnids.size() > 0) {
+  List queries = new ArrayList();
+
+  // Prepare prefix and suffix
+  StringBuilder prefix = new StringBuilder();
+  StringBuilder suffix = new StringBuilder();
+
+  prefix.append("delete from TXN_COMPONENTS where ");
 
-  // Remove entries from txn_components, as there may be aborted txn 
components
-  StringBuilder buf = new StringBuilder();
-  //todo: add a safeguard to make sure IN clause is not too large; 
break up by txn id
-  buf.append("delete from TXN_COMPONENTS where tc_txnid in (");
-  boolean first = true;
-  for (long id : txnids) {
-if (first) first = false;
-else buf.append(", ");
-buf.append(id);
-  }
   //because 1 txn may include different partitions/tables even in auto 
commit mode
-  buf.append(") and tc_database = '");
-  buf.append(info.dbname);
-  buf.append("' and tc_table = '");
-  buf.append(info.tableName);
-  buf.append("'");
+  suffix.append(" and tc_database =

hive git commit: HIVE-12439 : CompactionTxnHandler.markCleaned() and TxnHandler.openTxns() misc improvements (Wei Zheng, reviewed by Eugene Koifman)

2016-03-21 Thread weiz

Repository: hive
Updated Branches:
  refs/heads/master a6155b75e -> db8fb8a42


HIVE-12439 : CompactionTxnHandler.markCleaned() and TxnHandler.openTxns() misc 
improvements (Wei Zheng, reviewed by Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/db8fb8a4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/db8fb8a4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/db8fb8a4

Branch: refs/heads/master
Commit: db8fb8a42a690eaa937d1a0163eaf505c3c48a07
Parents: a6155b7
Author: Wei Zheng 
Authored: Mon Mar 21 11:38:38 2016 -0700
Committer: Wei Zheng 
Committed: Mon Mar 21 11:38:38 2016 -0700

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |   7 +
 .../metastore/txn/CompactionTxnHandler.java | 120 ---
 .../hadoop/hive/metastore/txn/TxnDbUtil.java|   4 +-
 .../hadoop/hive/metastore/txn/TxnHandler.java   | 151 +++
 .../hadoop/hive/metastore/txn/TxnUtils.java |  95 
 .../hadoop/hive/metastore/txn/TestTxnUtils.java | 135 +
 6 files changed, 390 insertions(+), 122 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/db8fb8a4/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 98c6372..0f8d67f 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -737,6 +737,13 @@ public class HiveConf extends Configuration {
 "select query has incorrect syntax or something similar inside a 
transaction, the\n" +
 "entire transaction will fail and fall-back to DataNucleus will not be 
possible. You\n" +
 "should disable the usage of direct SQL inside transactions if that 
happens in your case."),
+METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH("hive.direct.sql.max.query.length", 
100, "The maximum\n" +
+" size of a query string (in KB)."),
+
METASTORE_DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE("hive.direct.sql.max.elements.in.clause",
 1000,
+"The maximum number of values in a IN clause. Once exceeded, it will 
be broken into\n" +
+" multiple OR separated IN clauses."),
+
METASTORE_DIRECT_SQL_MAX_ELEMENTS_VALUES_CLAUSE("hive.direct.sql.max.elements.values.clause",
+1000, "The maximum number of values in a VALUES clause for INSERT 
statement."),
 
METASTORE_ORM_RETRIEVE_MAPNULLS_AS_EMPTY_STRINGS("hive.metastore.orm.retrieveMapNullsAsEmptyStrings",false,
 "Thrift does not support nulls in maps, so any nulls present in maps 
retrieved from ORM must " +
 "either be pruned or converted to empty strings. Some backing dbs such 
as Oracle persist empty strings " +

http://git-wip-us.apache.org/repos/asf/hive/blob/db8fb8a4/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
--
diff --git 
a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
 
b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
index da2b395..15c01da 100644
--- 
a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
+++ 
b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
@@ -369,36 +369,38 @@ class CompactionTxnHandler extends TxnHandler {
 rs = stmt.executeQuery(s);
 List txnids = new ArrayList<>();
 while (rs.next()) txnids.add(rs.getLong(1));
+// Remove entries from txn_components, as there may be aborted txn 
components
 if (txnids.size() > 0) {
+  List queries = new ArrayList();
+
+  // Prepare prefix and suffix
+  StringBuilder prefix = new StringBuilder();
+  StringBuilder suffix = new StringBuilder();
+
+  prefix.append("delete from TXN_COMPONENTS where ");
 
-  // Remove entries from txn_components, as there may be aborted txn 
components
-  StringBuilder buf = new StringBuilder();
-  //todo: add a safeguard to make sure IN clause is not too large; 
break up by txn id
-  buf.append("delete from TXN_COMPONENTS where tc_txnid in (");
-  boolean first = true;
-  for (long id : txnids) {
-if (first) first = false;
-else buf.append(", ");
-buf.append(id);
-  }
   //because 1 txn may include different partitions/tables even in auto 
commit mode
-  buf.append(") and tc_database = '");
-  buf.append(info.dbname);
-  buf.append("' and tc_table = '");
-  buf.append(info.tableName);

hive git commit: HIVE-12439 : CompactionTxnHandler.markCleaned() and TxnHandler.openTxns() misc improvements (Wei Zheng, reviewed by Eugene Koifman)

hive git commit: HIVE-12439 : CompactionTxnHandler.markCleaned() and TxnHandler.openTxns() misc improvements (Wei Zheng, reviewed by Eugene Koifman)

2 matches

Site Navigation

Mail list logo

Footer information