hive git commit: HIVE-13327: SessionID added to HS2 threadname does not trim spaces (Prasanth Jayachandran reviewed by Gopal V)

2016-03-21 Thread prasanthj 
Repository: hive
Updated Branches:
  refs/heads/master db8fb8a42 -> d3d5d33c3


HIVE-13327: SessionID added to HS2 threadname does not trim spaces (Prasanth 
Jayachandran reviewed by Gopal V)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d3d5d33c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d3d5d33c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d3d5d33c

Branch: refs/heads/master
Commit: d3d5d33c3e0fd483b56a7eef7b248fedf319024e
Parents: db8fb8a
Author: Prasanth Jayachandran 
Authored: Mon Mar 21 23:49:00 2016 -0500
Committer: Prasanth Jayachandran 
Committed: Mon Mar 21 23:49:00 2016 -0500

--
 ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/d3d5d33c/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java 
b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 78bbb1f..23b8a96 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -417,7 +417,7 @@ public class SessionState {
 if (currThreadName.contains(logPrefix)) {
   final String[] names = currThreadName.split(logPrefix);
   LOG.info("Resetting thread name to {}", names[names.length - 1]);
-  Thread.currentThread().setName(names[names.length - 1]);
+  Thread.currentThread().setName(names[names.length - 1].trim());
 }
   }

hive git commit: HIVE-12439 : CompactionTxnHandler.markCleaned() and TxnHandler.openTxns() misc improvements (Wei Zheng, reviewed by Eugene Koifman)

2016-03-21 Thread weiz 
Repository: hive
Updated Branches:
  refs/heads/branch-1 505c5585c -> 0aaddb7d7


HIVE-12439 : CompactionTxnHandler.markCleaned() and TxnHandler.openTxns() misc 
improvements (Wei Zheng, reviewed by Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0aaddb7d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0aaddb7d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0aaddb7d

Branch: refs/heads/branch-1
Commit: 0aaddb7d753a2936c973d9ab99e6edb2554f94ae
Parents: 505c558
Author: Wei Zheng 
Authored: Mon Mar 21 14:50:12 2016 -0700
Committer: Wei Zheng 
Committed: Mon Mar 21 14:50:12 2016 -0700

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |   7 +
 .../metastore/txn/CompactionTxnHandler.java | 120 +
 .../hadoop/hive/metastore/txn/TxnDbUtil.java|   4 +-
 .../hadoop/hive/metastore/txn/TxnHandler.java   | 245 ++-
 .../hive/metastore/txn/TestTxnHandler.java  |  83 ++-
 5 files changed, 333 insertions(+), 126 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/0aaddb7d/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 4a575b3..b78bea2 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -573,6 +573,13 @@ public class HiveConf extends Configuration {
 "select query has incorrect syntax or something similar inside a 
transaction, the\n" +
 "entire transaction will fail and fall-back to DataNucleus will not be 
possible. You\n" +
 "should disable the usage of direct SQL inside transactions if that 
happens in your case."),
+METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH("hive.direct.sql.max.query.length", 
100, "The maximum\n" +
+" size of a query string (in KB)."),
+
METASTORE_DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE("hive.direct.sql.max.elements.in.clause",
 1000,
+"The maximum number of values in a IN clause. Once exceeded, it will 
be broken into\n" +
+" multiple OR separated IN clauses."),
+
METASTORE_DIRECT_SQL_MAX_ELEMENTS_VALUES_CLAUSE("hive.direct.sql.max.elements.values.clause",
+1000, "The maximum number of values in a VALUES clause for INSERT 
statement."),
 
METASTORE_ORM_RETRIEVE_MAPNULLS_AS_EMPTY_STRINGS("hive.metastore.orm.retrieveMapNullsAsEmptyStrings",false,
 "Thrift does not support nulls in maps, so any nulls present in maps 
retrieved from ORM must " +
 "either be pruned or converted to empty strings. Some backing dbs such 
as Oracle persist empty strings " +

http://git-wip-us.apache.org/repos/asf/hive/blob/0aaddb7d/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
--
diff --git 
a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
 
b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
index 4d736b9..28e06ed 100644
--- 
a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
+++ 
b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
@@ -364,36 +364,38 @@ public class CompactionTxnHandler extends TxnHandler {
 rs = stmt.executeQuery(s);
 List txnids = new ArrayList<>();
 while (rs.next()) txnids.add(rs.getLong(1));
+// Remove entries from txn_components, as there may be aborted txn 
components
 if (txnids.size() > 0) {
+  List queries = new ArrayList();
+
+  // Prepare prefix and suffix
+  StringBuilder prefix = new StringBuilder();
+  StringBuilder suffix = new StringBuilder();
+
+  prefix.append("delete from TXN_COMPONENTS where ");
 
-  // Remove entries from txn_components, as there may be aborted txn 
components
-  StringBuilder buf = new StringBuilder();
-  //todo: add a safeguard to make sure IN clause is not too large; 
break up by txn id
-  buf.append("delete from TXN_COMPONENTS where tc_txnid in (");
-  boolean first = true;
-  for (long id : txnids) {
-if (first) first = false;
-else buf.append(", ");
-buf.append(id);
-  }
   //because 1 txn may include different partitions/tables even in auto 
commit mode
-  buf.append(") and tc_database = '");
-  buf.append(info.dbname);
-  buf.append("' and tc_table = '");
-  buf.append(info.tableName);
-  buf.append("'");
+

hive git commit: HIVE-12439 : CompactionTxnHandler.markCleaned() and TxnHandler.openTxns() misc improvements (Wei Zheng, reviewed by Eugene Koifman)

2016-03-21 Thread weiz 
Repository: hive
Updated Branches:
  refs/heads/master a6155b75e -> db8fb8a42


HIVE-12439 : CompactionTxnHandler.markCleaned() and TxnHandler.openTxns() misc 
improvements (Wei Zheng, reviewed by Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/db8fb8a4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/db8fb8a4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/db8fb8a4

Branch: refs/heads/master
Commit: db8fb8a42a690eaa937d1a0163eaf505c3c48a07
Parents: a6155b7
Author: Wei Zheng 
Authored: Mon Mar 21 11:38:38 2016 -0700
Committer: Wei Zheng 
Committed: Mon Mar 21 11:38:38 2016 -0700

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |   7 +
 .../metastore/txn/CompactionTxnHandler.java | 120 ---
 .../hadoop/hive/metastore/txn/TxnDbUtil.java|   4 +-
 .../hadoop/hive/metastore/txn/TxnHandler.java   | 151 +++
 .../hadoop/hive/metastore/txn/TxnUtils.java |  95 
 .../hadoop/hive/metastore/txn/TestTxnUtils.java | 135 +
 6 files changed, 390 insertions(+), 122 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/db8fb8a4/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 98c6372..0f8d67f 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -737,6 +737,13 @@ public class HiveConf extends Configuration {
 "select query has incorrect syntax or something similar inside a 
transaction, the\n" +
 "entire transaction will fail and fall-back to DataNucleus will not be 
possible. You\n" +
 "should disable the usage of direct SQL inside transactions if that 
happens in your case."),
+METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH("hive.direct.sql.max.query.length", 
100, "The maximum\n" +
+" size of a query string (in KB)."),
+
METASTORE_DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE("hive.direct.sql.max.elements.in.clause",
 1000,
+"The maximum number of values in a IN clause. Once exceeded, it will 
be broken into\n" +
+" multiple OR separated IN clauses."),
+
METASTORE_DIRECT_SQL_MAX_ELEMENTS_VALUES_CLAUSE("hive.direct.sql.max.elements.values.clause",
+1000, "The maximum number of values in a VALUES clause for INSERT 
statement."),
 
METASTORE_ORM_RETRIEVE_MAPNULLS_AS_EMPTY_STRINGS("hive.metastore.orm.retrieveMapNullsAsEmptyStrings",false,
 "Thrift does not support nulls in maps, so any nulls present in maps 
retrieved from ORM must " +
 "either be pruned or converted to empty strings. Some backing dbs such 
as Oracle persist empty strings " +

http://git-wip-us.apache.org/repos/asf/hive/blob/db8fb8a4/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
--
diff --git 
a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
 
b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
index da2b395..15c01da 100644
--- 
a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
+++ 
b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
@@ -369,36 +369,38 @@ class CompactionTxnHandler extends TxnHandler {
 rs = stmt.executeQuery(s);
 List txnids = new ArrayList<>();
 while (rs.next()) txnids.add(rs.getLong(1));
+// Remove entries from txn_components, as there may be aborted txn 
components
 if (txnids.size() > 0) {
+  List queries = new ArrayList();
+
+  // Prepare prefix and suffix
+  StringBuilder prefix = new StringBuilder();
+  StringBuilder suffix = new StringBuilder();
+
+  prefix.append("delete from TXN_COMPONENTS where ");
 
-  // Remove entries from txn_components, as there may be aborted txn 
components
-  StringBuilder buf = new StringBuilder();
-  //todo: add a safeguard to make sure IN clause is not too large; 
break up by txn id
-  buf.append("delete from TXN_COMPONENTS where tc_txnid in (");
-  boolean first = true;
-  for (long id : txnids) {
-if (first) first = false;
-else buf.append(", ");
-buf.append(id);
-  }
   //because 1 txn may include different partitions/tables even in auto 
commit mode
-  buf.append(") and tc_database = '");
-  buf.append(info.dbname);
-  buf.append("' and tc_table = '");
-

hive git commit: HIVE-13311: MetaDataFormatUtils throws NPE when HiveDecimal.create is null (Reuben Kuhnert, reviewed by Sergio Pena)

2016-03-21 Thread spena 
Repository: hive
Updated Branches:
  refs/heads/master 15220e8b5 -> 48aefe450


HIVE-13311: MetaDataFormatUtils throws NPE when HiveDecimal.create is null 
(Reuben Kuhnert, reviewed by Sergio Pena)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/48aefe45
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/48aefe45
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/48aefe45

Branch: refs/heads/master
Commit: 48aefe4506ea5f6934a129c09ca9e13269adf895
Parents: 15220e8
Author: Sergio Pena 
Authored: Mon Mar 21 12:31:59 2016 -0500
Committer: Sergio Pena 
Committed: Mon Mar 21 12:32:59 2016 -0500

--
 .../hive/ql/metadata/formatting/MetaDataFormatUtils.java | 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/48aefe45/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
index a78700d..e76fb91 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
@@ -150,7 +150,13 @@ public final class MetaDataFormatUtils {
 if (val == null) {
   return "";
 }
-return HiveDecimal.create(new BigInteger(val.getUnscaled()), 
val.getScale()).toString();
+
+HiveDecimal result = HiveDecimal.create(new BigInteger(val.getUnscaled()), 
val.getScale());
+if (result != null) {
+  return result.toString();
+} else {
+  return "";
+}
   }
 
   private static String 
convertToString(org.apache.hadoop.hive.metastore.api.Date val) {

hive git commit: HIVE-13294: AvroSerde leaks the connection in a case when reading schema from a url (Chaoyu Tang, reviewed by Aihua Xu)

2016-03-21 Thread ctang 
Repository: hive
Updated Branches:
  refs/heads/branch-2.0 896016126 -> 2552a93ef


HIVE-13294: AvroSerde leaks the connection in a case when reading schema from a 
url (Chaoyu Tang, reviewed by Aihua Xu)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2552a93e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2552a93e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2552a93e

Branch: refs/heads/branch-2.0
Commit: 2552a93ef5264fadc7c80e85e2ed879ed7f63234
Parents: 8960161
Author: ctang 
Authored: Mon Mar 21 12:21:58 2016 -0400
Committer: ctang 
Committed: Mon Mar 21 12:23:41 2016 -0400

--
 .../hadoop/hive/serde2/avro/AvroSerdeUtils.java | 20 +++-
 1 file changed, 19 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/2552a93e/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java
--
diff --git 
a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java 
b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java
index 08ae6ef..08ee62b 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java
@@ -134,7 +134,7 @@ public class AvroSerdeUtils {
   Schema s = getSchemaFromFS(schemaString, conf);
   if (s == null) {
 //in case schema is not a file system
-return AvroSerdeUtils.getSchemaFor(new URL(schemaString).openStream());
+return AvroSerdeUtils.getSchemaFor(new URL(schemaString));
   }
   return s;
 } catch (IOException ioe) {
@@ -260,4 +260,22 @@ public class AvroSerdeUtils {
 }
 return schema;
   }
+
+  public static Schema getSchemaFor(URL url) {
+InputStream in = null;
+try {
+  in = url.openStream();
+  return getSchemaFor(in);
+} catch (Exception e) {
+  throw new RuntimeException("Failed to parse Avro schema", e);
+} finally {
+  if (in != null) {
+try {
+  in.close();
+} catch (IOException e) {
+  // Ignore
+}
+  }
+}
+  }
 }