[hive] branch master updated: HIVE-21177: ACID: When there are no delete deltas skip finding min max keys (Eugene Koifman, reviewed by Prasanth Jayachandran)

[ekoifman]

This is an automated email from the ASF dual-hosted git repository.

ekoifman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
 new 07b76f4  HIVE-21177: ACID: When there are no delete deltas skip 
finding min max keys (Eugene Koifman, reviewed by Prasanth Jayachandran)
07b76f4 is described below

commit 07b76f413cb174413f0530a6aae5ae442a301b46
Author: Eugene Koifman 
AuthorDate: Thu Feb 7 09:49:19 2019 -0800

HIVE-21177: ACID: When there are no delete deltas skip finding min max keys 
(Eugene Koifman, reviewed by Prasanth Jayachandran)
---
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java| 202 ++---
 .../hadoop/hive/ql/io/orc/OrcInputFormat.java  |   2 +-
 .../hadoop/hive/ql/io/orc/OrcRawRecordMerger.java  |   5 +-
 .../hadoop/hive/ql/txn/compactor/Cleaner.java  |  12 +-
 .../hadoop/hive/ql/txn/compactor/CompactorMR.java  |  19 +-
 .../apache/hadoop/hive/ql/io/TestAcidUtils.java|  70 +++
 .../hive/ql/io/orc/TestInputOutputFormat.java  |  74 +++-
 .../hive/ql/io/orc/TestOrcRawRecordMerger.java |   6 +-
 8 files changed, 211 insertions(+), 179 deletions(-)

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index 9b51847..3961baa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -83,6 +83,7 @@ import org.slf4j.LoggerFactory;
 
 import com.google.common.annotations.VisibleForTesting;
 
+import javax.annotation.concurrent.Immutable;
 import java.nio.charset.Charset;
 
 /**
@@ -435,16 +436,16 @@ public class AcidUtils {
   }
 
   public static final class DirectoryImpl implements Directory {
-private final List abortedDirectories;
+private final List abortedDirectories;
 private final boolean isBaseInRawFormat;
 private final List original;
-private final List obsolete;
+private final List obsolete;
 private final List deltas;
 private final Path base;
 
-public DirectoryImpl(List abortedDirectories,
+public DirectoryImpl(List abortedDirectories,
 boolean isBaseInRawFormat, List original,
-List obsolete, List deltas, Path base) {
+List obsolete, List deltas, Path base) {
   this.abortedDirectories = abortedDirectories == null ?
   Collections.emptyList() : abortedDirectories;
   this.isBaseInRawFormat = isBaseInRawFormat;
@@ -475,12 +476,12 @@ public class AcidUtils {
 }
 
 @Override
-public List getObsolete() {
+public List getObsolete() {
   return obsolete;
 }
 
 @Override
-public List getAbortedDirectories() {
+public List getAbortedDirectories() {
   return abortedDirectories;
 }
   }
@@ -740,7 +741,7 @@ public class AcidUtils {
 /**
  * Get the list of base and delta directories that are valid and not
  * obsolete.  Not {@code null}.  List must be sorted in a specific way.
- * See {@link 
org.apache.hadoop.hive.ql.io.AcidUtils.ParsedDelta#compareTo(org.apache.hadoop.hive.ql.io.AcidUtils.ParsedDelta)}
+ * See {@link 
org.apache.hadoop.hive.ql.io.AcidUtils.ParsedDeltaLight#compareTo(org.apache.hadoop.hive.ql.io.AcidUtils.ParsedDeltaLight)}
  * for details.
  * @return the minimal list of current directories
  */
@@ -752,13 +753,13 @@ public class AcidUtils {
  * list of original files, bases, and deltas that have been replaced by
  * more up to date ones.  Not {@code null}.
  */
-List getObsolete();
+List getObsolete();
 
 /**
  * Get the list of directories that has nothing but aborted transactions.
  * @return the list of aborted directories
  */
-List getAbortedDirectories();
+List getAbortedDirectories();
   }
   /**
* Since version 3 but prior to version 4, format of a base is "base_X" 
where X is a writeId.
@@ -804,18 +805,48 @@ public class AcidUtils {
   Long.parseLong(filename.substring(idxOfv + 
VISIBILITY_PREFIX.length())), path);
 }
   }
+
   /**
-   * Immutable
+   * In addition to {@link ParsedDeltaLight} this knows if the data is in raw 
format, i.e. doesn't
+   * have acid metadata columns embedded in the files.  To determine this in 
some cases
+   * requires looking at the footer of the data file which can be expensive so 
if this info is
+   * not needed {@link ParsedDeltaLight} should be used.
*/
-  public static final class ParsedDelta implements Comparable {
-private final long minWriteId;
-private final long maxWriteId;
-private final FileStatus path;
+  @Immutable
+  public static final class ParsedDelta extends ParsedDeltaLight {
+private final boolean isRawFormat;
+/**
+ * for pre 1.3.x delta files
+ */
+private ParsedDelta(long min, long max, Path path, boolean isDeleteDelta,
+boolean i

[hive] branch master updated: HIVE-21159 Modify Merge statement logic to perform Update split early (Eugene Koifman, reviewed by Vaibhav Gumashta)

[ekoifman]

This is an automated email from the ASF dual-hosted git repository.

ekoifman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
 new 4a4b9ca  HIVE-21159 Modify Merge statement logic to perform Update 
split early (Eugene Koifman, reviewed by Vaibhav Gumashta)
4a4b9ca is described below

commit 4a4b9ca87ed41e6390ff376c9d6529e7db6d60ef
Author: Eugene Koifman 
AuthorDate: Mon Feb 4 12:29:05 2019 -0800

HIVE-21159 Modify Merge statement logic to perform Update split early 
(Eugene Koifman, reviewed by Vaibhav Gumashta)
---
 .../java/org/apache/hadoop/hive/conf/HiveConf.java |  5 ++
 ql/src/java/org/apache/hadoop/hive/ql/Context.java | 10 ++-
 .../hive/ql/parse/MergeSemanticAnalyzer.java   | 91 --
 .../hive/ql/parse/RewriteSemanticAnalyzer.java |  2 +-
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 12 ++-
 .../org/apache/hadoop/hive/ql/TestTxnCommands.java | 26 +--
 .../apache/hadoop/hive/ql/TestTxnCommands2.java|  1 +
 .../hadoop/hive/ql/TxnCommandsBaseForTests.java|  1 +
 .../hadoop/hive/ql/lockmgr/TestDbTxnManager2.java  | 71 +++--
 .../hadoop/hive/metastore/txn/TxnHandler.java  | 14 +++-
 10 files changed, 170 insertions(+), 63 deletions(-)

diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 93ecb8f..414070e 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2754,6 +2754,11 @@ public class HiveConf extends Configuration {
 MERGE_CARDINALITY_VIOLATION_CHECK("hive.merge.cardinality.check", true,
   "Set to true to ensure that each SQL Merge statement ensures that for 
each row in the target\n" +
 "table there is at most 1 matching row in the source table per SQL 
Specification."),
+MERGE_SPLIT_UPDATE("hive.merge.split.update", false,
+"If true, SQL Merge statement will handle WHEN MATCHED UPDATE by 
splitting it into 2\n" +
+"branches of a multi-insert, representing delete of existing row 
and an insert of\n" +
+"the new version of the row.  Updating bucketing and partitioning 
columns should\n" +
+"only be permitted if this is true."),
 OPTIMIZE_ACID_META_COLUMNS("hive.optimize.acid.meta.columns", true,
 "If true, don't decode Acid metadata columns from storage unless" +
 " they are needed."),
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java 
b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
index 18089d5..b11d5a1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
@@ -200,7 +200,13 @@ public class Context {
 return getTokenRewriteStream().toString(n.getTokenStartIndex(), 
n.getTokenStopIndex() + 1).trim();
   }
   /**
-   * The suffix is always relative to a given ASTNode
+   * The suffix is always relative to a given ASTNode.
+   * We need this so that FileSinkOperatorS corresponding to different 
branches of a multi-insert
+   * statement which represents a SQL Merge statement get marked correctly with
+   * {@link org.apache.hadoop.hive.ql.io.AcidUtils.Operation}.  See usages
+   * of {@link #getDestNamePrefix(ASTNode, QB)} and
+   * {@link org.apache.hadoop.hive.ql.parse.SemanticAnalyzer#updating(String)} 
and
+   * {@link org.apache.hadoop.hive.ql.parse.SemanticAnalyzer#deleting(String)}.
*/
   public DestClausePrefix getDestNamePrefix(ASTNode curNode, QB queryBlock) {
 assert curNode != null : "must supply curNode";
@@ -255,7 +261,7 @@ public class Context {
   case DELETE:
 return DestClausePrefix.DELETE;
   case MERGE:
-  /* This is the structrue expected here
+  /* This is the structure expected here
 HiveParser.TOK_QUERY;
   HiveParser.TOK_FROM
   HiveParser.TOK_INSERT;
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java
index 44f7b43..db6d551 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java
@@ -64,7 +64,7 @@ public class MergeSemanticAnalyzer extends 
RewriteSemanticAnalyzer {
 
   private static final String INDENT = "  ";
 
-  private IdentifierQuoter quotedIdenfierHelper;
+  private IdentifierQuoter quotedIdentifierHelper;
 
   /**
* This allows us to take an arbitrary ASTNode and turn it back into SQL 
that produced it.
@@ -112,7 +112,7 @@ public class MergeSemanticAnalyzer extends 
RewriteSemanticAnalyzer {
* needing to understand

hive git commit: HIVE-20960 Make MM compactor run in a transaction and remove CompactorMR.createCompactorMarker() (Eugene Koifman, reviewed by Vaibhav Gumashta)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master f713140ba -> cb9d5ccd8


HIVE-20960 Make MM compactor run in a transaction and remove 
CompactorMR.createCompactorMarker() (Eugene Koifman, reviewed by Vaibhav 
Gumashta)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cb9d5ccd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cb9d5ccd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cb9d5ccd

Branch: refs/heads/master
Commit: cb9d5ccd87997985b4623369247aead32bd91390
Parents: f713140
Author: Eugene Koifman 
Authored: Fri Jan 11 12:54:28 2019 -0800
Committer: Eugene Koifman 
Committed: Fri Jan 11 12:54:28 2019 -0800

--
 .../hive/ql/txn/compactor/TestCompactor.java| 76 ---
 .../java/org/apache/hadoop/hive/ql/Driver.java  | 13 +++-
 .../org/apache/hadoop/hive/ql/DriverUtils.java  | 18 -
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java | 78 ++--
 .../hive/ql/stats/StatsUpdaterThread.java   |  2 +-
 .../hive/ql/txn/compactor/CompactorMR.java  | 59 +++
 .../hadoop/hive/ql/txn/compactor/Worker.java|  7 ++
 .../hadoop/hive/ql/TestTxnConcatenate.java  | 28 +++
 .../hive/ql/stats/TestStatsUpdaterThread.java   |  2 +-
 9 files changed, 160 insertions(+), 123 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/cb9d5ccd/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
index 2b22a62..dc7b287 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
@@ -55,7 +55,6 @@ import 
org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
 import org.apache.hadoop.hive.metastore.api.CompactionRequest;
 import org.apache.hadoop.hive.metastore.api.CompactionType;
 import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
 import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
@@ -97,12 +96,10 @@ import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
-import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-@RunWith(Parameterized.class)
 public class TestCompactor {
   private static final AtomicInteger salt = new AtomicInteger(new 
Random().nextInt());
   private static final Logger LOG = 
LoggerFactory.getLogger(TestCompactor.class);
@@ -117,12 +114,6 @@ public class TestCompactor {
 return Arrays.asList(new Object[][]{{true}, {false}});
   }
 
-  private boolean newStreamingAPI;
-
-  public TestCompactor(boolean newStreamingAPI) {
-this.newStreamingAPI = newStreamingAPI;
-  }
-
   @Rule
   public TemporaryFolder stagingFolder = new TemporaryFolder();
 
@@ -366,6 +357,13 @@ public class TestCompactor {
*/
   @Test
   public void testStatsAfterCompactionPartTbl() throws Exception {
+testStatsAfterCompactionPartTbl(false);
+  }
+  @Test
+  public void testStatsAfterCompactionPartTblNew() throws Exception {
+testStatsAfterCompactionPartTbl(true);
+  }
+  private void testStatsAfterCompactionPartTbl(boolean newStreamingAPI) throws 
Exception {
 //as of (8/27/2014) Hive 0.14, ACID/Orc requires HiveInputFormat
 String tblName = "compaction_test";
 String tblNameStg = tblName + "_stg";
@@ -710,6 +708,13 @@ public class TestCompactor {
 
   @Test
   public void minorCompactAfterAbort() throws Exception {
+minorCompactAfterAbort(false);
+  }
+  @Test
+  public void minorCompactAfterAbortNew() throws Exception {
+minorCompactAfterAbort(true);
+  }
+  private void minorCompactAfterAbort(boolean newStreamingAPI) throws 
Exception {
 String dbName = "default";
 String tblName = "cws";
 String columnNamesProperty = "a,b";
@@ -719,7 +724,7 @@ public class TestCompactor {
   " CLUSTERED BY(a) INTO 1 BUCKETS" + //currently ACID requires table to 
be bucketed
   " STORED AS ORC  TBLPROPERTIES ('transactional'='true')", driver);
 
-processStreamingAPI(dbName, tblName);
+processStreamingAPI(dbName, tblName, newStreamingAPI);
 // Now, compact
 TxnStore txnHandler = TxnUtils.getTxnStore(conf);
 txnHandler.compact(new CompactionRequest(dbName, tblName, 
CompactionType.MINOR));
@@ -750,6 +755,13 

hive git commit: HIVE-21036 extend OpenTxnRequest with transaction type (Igor Kryvenko via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master dcc895016 -> f713140ba


HIVE-21036 extend OpenTxnRequest with transaction type (Igor Kryvenko via 
Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f713140b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f713140b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f713140b

Branch: refs/heads/master
Commit: f713140ba60d50017d2b97b1f5c5aea7217512dd
Parents: dcc8950
Author: Igor Kryvenko 
Authored: Thu Jan 10 15:50:41 2019 -0800
Committer: Eugene Koifman 
Committed: Thu Jan 10 15:50:41 2019 -0800

--
 .../hadoop/hive/ql/txn/compactor/Worker.java|   3 +-
 .../hive/metastore/api/OpenTxnRequest.java  | 129 ++-
 .../hadoop/hive/metastore/api/TxnType.java  |  51 
 .../src/gen/thrift/gen-php/metastore/Types.php  |  36 ++
 .../gen/thrift/gen-py/hive_metastore/ttypes.py  |  35 -
 .../gen/thrift/gen-rb/hive_metastore_types.rb   |  16 ++-
 .../hive/metastore/HiveMetaStoreClient.java |  17 ++-
 .../hadoop/hive/metastore/IMetaStoreClient.java |  10 ++
 .../src/main/thrift/hive_metastore.thrift   |   8 ++
 .../hadoop/hive/metastore/txn/TxnHandler.java   |  13 --
 .../HiveMetaStoreClientPreCatalog.java  |  15 ++-
 .../hive/metastore/TestHiveMetaStoreTxns.java   |   9 ++
 12 files changed, 315 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java 
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
index b645870..e3fab69 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.TxnType;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.thrift.TException;
@@ -155,7 +156,7 @@ public class Worker extends RemoteCompactorThread 
implements MetaStoreThread {
 if (ci.runAs == null) {
   ci.runAs = findUserToRunAs(sd.getLocation(), t);
 }
-long compactorTxnId = msc.openTxns(ci.runAs, 1).getTxn_ids().get(0);
+long compactorTxnId = msc.openTxn(ci.runAs, TxnType.COMPACTION);
 
 heartbeater = new CompactionHeartbeater(compactorTxnId, fullTableName, 
conf);
 heartbeater.start();

http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/OpenTxnRequest.java
--
diff --git 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/OpenTxnRequest.java
 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/OpenTxnRequest.java
index 19b2c01..b5efffd 100644
--- 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/OpenTxnRequest.java
+++ 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/OpenTxnRequest.java
@@ -44,6 +44,7 @@ import org.slf4j.LoggerFactory;
   private static final org.apache.thrift.protocol.TField AGENT_INFO_FIELD_DESC 
= new org.apache.thrift.protocol.TField("agentInfo", 
org.apache.thrift.protocol.TType.STRING, (short)4);
   private static final org.apache.thrift.protocol.TField 
REPL_POLICY_FIELD_DESC = new org.apache.thrift.protocol.TField("replPolicy", 
org.apache.thrift.protocol.TType.STRING, (short)5);
   private static final org.apache.thrift.protocol.TField 
REPL_SRC_TXN_IDS_FIELD_DESC = new 
org.apache.thrift.protocol.TField("replSrcTxnIds", 
org.apache.thrift.protocol.TType.LIST, (short)6);
+  private static final org.apache.thrift.protocol.TField TXN_TYPE_FIELD_DESC = 
new org.apache.thrift.protocol.TField("txn_type", 
org.apache.thrift.protocol.TType.I32, (short)7);
 
   private static final Map, SchemeFactory> schemes = 
new HashMap, SchemeFactory>();
   static {
@@ -57,6 +58,7 @@ import org.slf4j.LoggerFactory;
   private String agentInfo; // optional
   private String replPolicy; // optional
   private List replSrcTxnIds; // optional
+  private TxnType txn_type; // optional
 
   /** The set of fields this struct contains, along with 

[2/2] hive git commit: HIVE-20919 Break up UpdateDeleteSemanticAnalyzer (Miklos Gergely via Eugene Koifman)

[ekoifman]

HIVE-20919 Break up UpdateDeleteSemanticAnalyzer (Miklos Gergely via Eugene 
Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dcc89501
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dcc89501
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dcc89501

Branch: refs/heads/master
Commit: dcc8950164db00adac982d6764bbd8fe31e6897d
Parents: 4d03e31
Author: Miklos Gergely 
Authored: Wed Jan 9 15:15:16 2019 -0800
Committer: Eugene Koifman 
Committed: Wed Jan 9 15:15:16 2019 -0800

--
 .../ql/parse/AcidExportSemanticAnalyzer.java|  299 
 .../hive/ql/parse/MergeSemanticAnalyzer.java|  760 ++
 .../hive/ql/parse/RewriteSemanticAnalyzer.java  |  451 ++
 .../hive/ql/parse/SemanticAnalyzerFactory.java  |8 +-
 .../ql/parse/UpdateDeleteSemanticAnalyzer.java  | 1427 +-
 .../hadoop/hive/ql/plan/AlterTableDesc.java |6 +-
 .../apache/hadoop/hive/ql/plan/ExportWork.java  |2 +-
 7 files changed, 1563 insertions(+), 1390 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/dcc89501/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java
new file mode 100644
index 000..41e3754
--- /dev/null
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.antlr.runtime.tree.Tree;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.exec.DDLTask;
+import org.apache.hadoop.hive.ql.exec.StatsTask;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.hooks.ReadEntity;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.HiveUtils;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
+import org.apache.hadoop.hive.ql.plan.CreateTableLikeDesc;
+import org.apache.hadoop.hive.ql.plan.DDLWork;
+import org.apache.hadoop.hive.ql.plan.DropTableDesc;
+import org.apache.hadoop.hive.ql.plan.ExportWork;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+/**
+ * A subclass of the {@link org.apache.hadoop.hive.ql.parse.SemanticAnalyzer} 
that just handles
+ * acid export statements. It works by rewriting the acid export into insert 
statements into a temporary table,
+ * and then export it from there.
+ */
+public class AcidExportSemanticAnalyzer extends RewriteSemanticAnalyzer {
+  AcidExportSemanticAnalyzer(QueryState queryState) throws SemanticException {
+super(queryState);
+  }
+
+  protected void analyze(ASTNode tree) throws SemanticException {
+if (tree.getToken().getType() != HiveParser.TOK_EXPORT) {
+  throw new RuntimeException("Asked to parse token " + tree.getName() + " 
in " +
+  "AcidExportSemanticAnalyzer");
+}
+analyzeAcidExport(tree);
+  }
+
+  /**
+   * Exporting an Acid table is more complicated than a flat table.  It may 

[1/2] hive git commit: HIVE-20919 Break up UpdateDeleteSemanticAnalyzer (Miklos Gergely via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 4d03e31d3 -> dcc895016


http://git-wip-us.apache.org/repos/asf/hive/blob/dcc89501/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
index 8651afd..179021e 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
@@ -17,603 +17,89 @@
  */
 package org.apache.hadoop.hive.ql.parse;
 
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.IdentityHashMap;
-import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.UUID;
 
-import org.antlr.runtime.TokenRewriteStream;
-import org.antlr.runtime.tree.Tree;
-import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.TableType;
-import org.apache.hadoop.hive.metastore.Warehouse;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 import org.apache.hadoop.hive.ql.Context;
-import org.apache.hadoop.hive.ql.DriverContext;
-import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.QueryState;
-import org.apache.hadoop.hive.ql.exec.DDLTask;
-import org.apache.hadoop.hive.ql.exec.StatsTask;
-import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.exec.TaskFactory;
-import org.apache.hadoop.hive.ql.hooks.Entity;
-import org.apache.hadoop.hive.ql.hooks.ReadEntity;
-import org.apache.hadoop.hive.ql.hooks.WriteEntity;
-import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.lib.Node;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.HiveUtils;
-import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
 import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
-import org.apache.hadoop.hive.ql.plan.CreateTableLikeDesc;
-import org.apache.hadoop.hive.ql.plan.DDLWork;
-import org.apache.hadoop.hive.ql.plan.DropTableDesc;
-import org.apache.hadoop.hive.ql.plan.ExportWork;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 
 /**
  * A subclass of the {@link org.apache.hadoop.hive.ql.parse.SemanticAnalyzer} 
that just handles
- * update, delete and merge statements.  It works by rewriting the updates and 
deletes into insert
+ * update and delete statements. It works by rewriting the updates and deletes 
into insert
  * statements (since they are actually inserts) and then doing some patch up 
to make them work as
  * updates and deletes instead.
  */
-public class UpdateDeleteSemanticAnalyzer extends SemanticAnalyzer {
-  private static final Logger LOG = 
LoggerFactory.getLogger(UpdateDeleteSemanticAnalyzer.class);
+public class UpdateDeleteSemanticAnalyzer extends RewriteSemanticAnalyzer {
 
-  private boolean useSuper = false;
+  private Context.Operation operation = Context.Operation.OTHER;
 
   UpdateDeleteSemanticAnalyzer(QueryState queryState) throws SemanticException 
{
 super(queryState);
   }
 
-  @Override
-  public void analyzeInternal(ASTNode tree) throws SemanticException {
-if (useSuper) {
-  super.analyzeInternal(tree);
-} else {
-  if (!getTxnMgr().supportsAcid()) {
-throw new 
SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TXNMGR.getMsg());
-  }
-  switch (tree.getToken().getType()) {
-case HiveParser.TOK_DELETE_FROM:
-  analyzeDelete(tree);
-  break;
-case HiveParser.TOK_UPDATE_TABLE:
-  analyzeUpdate(tree);
-  break;
-case HiveParser.TOK_MERGE:
-  analyzeMerge(tree);
-  break;
-case HiveParser.TOK_EXPORT:
-  analyzeAcidExport(tree);
-  break;
-default:
-  throw new RuntimeException("Asked to parse token " + tree.getName() 
+ " in " +
-  "UpdateDeleteSemanticAnalyzer");
-  }
-  cleanUpMetaColumnAccessControl();
-
-}
-  }
-  private boolean updating() {
-return currentOperation == Context.Operation.UPDATE;
-  }
-  private boolean deleting() {
-return currentOperation == 

[7/9] hive git commit: HIVE-20936 Allow the Worker thread in the metastore to run outside of it (Jaume Marhuenda via Eugene Koifman)

[ekoifman]

http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetPartitionsRequest.java
--
diff --git 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetPartitionsRequest.java
 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetPartitionsRequest.java
index d26cde2..7ec107e 100644
--- 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetPartitionsRequest.java
+++ 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetPartitionsRequest.java
@@ -960,13 +960,13 @@ import org.slf4j.LoggerFactory;
   case 6: // GROUP_NAMES
 if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
   {
-org.apache.thrift.protocol.TList _list1000 = 
iprot.readListBegin();
-struct.groupNames = new ArrayList(_list1000.size);
-String _elem1001;
-for (int _i1002 = 0; _i1002 < _list1000.size; ++_i1002)
+org.apache.thrift.protocol.TList _list1008 = 
iprot.readListBegin();
+struct.groupNames = new ArrayList(_list1008.size);
+String _elem1009;
+for (int _i1010 = 0; _i1010 < _list1008.size; ++_i1010)
 {
-  _elem1001 = iprot.readString();
-  struct.groupNames.add(_elem1001);
+  _elem1009 = iprot.readString();
+  struct.groupNames.add(_elem1009);
 }
 iprot.readListEnd();
   }
@@ -1040,9 +1040,9 @@ import org.slf4j.LoggerFactory;
   oprot.writeFieldBegin(GROUP_NAMES_FIELD_DESC);
   {
 oprot.writeListBegin(new 
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, 
struct.groupNames.size()));
-for (String _iter1003 : struct.groupNames)
+for (String _iter1011 : struct.groupNames)
 {
-  oprot.writeString(_iter1003);
+  oprot.writeString(_iter1011);
 }
 oprot.writeListEnd();
   }
@@ -1120,9 +1120,9 @@ import org.slf4j.LoggerFactory;
   if (struct.isSetGroupNames()) {
 {
   oprot.writeI32(struct.groupNames.size());
-  for (String _iter1004 : struct.groupNames)
+  for (String _iter1012 : struct.groupNames)
   {
-oprot.writeString(_iter1004);
+oprot.writeString(_iter1012);
   }
 }
   }
@@ -1160,13 +1160,13 @@ import org.slf4j.LoggerFactory;
   }
   if (incoming.get(5)) {
 {
-  org.apache.thrift.protocol.TList _list1005 = new 
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, 
iprot.readI32());
-  struct.groupNames = new ArrayList(_list1005.size);
-  String _elem1006;
-  for (int _i1007 = 0; _i1007 < _list1005.size; ++_i1007)
+  org.apache.thrift.protocol.TList _list1013 = new 
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, 
iprot.readI32());
+  struct.groupNames = new ArrayList(_list1013.size);
+  String _elem1014;
+  for (int _i1015 = 0; _i1015 < _list1013.size; ++_i1015)
   {
-_elem1006 = iprot.readString();
-struct.groupNames.add(_elem1006);
+_elem1014 = iprot.readString();
+struct.groupNames.add(_elem1014);
   }
 }
 struct.setGroupNamesIsSet(true);

http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetPartitionsResponse.java
--
diff --git 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetPartitionsResponse.java
 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetPartitionsResponse.java
index 3db9095..faac848 100644
--- 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetPartitionsResponse.java
+++ 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetPartitionsResponse.java
@@ -350,14 +350,14 @@ import org.slf4j.LoggerFactory;
   case 1: // PARTITION_SPEC
 if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
   {
-org.apache.thrift.protocol.TList _list992 = 
iprot.readListBegin();
-struct.partitionSpec = new 
ArrayList(_list992.size);
-PartitionSpec 

[6/9] hive git commit: HIVE-20936 Allow the Worker thread in the metastore to run outside of it (Jaume Marhuenda via Eugene Koifman)

[ekoifman]

http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
--
diff --git 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
index b6a0893..1bdbbbf 100644
--- 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
+++ 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
@@ -374,6 +374,20 @@ import org.slf4j.LoggerFactory;
 
 public void add_dynamic_partitions(AddDynamicPartitions rqst) throws 
NoSuchTxnException, TxnAbortedException, org.apache.thrift.TException;
 
+public OptionalCompactionInfoStruct find_next_compact(String workerId) 
throws MetaException, org.apache.thrift.TException;
+
+public void update_compactor_state(CompactionInfoStruct cr, long txn_id) 
throws org.apache.thrift.TException;
+
+public List find_columns_with_stats(CompactionInfoStruct cr) 
throws org.apache.thrift.TException;
+
+public void mark_cleaned(CompactionInfoStruct cr) throws MetaException, 
org.apache.thrift.TException;
+
+public void mark_compacted(CompactionInfoStruct cr) throws MetaException, 
org.apache.thrift.TException;
+
+public void mark_failed(CompactionInfoStruct cr) throws MetaException, 
org.apache.thrift.TException;
+
+public void set_hadoop_jobid(String jobId, long cq_id) throws 
org.apache.thrift.TException;
+
 public NotificationEventResponse 
get_next_notification(NotificationEventRequest rqst) throws 
org.apache.thrift.TException;
 
 public CurrentNotificationEventId get_current_notificationEventId() throws 
org.apache.thrift.TException;
@@ -808,6 +822,20 @@ import org.slf4j.LoggerFactory;
 
 public void add_dynamic_partitions(AddDynamicPartitions rqst, 
org.apache.thrift.async.AsyncMethodCallback resultHandler) throws 
org.apache.thrift.TException;
 
+public void find_next_compact(String workerId, 
org.apache.thrift.async.AsyncMethodCallback resultHandler) throws 
org.apache.thrift.TException;
+
+public void update_compactor_state(CompactionInfoStruct cr, long txn_id, 
org.apache.thrift.async.AsyncMethodCallback resultHandler) throws 
org.apache.thrift.TException;
+
+public void find_columns_with_stats(CompactionInfoStruct cr, 
org.apache.thrift.async.AsyncMethodCallback resultHandler) throws 
org.apache.thrift.TException;
+
+public void mark_cleaned(CompactionInfoStruct cr, 
org.apache.thrift.async.AsyncMethodCallback resultHandler) throws 
org.apache.thrift.TException;
+
+public void mark_compacted(CompactionInfoStruct cr, 
org.apache.thrift.async.AsyncMethodCallback resultHandler) throws 
org.apache.thrift.TException;
+
+public void mark_failed(CompactionInfoStruct cr, 
org.apache.thrift.async.AsyncMethodCallback resultHandler) throws 
org.apache.thrift.TException;
+
+public void set_hadoop_jobid(String jobId, long cq_id, 
org.apache.thrift.async.AsyncMethodCallback resultHandler) throws 
org.apache.thrift.TException;
+
 public void get_next_notification(NotificationEventRequest rqst, 
org.apache.thrift.async.AsyncMethodCallback resultHandler) throws 
org.apache.thrift.TException;
 
 public void 
get_current_notificationEventId(org.apache.thrift.async.AsyncMethodCallback 
resultHandler) throws org.apache.thrift.TException;
@@ -5749,6 +5777,166 @@ import org.slf4j.LoggerFactory;
   return;
 }
 
+public OptionalCompactionInfoStruct find_next_compact(String workerId) 
throws MetaException, org.apache.thrift.TException
+{
+  send_find_next_compact(workerId);
+  return recv_find_next_compact();
+}
+
+public void send_find_next_compact(String workerId) throws 
org.apache.thrift.TException
+{
+  find_next_compact_args args = new find_next_compact_args();
+  args.setWorkerId(workerId);
+  sendBase("find_next_compact", args);
+}
+
+public OptionalCompactionInfoStruct recv_find_next_compact() throws 
MetaException, org.apache.thrift.TException
+{
+  find_next_compact_result result = new find_next_compact_result();
+  receiveBase(result, "find_next_compact");
+  if (result.isSetSuccess()) {
+return result.success;
+  }
+  if (result.o1 != null) {
+throw result.o1;
+  }
+  throw new 
org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT,
 "find_next_compact failed: unknown result");
+}
+
+public void update_compactor_state(CompactionInfoStruct cr, long txn_id) 
throws org.apache.thrift.TException
+{
+  

[3/9] hive git commit: HIVE-20936 Allow the Worker thread in the metastore to run outside of it (Jaume Marhuenda via Eugene Koifman)

[ekoifman]

http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php
--
diff --git 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php
 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php
index 48396ea..2953a8f 100644
--- 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php
+++ 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php
@@ -21081,6 +21081,414 @@ class CompactionRequest {
 
 }
 
+class OptionalCompactionInfoStruct {
+  static $_TSPEC;
+
+  /**
+   * @var \metastore\CompactionInfoStruct
+   */
+  public $ci = null;
+
+  public function __construct($vals=null) {
+if (!isset(self::$_TSPEC)) {
+  self::$_TSPEC = array(
+1 => array(
+  'var' => 'ci',
+  'type' => TType::STRUCT,
+  'class' => '\metastore\CompactionInfoStruct',
+  ),
+);
+}
+if (is_array($vals)) {
+  if (isset($vals['ci'])) {
+$this->ci = $vals['ci'];
+  }
+}
+  }
+
+  public function getName() {
+return 'OptionalCompactionInfoStruct';
+  }
+
+  public function read($input)
+  {
+$xfer = 0;
+$fname = null;
+$ftype = 0;
+$fid = 0;
+$xfer += $input->readStructBegin($fname);
+while (true)
+{
+  $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+  if ($ftype == TType::STOP) {
+break;
+  }
+  switch ($fid)
+  {
+case 1:
+  if ($ftype == TType::STRUCT) {
+$this->ci = new \metastore\CompactionInfoStruct();
+$xfer += $this->ci->read($input);
+  } else {
+$xfer += $input->skip($ftype);
+  }
+  break;
+default:
+  $xfer += $input->skip($ftype);
+  break;
+  }
+  $xfer += $input->readFieldEnd();
+}
+$xfer += $input->readStructEnd();
+return $xfer;
+  }
+
+  public function write($output) {
+$xfer = 0;
+$xfer += $output->writeStructBegin('OptionalCompactionInfoStruct');
+if ($this->ci !== null) {
+  if (!is_object($this->ci)) {
+throw new TProtocolException('Bad type in structure.', 
TProtocolException::INVALID_DATA);
+  }
+  $xfer += $output->writeFieldBegin('ci', TType::STRUCT, 1);
+  $xfer += $this->ci->write($output);
+  $xfer += $output->writeFieldEnd();
+}
+$xfer += $output->writeFieldStop();
+$xfer += $output->writeStructEnd();
+return $xfer;
+  }
+
+}
+
+class CompactionInfoStruct {
+  static $_TSPEC;
+
+  /**
+   * @var int
+   */
+  public $id = null;
+  /**
+   * @var string
+   */
+  public $dbname = null;
+  /**
+   * @var string
+   */
+  public $tablename = null;
+  /**
+   * @var string
+   */
+  public $partitionname = null;
+  /**
+   * @var int
+   */
+  public $type = null;
+  /**
+   * @var string
+   */
+  public $runas = null;
+  /**
+   * @var string
+   */
+  public $properties = null;
+  /**
+   * @var bool
+   */
+  public $toomanyaborts = null;
+  /**
+   * @var string
+   */
+  public $state = null;
+  /**
+   * @var string
+   */
+  public $workerId = null;
+  /**
+   * @var int
+   */
+  public $start = null;
+  /**
+   * @var int
+   */
+  public $highestWriteId = null;
+
+  public function __construct($vals=null) {
+if (!isset(self::$_TSPEC)) {
+  self::$_TSPEC = array(
+1 => array(
+  'var' => 'id',
+  'type' => TType::I64,
+  ),
+2 => array(
+  'var' => 'dbname',
+  'type' => TType::STRING,
+  ),
+3 => array(
+  'var' => 'tablename',
+  'type' => TType::STRING,
+  ),
+4 => array(
+  'var' => 'partitionname',
+  'type' => TType::STRING,
+  ),
+5 => array(
+  'var' => 'type',
+  'type' => TType::I32,
+  ),
+6 => array(
+  'var' => 'runas',
+  'type' => TType::STRING,
+  ),
+7 => array(
+  'var' => 'properties',
+  'type' => TType::STRING,
+  ),
+8 => array(
+  'var' => 'toomanyaborts',
+  'type' => TType::BOOL,
+  ),
+9 => array(
+  'var' => 'state',
+  'type' => TType::STRING,
+  ),
+10 => array(
+  'var' => 'workerId',
+  'type' => TType::STRING,
+  ),
+11 => array(
+  'var' => 'start',
+  'type' => TType::I64,
+  ),
+12 => array(
+  'var' => 'highestWriteId',
+  'type' => TType::I64,
+  ),
+);
+}
+if (is_array($vals)) {
+  if (isset($vals['id'])) {
+$this->id = $vals['id'];
+  }
+  if (isset($vals['dbname'])) {
+$this->dbname = $vals['dbname'];
+  }
+  if (isset($vals['tablename'])) {
+

[2/9] hive git commit: HIVE-20936 Allow the Worker thread in the metastore to run outside of it (Jaume Marhuenda via Eugene Koifman)

[ekoifman]

http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py
--
diff --git 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py
 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py
index a896849..37db81f 100644
--- 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py
+++ 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py
@@ -1324,6 +1324,57 @@ class Iface(fb303.FacebookService.Iface):
 """
 pass
 
+  def find_next_compact(self, workerId):
+"""
+Parameters:
+ - workerId
+"""
+pass
+
+  def update_compactor_state(self, cr, txn_id):
+"""
+Parameters:
+ - cr
+ - txn_id
+"""
+pass
+
+  def find_columns_with_stats(self, cr):
+"""
+Parameters:
+ - cr
+"""
+pass
+
+  def mark_cleaned(self, cr):
+"""
+Parameters:
+ - cr
+"""
+pass
+
+  def mark_compacted(self, cr):
+"""
+Parameters:
+ - cr
+"""
+pass
+
+  def mark_failed(self, cr):
+"""
+Parameters:
+ - cr
+"""
+pass
+
+  def set_hadoop_jobid(self, jobId, cq_id):
+"""
+Parameters:
+ - jobId
+ - cq_id
+"""
+pass
+
   def get_next_notification(self, rqst):
 """
 Parameters:
@@ -7675,6 +7726,225 @@ class Client(fb303.FacebookService.Client, Iface):
   raise result.o2
 return
 
+  def find_next_compact(self, workerId):
+"""
+Parameters:
+ - workerId
+"""
+self.send_find_next_compact(workerId)
+return self.recv_find_next_compact()
+
+  def send_find_next_compact(self, workerId):
+self._oprot.writeMessageBegin('find_next_compact', TMessageType.CALL, 
self._seqid)
+args = find_next_compact_args()
+args.workerId = workerId
+args.write(self._oprot)
+self._oprot.writeMessageEnd()
+self._oprot.trans.flush()
+
+  def recv_find_next_compact(self):
+iprot = self._iprot
+(fname, mtype, rseqid) = iprot.readMessageBegin()
+if mtype == TMessageType.EXCEPTION:
+  x = TApplicationException()
+  x.read(iprot)
+  iprot.readMessageEnd()
+  raise x
+result = find_next_compact_result()
+result.read(iprot)
+iprot.readMessageEnd()
+if result.success is not None:
+  return result.success
+if result.o1 is not None:
+  raise result.o1
+raise TApplicationException(TApplicationException.MISSING_RESULT, 
"find_next_compact failed: unknown result")
+
+  def update_compactor_state(self, cr, txn_id):
+"""
+Parameters:
+ - cr
+ - txn_id
+"""
+self.send_update_compactor_state(cr, txn_id)
+self.recv_update_compactor_state()
+
+  def send_update_compactor_state(self, cr, txn_id):
+self._oprot.writeMessageBegin('update_compactor_state', TMessageType.CALL, 
self._seqid)
+args = update_compactor_state_args()
+args.cr = cr
+args.txn_id = txn_id
+args.write(self._oprot)
+self._oprot.writeMessageEnd()
+self._oprot.trans.flush()
+
+  def recv_update_compactor_state(self):
+iprot = self._iprot
+(fname, mtype, rseqid) = iprot.readMessageBegin()
+if mtype == TMessageType.EXCEPTION:
+  x = TApplicationException()
+  x.read(iprot)
+  iprot.readMessageEnd()
+  raise x
+result = update_compactor_state_result()
+result.read(iprot)
+iprot.readMessageEnd()
+return
+
+  def find_columns_with_stats(self, cr):
+"""
+Parameters:
+ - cr
+"""
+self.send_find_columns_with_stats(cr)
+return self.recv_find_columns_with_stats()
+
+  def send_find_columns_with_stats(self, cr):
+self._oprot.writeMessageBegin('find_columns_with_stats', 
TMessageType.CALL, self._seqid)
+args = find_columns_with_stats_args()
+args.cr = cr
+args.write(self._oprot)
+self._oprot.writeMessageEnd()
+self._oprot.trans.flush()
+
+  def recv_find_columns_with_stats(self):
+iprot = self._iprot
+(fname, mtype, rseqid) = iprot.readMessageBegin()
+if mtype == TMessageType.EXCEPTION:
+  x = TApplicationException()
+  x.read(iprot)
+  iprot.readMessageEnd()
+  raise x
+result = find_columns_with_stats_result()
+result.read(iprot)
+iprot.readMessageEnd()
+if result.success is not None:
+  return result.success
+raise TApplicationException(TApplicationException.MISSING_RESULT, 
"find_columns_with_stats failed: unknown result")
+
+  def mark_cleaned(self, cr):
+"""
+Parameters:
+ - cr
+"""
+self.send_mark_cleaned(cr)
+self.recv_mark_cleaned()
+
+  def send_mark_cleaned(self, cr):
+self._oprot.writeMessageBegin('mark_cleaned', TMessageType.CALL, 
self._seqid)
+args = mark_cleaned_args()
+args.cr = cr
+

[5/9] hive git commit: HIVE-20936 Allow the Worker thread in the metastore to run outside of it (Jaume Marhuenda via Eugene Koifman)

[ekoifman]

http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/WMFullResourcePlan.java
--
diff --git 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/WMFullResourcePlan.java
 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/WMFullResourcePlan.java
index 6d3ac0c..f92e23e 100644
--- 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/WMFullResourcePlan.java
+++ 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/WMFullResourcePlan.java
@@ -755,14 +755,14 @@ import org.slf4j.LoggerFactory;
   case 2: // POOLS
 if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
   {
-org.apache.thrift.protocol.TList _list880 = 
iprot.readListBegin();
-struct.pools = new ArrayList(_list880.size);
-WMPool _elem881;
-for (int _i882 = 0; _i882 < _list880.size; ++_i882)
+org.apache.thrift.protocol.TList _list888 = 
iprot.readListBegin();
+struct.pools = new ArrayList(_list888.size);
+WMPool _elem889;
+for (int _i890 = 0; _i890 < _list888.size; ++_i890)
 {
-  _elem881 = new WMPool();
-  _elem881.read(iprot);
-  struct.pools.add(_elem881);
+  _elem889 = new WMPool();
+  _elem889.read(iprot);
+  struct.pools.add(_elem889);
 }
 iprot.readListEnd();
   }
@@ -774,14 +774,14 @@ import org.slf4j.LoggerFactory;
   case 3: // MAPPINGS
 if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
   {
-org.apache.thrift.protocol.TList _list883 = 
iprot.readListBegin();
-struct.mappings = new ArrayList(_list883.size);
-WMMapping _elem884;
-for (int _i885 = 0; _i885 < _list883.size; ++_i885)
+org.apache.thrift.protocol.TList _list891 = 
iprot.readListBegin();
+struct.mappings = new ArrayList(_list891.size);
+WMMapping _elem892;
+for (int _i893 = 0; _i893 < _list891.size; ++_i893)
 {
-  _elem884 = new WMMapping();
-  _elem884.read(iprot);
-  struct.mappings.add(_elem884);
+  _elem892 = new WMMapping();
+  _elem892.read(iprot);
+  struct.mappings.add(_elem892);
 }
 iprot.readListEnd();
   }
@@ -793,14 +793,14 @@ import org.slf4j.LoggerFactory;
   case 4: // TRIGGERS
 if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
   {
-org.apache.thrift.protocol.TList _list886 = 
iprot.readListBegin();
-struct.triggers = new ArrayList(_list886.size);
-WMTrigger _elem887;
-for (int _i888 = 0; _i888 < _list886.size; ++_i888)
+org.apache.thrift.protocol.TList _list894 = 
iprot.readListBegin();
+struct.triggers = new ArrayList(_list894.size);
+WMTrigger _elem895;
+for (int _i896 = 0; _i896 < _list894.size; ++_i896)
 {
-  _elem887 = new WMTrigger();
-  _elem887.read(iprot);
-  struct.triggers.add(_elem887);
+  _elem895 = new WMTrigger();
+  _elem895.read(iprot);
+  struct.triggers.add(_elem895);
 }
 iprot.readListEnd();
   }
@@ -812,14 +812,14 @@ import org.slf4j.LoggerFactory;
   case 5: // POOL_TRIGGERS
 if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
   {
-org.apache.thrift.protocol.TList _list889 = 
iprot.readListBegin();
-struct.poolTriggers = new 
ArrayList(_list889.size);
-WMPoolTrigger _elem890;
-for (int _i891 = 0; _i891 < _list889.size; ++_i891)
+org.apache.thrift.protocol.TList _list897 = 
iprot.readListBegin();
+struct.poolTriggers = new 
ArrayList(_list897.size);
+WMPoolTrigger _elem898;
+for (int _i899 = 0; _i899 < _list897.size; ++_i899)
 {
-  _elem890 = new WMPoolTrigger();
-  _elem890.read(iprot);
-  struct.poolTriggers.add(_elem890);
+  _elem898 = new WMPoolTrigger();
+  _elem898.read(iprot);
+  

[1/9] hive git commit: HIVE-20936 Allow the Worker thread in the metastore to run outside of it (Jaume Marhuenda via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 49f8f84d8 -> 926c1e8e3


http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
--
diff --git 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
index b85731e..f8b8628 100644
--- 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
+++ 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
@@ -14703,6 +14703,288 @@ class CompactionRequest:
   def __ne__(self, other):
 return not (self == other)
 
+class OptionalCompactionInfoStruct:
+  """
+  Attributes:
+   - ci
+  """
+
+  thrift_spec = (
+None, # 0
+(1, TType.STRUCT, 'ci', (CompactionInfoStruct, 
CompactionInfoStruct.thrift_spec), None, ), # 1
+  )
+
+  def __init__(self, ci=None,):
+self.ci = ci
+
+  def read(self, iprot):
+if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and 
isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is 
not None and fastbinary is not None:
+  fastbinary.decode_binary(self, iprot.trans, (self.__class__, 
self.thrift_spec))
+  return
+iprot.readStructBegin()
+while True:
+  (fname, ftype, fid) = iprot.readFieldBegin()
+  if ftype == TType.STOP:
+break
+  if fid == 1:
+if ftype == TType.STRUCT:
+  self.ci = CompactionInfoStruct()
+  self.ci.read(iprot)
+else:
+  iprot.skip(ftype)
+  else:
+iprot.skip(ftype)
+  iprot.readFieldEnd()
+iprot.readStructEnd()
+
+  def write(self, oprot):
+if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and 
self.thrift_spec is not None and fastbinary is not None:
+  oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, 
self.thrift_spec)))
+  return
+oprot.writeStructBegin('OptionalCompactionInfoStruct')
+if self.ci is not None:
+  oprot.writeFieldBegin('ci', TType.STRUCT, 1)
+  self.ci.write(oprot)
+  oprot.writeFieldEnd()
+oprot.writeFieldStop()
+oprot.writeStructEnd()
+
+  def validate(self):
+return
+
+
+  def __hash__(self):
+value = 17
+value = (value * 31) ^ hash(self.ci)
+return value
+
+  def __repr__(self):
+L = ['%s=%r' % (key, value)
+  for key, value in self.__dict__.iteritems()]
+return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+  def __eq__(self, other):
+return isinstance(other, self.__class__) and self.__dict__ == 
other.__dict__
+
+  def __ne__(self, other):
+return not (self == other)
+
+class CompactionInfoStruct:
+  """
+  Attributes:
+   - id
+   - dbname
+   - tablename
+   - partitionname
+   - type
+   - runas
+   - properties
+   - toomanyaborts
+   - state
+   - workerId
+   - start
+   - highestWriteId
+  """
+
+  thrift_spec = (
+None, # 0
+(1, TType.I64, 'id', None, None, ), # 1
+(2, TType.STRING, 'dbname', None, None, ), # 2
+(3, TType.STRING, 'tablename', None, None, ), # 3
+(4, TType.STRING, 'partitionname', None, None, ), # 4
+(5, TType.I32, 'type', None, None, ), # 5
+(6, TType.STRING, 'runas', None, None, ), # 6
+(7, TType.STRING, 'properties', None, None, ), # 7
+(8, TType.BOOL, 'toomanyaborts', None, None, ), # 8
+(9, TType.STRING, 'state', None, None, ), # 9
+(10, TType.STRING, 'workerId', None, None, ), # 10
+(11, TType.I64, 'start', None, None, ), # 11
+(12, TType.I64, 'highestWriteId', None, None, ), # 12
+  )
+
+  def __init__(self, id=None, dbname=None, tablename=None, partitionname=None, 
type=None, runas=None, properties=None, toomanyaborts=None, state=None, 
workerId=None, start=None, highestWriteId=None,):
+self.id = id
+self.dbname = dbname
+self.tablename = tablename
+self.partitionname = partitionname
+self.type = type
+self.runas = runas
+self.properties = properties
+self.toomanyaborts = toomanyaborts
+self.state = state
+self.workerId = workerId
+self.start = start
+self.highestWriteId = highestWriteId
+
+  def read(self, iprot):
+if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and 
isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is 
not None and fastbinary is not None:
+  fastbinary.decode_binary(self, iprot.trans, (self.__class__, 
self.thrift_spec))
+  return
+iprot.readStructBegin()
+while True:
+  (fname, ftype, fid) = iprot.readFieldBegin()
+  if ftype == TType.STOP:
+break
+  if fid == 1:
+if ftype == TType.I64:
+  self.id = iprot.readI64()
+else:
+  iprot.skip(ftype)
+  elif fid == 2:
+if ftype == TType.STRING:
+  self.dbname = 

[8/9] hive git commit: HIVE-20936 Allow the Worker thread in the metastore to run outside of it (Jaume Marhuenda via Eugene Koifman)

[ekoifman]

http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CompactionInfoStruct.java
--
diff --git 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CompactionInfoStruct.java
 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CompactionInfoStruct.java
new file mode 100644
index 000..4aee45c
--- /dev/null
+++ 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CompactionInfoStruct.java
@@ -0,0 +1,1533 @@
+/**
+ * Autogenerated by Thrift Compiler (0.9.3)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ *  @generated
+ */
+package org.apache.hadoop.hive.metastore.api;
+
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+
+import org.apache.thrift.scheme.TupleScheme;
+import org.apache.thrift.protocol.TTupleProtocol;
+import org.apache.thrift.protocol.TProtocolException;
+import org.apache.thrift.EncodingUtils;
+import org.apache.thrift.TException;
+import org.apache.thrift.async.AsyncMethodCallback;
+import org.apache.thrift.server.AbstractNonblockingServer.*;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.EnumSet;
+import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import javax.annotation.Generated;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
+@org.apache.hadoop.classification.InterfaceAudience.Public 
@org.apache.hadoop.classification.InterfaceStability.Stable public class 
CompactionInfoStruct implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, 
Comparable {
+  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new 
org.apache.thrift.protocol.TStruct("CompactionInfoStruct");
+
+  private static final org.apache.thrift.protocol.TField ID_FIELD_DESC = new 
org.apache.thrift.protocol.TField("id", org.apache.thrift.protocol.TType.I64, 
(short)1);
+  private static final org.apache.thrift.protocol.TField DBNAME_FIELD_DESC = 
new org.apache.thrift.protocol.TField("dbname", 
org.apache.thrift.protocol.TType.STRING, (short)2);
+  private static final org.apache.thrift.protocol.TField TABLENAME_FIELD_DESC 
= new org.apache.thrift.protocol.TField("tablename", 
org.apache.thrift.protocol.TType.STRING, (short)3);
+  private static final org.apache.thrift.protocol.TField 
PARTITIONNAME_FIELD_DESC = new 
org.apache.thrift.protocol.TField("partitionname", 
org.apache.thrift.protocol.TType.STRING, (short)4);
+  private static final org.apache.thrift.protocol.TField TYPE_FIELD_DESC = new 
org.apache.thrift.protocol.TField("type", org.apache.thrift.protocol.TType.I32, 
(short)5);
+  private static final org.apache.thrift.protocol.TField RUNAS_FIELD_DESC = 
new org.apache.thrift.protocol.TField("runas", 
org.apache.thrift.protocol.TType.STRING, (short)6);
+  private static final org.apache.thrift.protocol.TField PROPERTIES_FIELD_DESC 
= new org.apache.thrift.protocol.TField("properties", 
org.apache.thrift.protocol.TType.STRING, (short)7);
+  private static final org.apache.thrift.protocol.TField 
TOOMANYABORTS_FIELD_DESC = new 
org.apache.thrift.protocol.TField("toomanyaborts", 
org.apache.thrift.protocol.TType.BOOL, (short)8);
+  private static final org.apache.thrift.protocol.TField STATE_FIELD_DESC = 
new org.apache.thrift.protocol.TField("state", 
org.apache.thrift.protocol.TType.STRING, (short)9);
+  private static final org.apache.thrift.protocol.TField WORKER_ID_FIELD_DESC 
= new org.apache.thrift.protocol.TField("workerId", 
org.apache.thrift.protocol.TType.STRING, (short)10);
+  private static final org.apache.thrift.protocol.TField START_FIELD_DESC = 
new org.apache.thrift.protocol.TField("start", 
org.apache.thrift.protocol.TType.I64, (short)11);
+  private static final org.apache.thrift.protocol.TField 
HIGHEST_WRITE_ID_FIELD_DESC = new 
org.apache.thrift.protocol.TField("highestWriteId", 
org.apache.thrift.protocol.TType.I64, (short)12);
+
+  private static final Map, SchemeFactory> schemes = 
new HashMap, SchemeFactory>();
+  static {
+schemes.put(StandardScheme.class, new 
CompactionInfoStructStandardSchemeFactory());
+schemes.put(TupleScheme.class, new 
CompactionInfoStructTupleSchemeFactory());
+  }
+
+  private long id; // required
+  private String dbname; // required
+  private String tablename; // required
+  private String partitionname; // optional
+  private 

[9/9] hive git commit: HIVE-20936 Allow the Worker thread in the metastore to run outside of it (Jaume Marhuenda via Eugene Koifman)

[ekoifman]

HIVE-20936 Allow the Worker thread in the metastore to run outside of it (Jaume 
Marhuenda via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/926c1e8e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/926c1e8e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/926c1e8e

Branch: refs/heads/master
Commit: 926c1e8e352582a2bb89d458fb45c89c698c56fc
Parents: 49f8f84
Author: Jaume Marhuenda 
Authored: Fri Dec 21 20:32:58 2018 -0800
Committer: Eugene Koifman 
Committed: Fri Dec 21 20:33:35 2018 -0800

--
 .../hive/hcatalog/streaming/TestStreaming.java  | 3 +-
 .../hive/ql/txn/compactor/TestCompactor.java| 2 +-
 .../hadoop/hive/ql/txn/compactor/Cleaner.java   | 4 +-
 .../hive/ql/txn/compactor/CompactorMR.java  |18 +-
 .../hive/ql/txn/compactor/CompactorThread.java  |71 +-
 .../hadoop/hive/ql/txn/compactor/Initiator.java | 4 +-
 .../txn/compactor/MetaStoreCompactorThread.java |86 +
 .../ql/txn/compactor/RemoteCompactorThread.java |65 +
 .../hadoop/hive/ql/txn/compactor/Worker.java|90 +-
 .../apache/hadoop/hive/ql/TestTxnCommands2.java | 6 +-
 .../hadoop/hive/ql/TxnCommandsBaseForTests.java | 9 +-
 .../apache/hive/service/server/HiveServer2.java |18 +
 .../metastore/api/AlterPartitionsRequest.java   |36 +-
 .../metastore/api/ClearFileMetadataRequest.java |32 +-
 .../hive/metastore/api/ClientCapabilities.java  |32 +-
 .../metastore/api/CompactionInfoStruct.java |  1533 +++
 .../metastore/api/FindSchemasByColsResp.java|36 +-
 .../hive/metastore/api/FireEventRequest.java|32 +-
 .../metastore/api/GetAllFunctionsResponse.java  |36 +-
 .../api/GetFileMetadataByExprRequest.java   |32 +-
 .../api/GetFileMetadataByExprResult.java|48 +-
 .../metastore/api/GetFileMetadataRequest.java   |32 +-
 .../metastore/api/GetFileMetadataResult.java|44 +-
 .../metastore/api/GetPartitionsFilterSpec.java  |32 +-
 .../api/GetPartitionsProjectionSpec.java|32 +-
 .../metastore/api/GetPartitionsRequest.java |32 +-
 .../metastore/api/GetPartitionsResponse.java|36 +-
 .../hive/metastore/api/GetTablesRequest.java|32 +-
 .../hive/metastore/api/GetTablesResult.java |36 +-
 .../metastore/api/InsertEventRequestData.java   |96 +-
 .../metastore/api/NotificationEventRequest.java |32 +-
 .../api/NotificationEventResponse.java  |36 +-
 .../api/OptionalCompactionInfoStruct.java   |   395 +
 .../metastore/api/PutFileMetadataRequest.java   |64 +-
 .../metastore/api/RenamePartitionRequest.java   |32 +-
 .../hive/metastore/api/SchemaVersion.java   |36 +-
 .../hive/metastore/api/ThriftHiveMetastore.java | 11977 +
 .../hive/metastore/api/WMFullResourcePlan.java  |   144 +-
 .../api/WMGetAllResourcePlanResponse.java   |36 +-
 .../WMGetTriggersForResourePlanResponse.java|36 +-
 .../api/WMValidateResourcePlanResponse.java |64 +-
 .../api/WriteNotificationLogRequest.java|32 +-
 .../gen-php/metastore/ThriftHiveMetastore.php   |  2947 +++-
 .../src/gen/thrift/gen-php/metastore/Types.php  |  1064 +-
 .../hive_metastore/ThriftHiveMetastore-remote   |49 +
 .../hive_metastore/ThriftHiveMetastore.py   |  3643 +++--
 .../gen/thrift/gen-py/hive_metastore/ttypes.py  |   726 +-
 .../gen/thrift/gen-rb/hive_metastore_types.rb   |61 +
 .../gen/thrift/gen-rb/thrift_hive_metastore.rb  |   397 +
 .../hive/metastore/HiveMetaStoreClient.java |35 +
 .../hadoop/hive/metastore/IMetaStoreClient.java |59 +
 .../hive/metastore/conf/MetastoreConf.java  | 4 +
 .../src/main/thrift/hive_metastore.thrift   |26 +
 .../hadoop/hive/metastore/HiveMetaStore.java|44 +-
 .../hadoop/hive/metastore/MetaStoreThread.java  | 3 +-
 .../hive/metastore/txn/CompactionInfo.java  |73 +-
 .../metastore/utils/MetaStoreServerUtils.java   | 2 -
 .../HiveMetaStoreClientPreCatalog.java  |35 +
 .../apache/hive/streaming/TestStreaming.java| 3 +-
 59 files changed, 18604 insertions(+), 6016 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
--
diff --git 
a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
 
b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
index b290a40..5935220 100644
--- 
a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
+++ 
b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
@@ -58,7 +58,6 @@ import 

[4/9] hive git commit: HIVE-20936 Allow the Worker thread in the metastore to run outside of it (Jaume Marhuenda via Eugene Koifman)

[ekoifman]

http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
--
diff --git 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
index 3170798..b94dd25 100644
--- 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
+++ 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
@@ -1294,6 +1294,42 @@ interface ThriftHiveMetastoreIf extends 
\FacebookServiceIf {
*/
   public function add_dynamic_partitions(\metastore\AddDynamicPartitions 
$rqst);
   /**
+   * @param string $workerId
+   * @return \metastore\OptionalCompactionInfoStruct
+   * @throws \metastore\MetaException
+   */
+  public function find_next_compact($workerId);
+  /**
+   * @param \metastore\CompactionInfoStruct $cr
+   * @param int $txn_id
+   */
+  public function update_compactor_state(\metastore\CompactionInfoStruct $cr, 
$txn_id);
+  /**
+   * @param \metastore\CompactionInfoStruct $cr
+   * @return string[]
+   */
+  public function find_columns_with_stats(\metastore\CompactionInfoStruct $cr);
+  /**
+   * @param \metastore\CompactionInfoStruct $cr
+   * @throws \metastore\MetaException
+   */
+  public function mark_cleaned(\metastore\CompactionInfoStruct $cr);
+  /**
+   * @param \metastore\CompactionInfoStruct $cr
+   * @throws \metastore\MetaException
+   */
+  public function mark_compacted(\metastore\CompactionInfoStruct $cr);
+  /**
+   * @param \metastore\CompactionInfoStruct $cr
+   * @throws \metastore\MetaException
+   */
+  public function mark_failed(\metastore\CompactionInfoStruct $cr);
+  /**
+   * @param string $jobId
+   * @param int $cq_id
+   */
+  public function set_hadoop_jobid($jobId, $cq_id);
+  /**
* @param \metastore\NotificationEventRequest $rqst
* @return \metastore\NotificationEventResponse
*/
@@ -11090,6 +11126,362 @@ class ThriftHiveMetastoreClient extends 
\FacebookServiceClient implements \metas
 return;
   }
 
+  public function find_next_compact($workerId)
+  {
+$this->send_find_next_compact($workerId);
+return $this->recv_find_next_compact();
+  }
+
+  public function send_find_next_compact($workerId)
+  {
+$args = new \metastore\ThriftHiveMetastore_find_next_compact_args();
+$args->workerId = $workerId;
+$bin_accel = ($this->output_ instanceof TBinaryProtocolAccelerated) && 
function_exists('thrift_protocol_write_binary');
+if ($bin_accel)
+{
+  thrift_protocol_write_binary($this->output_, 'find_next_compact', 
TMessageType::CALL, $args, $this->seqid_, $this->output_->isStrictWrite());
+}
+else
+{
+  $this->output_->writeMessageBegin('find_next_compact', 
TMessageType::CALL, $this->seqid_);
+  $args->write($this->output_);
+  $this->output_->writeMessageEnd();
+  $this->output_->getTransport()->flush();
+}
+  }
+
+  public function recv_find_next_compact()
+  {
+$bin_accel = ($this->input_ instanceof TBinaryProtocolAccelerated) && 
function_exists('thrift_protocol_read_binary');
+if ($bin_accel) $result = thrift_protocol_read_binary($this->input_, 
'\metastore\ThriftHiveMetastore_find_next_compact_result', 
$this->input_->isStrictRead());
+else
+{
+  $rseqid = 0;
+  $fname = null;
+  $mtype = 0;
+
+  $this->input_->readMessageBegin($fname, $mtype, $rseqid);
+  if ($mtype == TMessageType::EXCEPTION) {
+$x = new TApplicationException();
+$x->read($this->input_);
+$this->input_->readMessageEnd();
+throw $x;
+  }
+  $result = new \metastore\ThriftHiveMetastore_find_next_compact_result();
+  $result->read($this->input_);
+  $this->input_->readMessageEnd();
+}
+if ($result->success !== null) {
+  return $result->success;
+}
+if ($result->o1 !== null) {
+  throw $result->o1;
+}
+throw new \Exception("find_next_compact failed: unknown result");
+  }
+
+  public function update_compactor_state(\metastore\CompactionInfoStruct $cr, 
$txn_id)
+  {
+$this->send_update_compactor_state($cr, $txn_id);
+$this->recv_update_compactor_state();
+  }
+
+  public function send_update_compactor_state(\metastore\CompactionInfoStruct 
$cr, $txn_id)
+  {
+$args = new \metastore\ThriftHiveMetastore_update_compactor_state_args();
+$args->cr = $cr;
+$args->txn_id = $txn_id;
+$bin_accel = ($this->output_ instanceof TBinaryProtocolAccelerated) && 
function_exists('thrift_protocol_write_binary');
+if ($bin_accel)
+{
+  thrift_protocol_write_binary($this->output_, 'update_compactor_state', 
TMessageType::CALL, $args, $this->seqid_, $this->output_->isStrictWrite());
+}
+else
+{
+  

hive git commit: HIVE-20859 clean up invocation of Worker/Cleaner/Initiator in test code (Igor Kryvenko via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 04cc68116 -> 0d200d23c


HIVE-20859 clean up invocation of Worker/Cleaner/Initiator in test code (Igor 
Kryvenko via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0d200d23
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0d200d23
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0d200d23

Branch: refs/heads/master
Commit: 0d200d23c8db5ac0805ab5db953434b96b7bf9fc
Parents: 04cc681
Author: Igor Kryvenko 
Authored: Fri Dec 21 11:12:48 2018 -0800
Committer: Eugene Koifman 
Committed: Fri Dec 21 11:12:48 2018 -0800

--
 .../hive/ql/txn/compactor/TestCompactor.java| 359 +--
 1 file changed, 89 insertions(+), 270 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/0d200d23/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
index 5af047f..d3800cd 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
@@ -17,6 +17,9 @@
  */
 package org.apache.hadoop.hive.ql.txn.compactor;
 
+import static org.apache.hadoop.hive.ql.TestTxnCommands2.runCleaner;
+import static org.apache.hadoop.hive.ql.TestTxnCommands2.runInitiator;
+import static org.apache.hadoop.hive.ql.TestTxnCommands2.runWorker;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
@@ -78,7 +81,6 @@ import org.apache.hadoop.hive.ql.io.orc.OrcStruct;
 import org.apache.hadoop.hive.ql.io.orc.Reader;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
 import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.mapred.JobConf;
 import org.apache.hive.common.util.Retry;
 import org.apache.hive.hcatalog.common.HCatUtil;
 import org.apache.hive.hcatalog.streaming.DelimitedInputWriter;
@@ -230,27 +232,15 @@ public class TestCompactor {
 Assert.assertEquals("3\tmark\t1900\tsoon", 
valuesReadFromHiveDriver.get(2));
 Assert.assertEquals("4\tdouglas\t1901\tlast_century", 
valuesReadFromHiveDriver.get(3));
 Assert.assertEquals("5\tdoc\t1902\tyesterday", 
valuesReadFromHiveDriver.get(4));
-
-Initiator initiator = new Initiator();
-initiator.setThreadId((int) initiator.getId());
 conf.setIntVar(HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_NUM_THRESHOLD, 0);
-initiator.setConf(conf);
-AtomicBoolean stop = new AtomicBoolean();
-stop.set(true);
-initiator.init(stop, new AtomicBoolean());
-initiator.run();
+runInitiator(conf);
 
 TxnStore txnHandler = TxnUtils.getTxnStore(conf);
 ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
 List compacts = rsp.getCompacts();
 Assert.assertEquals(4, compacts.size());
 SortedSet partNames = new TreeSet();
-for (int i = 0; i < compacts.size(); i++) {
-  Assert.assertEquals("default", compacts.get(i).getDbname());
-  Assert.assertEquals(tblName, compacts.get(i).getTablename());
-  Assert.assertEquals("initiated", compacts.get(i).getState());
-  partNames.add(compacts.get(i).getPartitionname());
-}
+verifyCompactions(compacts, partNames, tblName);
 List names = new ArrayList(partNames);
 Assert.assertEquals("ds=last_century", names.get(0));
 Assert.assertEquals("ds=soon", names.get(1));
@@ -332,27 +322,16 @@ public class TestCompactor {
 Assert.assertEquals("4\tdouglas\t2000\tlast_century", 
valuesReadFromHiveDriver.get(3));
 Assert.assertEquals("5\tdoc\t2000\tyesterday", 
valuesReadFromHiveDriver.get(4));
 
-Initiator initiator = new Initiator();
-initiator.setThreadId((int) initiator.getId());
 // Set to 1 so insert doesn't set it off but update does
 conf.setIntVar(HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_NUM_THRESHOLD, 1);
-initiator.setConf(conf);
-AtomicBoolean stop = new AtomicBoolean();
-stop.set(true);
-initiator.init(stop, new AtomicBoolean());
-initiator.run();
+runInitiator(conf);
 
 TxnStore txnHandler = TxnUtils.getTxnStore(conf);
 ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
 List compacts = rsp.getCompacts();
 Assert.assertEquals(4, compacts.size());
 SortedSet partNames = new TreeSet();
-for (int i = 0; i < compacts.size(); i++) {
-  Assert.assertEquals("default", compacts.get(i).getDbname());
-  

hive git commit: HIVE-20943: Handle Compactor transaction abort properly (Eugene Koifman, reviewed by Vaibhav Gumashta)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 3f60b31e3 -> 1020be050


HIVE-20943: Handle Compactor transaction abort properly (Eugene Koifman, 
reviewed by Vaibhav Gumashta)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1020be05
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1020be05
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1020be05

Branch: refs/heads/master
Commit: 1020be050a85afb095c8493013412e95b7f62294
Parents: 3f60b31
Author: Eugene Koifman 
Authored: Tue Dec 18 19:05:38 2018 -0800
Committer: Eugene Koifman 
Committed: Tue Dec 18 19:05:38 2018 -0800

--
 .../compactor/TestCleanerWithReplication.java   | 12 ++-
 .../hadoop/hive/ql/txn/compactor/Worker.java| 38 ++--
 .../metastore/txn/TestCompactionTxnHandler.java |  4 +-
 .../apache/hadoop/hive/ql/TestTxnCommands2.java | 31 +--
 .../apache/hadoop/hive/ql/TestTxnCommands3.java | 73 +++-
 .../hadoop/hive/ql/TxnCommandsBaseForTests.java | 42 +++--
 .../hive/ql/txn/compactor/TestCleaner.java  | 21 +++--
 .../metastore/txn/CompactionTxnHandler.java | 92 +---
 .../hadoop/hive/metastore/txn/TxnHandler.java   | 53 ++-
 .../hadoop/hive/metastore/txn/TxnStore.java | 17 ++--
 10 files changed, 220 insertions(+), 163 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/1020be05/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCleanerWithReplication.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCleanerWithReplication.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCleanerWithReplication.java
index 14d3894..d956067 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCleanerWithReplication.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCleanerWithReplication.java
@@ -113,8 +113,9 @@ public class TestCleanerWithReplication extends 
CompactorTest {
 CompactionRequest rqst = new CompactionRequest(dbName, "camtc", 
CompactionType.MAJOR);
 txnHandler.compact(rqst);
 CompactionInfo ci = txnHandler.findNextToCompact("fred");
+ci.runAs = System.getProperty("user.name");
+txnHandler.updateCompactorState(ci, openTxn());
 txnHandler.markCompacted(ci);
-txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
 
 assertCleanerActions(6);
   }
@@ -135,8 +136,9 @@ public class TestCleanerWithReplication extends 
CompactorTest {
 rqst.setPartitionname("ds=today");
 txnHandler.compact(rqst);
 CompactionInfo ci = txnHandler.findNextToCompact("fred");
+ci.runAs = System.getProperty("user.name");
+txnHandler.updateCompactorState(ci, openTxn());
 txnHandler.markCompacted(ci);
-txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
 
 assertCleanerActions(6);
   }
@@ -155,8 +157,9 @@ public class TestCleanerWithReplication extends 
CompactorTest {
 CompactionRequest rqst = new CompactionRequest(dbName, "camitc", 
CompactionType.MINOR);
 txnHandler.compact(rqst);
 CompactionInfo ci = txnHandler.findNextToCompact("fred");
+ci.runAs = System.getProperty("user.name");
+txnHandler.updateCompactorState(ci, openTxn());
 txnHandler.markCompacted(ci);
-txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
 
 assertCleanerActions(4);
   }
@@ -177,8 +180,9 @@ public class TestCleanerWithReplication extends 
CompactorTest {
 rqst.setPartitionname("ds=today");
 txnHandler.compact(rqst);
 CompactionInfo ci = txnHandler.findNextToCompact("fred");
+ci.runAs = System.getProperty("user.name");
+txnHandler.updateCompactorState(ci, openTxn());
 txnHandler.markCompacted(ci);
-txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
 
 assertCleanerActions(4);
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/1020be05/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java 
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
index 2104341..49662cd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils;
 import org.apache.hadoop.hive.metastore.txn.TxnStore;
-import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
 import 

hive git commit: HIVE-19081: Add partition should prevent loading acid files (Igor Kryvenko via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 4d9df0fdf -> ec5ce5da8


HIVE-19081: Add partition should prevent loading acid files (Igor Kryvenko via 
Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ec5ce5da
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ec5ce5da
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ec5ce5da

Branch: refs/heads/master
Commit: ec5ce5da8079f50b8555b5178e739ec71eafa3fa
Parents: 4d9df0f
Author: Igor Kryvenko 
Authored: Mon Dec 17 13:48:42 2018 -0800
Committer: Eugene Koifman 
Committed: Mon Dec 17 16:47:11 2018 -0800

--
 .../org/apache/hadoop/hive/ql/ErrorMsg.java |  2 +-
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java | 43 
 .../hive/ql/parse/DDLSemanticAnalyzer.java  |  1 +
 .../hive/ql/parse/LoadSemanticAnalyzer.java | 23 +--
 .../hadoop/hive/ql/TestTxnAddPartition.java | 26 +++-
 5 files changed, 71 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/ec5ce5da/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java 
b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index 1a58311..d58f626 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -461,7 +461,7 @@ public enum ErrorMsg {
 "Grouping sets size cannot be greater than 64"),
   REBUILD_NO_MATERIALIZED_VIEW(10412, "Rebuild command only valid for 
materialized views"),
   LOAD_DATA_ACID_FILE(10413,
-  "\"{0}\" was created created by Acid write - it cannot be loaded into 
anther Acid table",
+  "\"{0}\" was created by Acid write - it cannot be loaded into anther 
Acid table",
   true),
   ACID_OP_ON_INSERTONLYTRAN_TABLE(10414, "Attempt to do update or delete on 
table {0} that is " +
 "insert-only transactional", true),

http://git-wip-us.apache.org/repos/asf/hive/blob/ec5ce5da/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index d36b4d1..badcc55 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -22,6 +22,8 @@ import static 
org.apache.hadoop.hive.ql.exec.Utilities.COPY_KEYWORD;
 
 import java.io.IOException;
 import java.io.Serializable;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -65,6 +67,8 @@ import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
 import org.apache.hadoop.hive.ql.lockmgr.LockException;
 import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
 import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.parse.LoadSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
 import org.apache.hadoop.hive.ql.plan.TableScanDesc;
 import org.apache.hadoop.hive.ql.session.SessionState;
@@ -2493,4 +2497,43 @@ public class AcidUtils {
 }
 return lockComponents;
   }
+
+  /**
+   * Safety check to make sure a file take from one acid table is not added 
into another acid table
+   * since the ROW__IDs embedded as part a write to one table won't make sense 
in different
+   * table/cluster.
+   */
+  public static void validateAcidFiles(Table table, FileStatus[] srcs, 
FileSystem fs) throws SemanticException {
+if (!AcidUtils.isFullAcidTable(table)) {
+  return;
+}
+validateAcidFiles(srcs, fs);
+  }
+
+  private static void validateAcidFiles(FileStatus[] srcs, FileSystem fs) 
throws SemanticException {
+try {
+  for (FileStatus oneSrc : srcs) {
+if (!AcidUtils.MetaDataFile.isRawFormatFile(oneSrc.getPath(), fs)) {
+  throw new SemanticException(ErrorMsg.LOAD_DATA_ACID_FILE, 
oneSrc.getPath().toString());
+}
+  }
+} catch (IOException ex) {
+  throw new SemanticException(ex);
+}
+  }
+
+  /**
+   * Safety check to make sure the given location is not the location of acid 
table and
+   * all it's files  will be not added into another acid table
+   */
+  public static void validateAcidPartitionLocation(String location, 
Configuration conf) throws SemanticException {
+try {
+  URI uri = new URI(location);
+  FileSystem fs = FileSystem.get(uri, conf);
+  FileStatus[] fileStatuses = LoadSemanticAnalyzer.matchFilesOrDir(fs, new 
Path(uri));
+  validateAcidFiles(fileStatuses, fs);
+} catch 

[1/2] hive git commit: HIVE-20941: Compactor produces a delete_delta_x_y even if there are no input delete events (Igor Kryvenko via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master b5b637127 -> e8e0396c1


HIVE-20941: Compactor produces a delete_delta_x_y even if there are no input 
delete events (Igor Kryvenko via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7a3cac29
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7a3cac29
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7a3cac29

Branch: refs/heads/master
Commit: 7a3cac298b882e5a81838ba74074e0633294e479
Parents: b5b6371
Author: Igor Kryvenko 
Authored: Thu Dec 13 15:52:40 2018 -0800
Committer: Eugene Koifman 
Committed: Thu Dec 13 15:52:40 2018 -0800

--
 .../hive/ql/txn/compactor/TestCompactor.java| 41 ++--
 .../hive/ql/txn/compactor/CompactorMR.java  | 49 
 .../apache/hadoop/hive/ql/TestTxnCommands2.java | 26 +++
 .../apache/hadoop/hive/ql/TestTxnCommands3.java | 14 ++
 4 files changed, 55 insertions(+), 75 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/7a3cac29/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
index beb36d7..5af047f 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
@@ -17,11 +17,11 @@
  */
 package org.apache.hadoop.hive.ql.txn.compactor;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 
 import java.io.File;
-import java.io.FileNotFoundException;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -80,7 +80,6 @@ import 
org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hive.common.util.Retry;
-import org.apache.hive.common.util.RetryTestRunner;
 import org.apache.hive.hcatalog.common.HCatUtil;
 import org.apache.hive.hcatalog.streaming.DelimitedInputWriter;
 import org.apache.hive.hcatalog.streaming.HiveEndPoint;
@@ -1459,25 +1458,10 @@ public class TestCompactor {
 checkExpectedTxnsPresent(null, new Path[]{minorCompactedDelta}, 
columnNamesProperty, columnTypesProperty,
   0, 1L, 2L, 1);
 
-// Verify that we have got correct set of delete_deltas.
+//Assert that we have no delete deltas if there are no input delete events.
 FileStatus[] deleteDeltaStat =
   fs.listStatus(new Path(table.getSd().getLocation()), 
AcidUtils.deleteEventDeltaDirFilter);
-String[] deleteDeltas = new String[deleteDeltaStat.length];
-Path minorCompactedDeleteDelta = null;
-for (int i = 0; i < deleteDeltas.length; i++) {
-  deleteDeltas[i] = deleteDeltaStat[i].getPath().getName();
-  if (deleteDeltas[i].equals("delete_delta_001_002_v005")) {
-minorCompactedDeleteDelta = deleteDeltaStat[i].getPath();
-  }
-}
-Arrays.sort(deleteDeltas);
-String[] expectedDeleteDeltas = new 
String[]{"delete_delta_001_002_v005"};
-if (!Arrays.deepEquals(expectedDeleteDeltas, deleteDeltas)) {
-  Assert.fail("Expected: " + Arrays.toString(expectedDeleteDeltas) + ", 
found: " + Arrays.toString(deleteDeltas));
-}
-// There should be no rows in the delete_delta because there have been no 
delete events.
-checkExpectedTxnsPresent(null, new Path[]{minorCompactedDeleteDelta}, 
columnNamesProperty, columnTypesProperty, 0,
-  0L, 0L, 1);
+assertEquals(0, deleteDeltaStat.length);
   }
 
   @Test
@@ -1550,25 +1534,10 @@ public class TestCompactor {
 checkExpectedTxnsPresent(null, new Path[]{resultFile}, 
columnNamesProperty, columnTypesProperty,
   0, 1L, 4L, 1);
 
-// Verify that we have got correct set of delete_deltas also
+//Assert that we have no delete deltas if there are no input delete events.
 FileStatus[] deleteDeltaStat =
   fs.listStatus(new Path(table.getSd().getLocation()), 
AcidUtils.deleteEventDeltaDirFilter);
-String[] deleteDeltas = new String[deleteDeltaStat.length];
-Path minorCompactedDeleteDelta = null;
-for (int i = 0; i < deleteDeltas.length; i++) {
-  deleteDeltas[i] = deleteDeltaStat[i].getPath().getName();
-  if (deleteDeltas[i].equals("delete_delta_001_004_v009")) {
-minorCompactedDeleteDelta = deleteDeltaStat[i].getPath();
-  }
-}
-Arrays.sort(deleteDeltas);
-String[] 

[2/2] hive git commit: HIVE-21020: log which table/partition is being processed by a txn in Worker (Eugene Koifman, reviewed by Jason Dere, Igor Kryvenko)

[ekoifman]

HIVE-21020: log which table/partition is being processed by a txn in Worker 
(Eugene Koifman, reviewed by Jason Dere, Igor Kryvenko)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e8e0396c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e8e0396c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e8e0396c

Branch: refs/heads/master
Commit: e8e0396c17bd126e116d0326279ed6da91e9ea37
Parents: 7a3cac2
Author: Eugene Koifman 
Authored: Thu Dec 13 15:54:53 2018 -0800
Committer: Eugene Koifman 
Committed: Thu Dec 13 15:54:53 2018 -0800

--
 ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/e8e0396c/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java 
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
index 4a1cac1..2104341 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hive.ql.txn.compactor;
 
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hive.common.JavaUtils;
 import org.apache.hadoop.hive.common.ValidCompactorWriteIdList;
 import org.apache.hadoop.hive.common.ValidTxnList;
 import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
@@ -198,7 +199,7 @@ public class Worker extends CompactorThread {
 jobName.append("-compactor-");
 jobName.append(ci.getFullPartitionName());
 
-LOG.info("Starting " + ci.type.toString() + " compaction for " + 
ci.getFullPartitionName());
+LOG.info("Starting " + ci.type.toString() + " compaction for " + 
ci.getFullPartitionName() + " in " + JavaUtils.txnIdToString(compactorTxnId));
 final StatsUpdater su = StatsUpdater.init(ci, 
txnHandler.findColumnsWithStats(ci), conf,
   runJobAsSelf(runAs) ? runAs : t.getOwner());
 final CompactorMR mr = new CompactorMR();

hive git commit: HIVE-20942 Worker should heartbeat its own txn (Igor Kryvenko via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master aae392ff8 -> d8c500b89


HIVE-20942 Worker should heartbeat its own txn (Igor Kryvenko via Eugene 
Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d8c500b8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d8c500b8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d8c500b8

Branch: refs/heads/master
Commit: d8c500b896daf4c63b08bb4788fbc5ee374292b8
Parents: aae392f
Author: Igor Kryvenko 
Authored: Fri Dec 7 15:04:49 2018 -0800
Committer: Eugene Koifman 
Committed: Fri Dec 7 15:04:49 2018 -0800

--
 .../hadoop/hive/ql/txn/compactor/Worker.java| 62 +---
 1 file changed, 55 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/d8c500b8/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java 
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
index 939f8c7..4a1cac1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
@@ -29,7 +29,9 @@ import org.apache.hadoop.hive.metastore.api.OpenTxnRequest;
 import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils;
+import org.apache.hadoop.hive.metastore.txn.TxnStore;
 import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
 import org.apache.hadoop.mapred.JobConf;
 import org.slf4j.Logger;
@@ -53,6 +55,7 @@ import java.security.PrivilegedExceptionAction;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
@@ -89,6 +92,7 @@ public class Worker extends CompactorThread {
   boolean launchedJob = false;
   // Make sure nothing escapes this run method and kills the metastore at 
large,
   // so wrap it in a big catch Throwable statement.
+  CompactionHeartbeater heartbeater = null;
   try {
 final CompactionInfo ci = txnHandler.findNextToCompact(workerName);
 LOG.debug("Processing compaction request " + ci);
@@ -176,13 +180,8 @@ public class Worker extends CompactorThread {
 //todo: now we can update compaction_queue entry with this id
 //also make sure to write to TXN_COMPONENTS so that if txn aborts, we 
don't delete the metadata about it from TXNS
 
-HeartbeatRequest heartbeatRequest = new HeartbeatRequest();
-heartbeatRequest.setTxnid(compactorTxnId);
-heartbeatRequest.setLockid(0);
-/**
- * todo: now set up a thread to do the heartbeat
- */
-txnHandler.heartbeat(heartbeatRequest);
+heartbeater = new CompactionHeartbeater(txnHandler, compactorTxnId, 
fullTableName, conf);
+heartbeater.start();
 
 ValidTxnList validTxnList = 
TxnCommonUtils.createValidReadTxnList(txnHandler.getOpenTxns(), compactorTxnId);
 GetValidWriteIdsRequest rqst = new 
GetValidWriteIdsRequest(Collections.singletonList(fullTableName));
@@ -225,6 +224,7 @@ public class Worker extends CompactorThread {
   ci.getFullPartitionName(), exception);
 }
   }
+  heartbeater.cancel();
   txnHandler.markCompacted(ci);
   txnHandler.commitTxn(new CommitTxnRequest(compactorTxnId));
   if (conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST)) {
@@ -239,6 +239,10 @@ public class Worker extends CompactorThread {
   } catch (Throwable t) {
 LOG.error("Caught an exception in the main loop of compactor worker " 
+ workerName + ", " +
 StringUtils.stringifyException(t));
+  } finally {
+if(heartbeater != null) {
+  heartbeater.cancel();
+}
   }
 
   // If we didn't try to launch a job it either means there was no work to 
do or we got
@@ -365,4 +369,48 @@ public class Worker extends CompactorThread {
   }
 }
   }
+
+  static final class CompactionHeartbeater extends Thread {
+static final private Logger LOG = 
LoggerFactory.getLogger(CompactionHeartbeater.class);
+private final TxnStore txnHandler;
+private final AtomicBoolean stop = new AtomicBoolean();
+private final long compactorTxnId;
+private final String tableName;
+private final HiveConf conf;
+private final long interval;
+public CompactionHeartbeater(TxnStore txnHandler, long compactorTxnId, 
String tableName, HiveConf conf) 

[3/3] hive git commit: HIVE-20823: Make Compactor run in a transaction (Eugene Koifman, reviewed by Vaibhav Gumashta)

[ekoifman]

org/apache/hadoop/hive/ql/TestAcidOnTez.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
index 40dd992..d6a4191 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
@@ -608,11 +608,16 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree  
~/dev/hiverwgit/itests/h
   LOG.warn(s);
 }
 String[][] expected2 = {
-   {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":0}\t1\t2", 
"warehouse/t/base_-9223372036854775808/bucket_0"},
-  {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":1}\t3\t4", 
"warehouse/t/base_-9223372036854775808/bucket_0"},
-  {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":2}\t5\t6", 
"warehouse/t/base_-9223372036854775808/bucket_0"},
-  {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":3}\t7\t8", 
"warehouse/t/base_-9223372036854775808/bucket_0"},
-  {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":4}\t9\t10", 
"warehouse/t/base_-9223372036854775808/bucket_0"}
+   {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":0}\t1\t2",
+   "warehouse/t/base_-9223372036854775808_v024/bucket_0"},
+  {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":1}\t3\t4",
+  "warehouse/t/base_-9223372036854775808_v024/bucket_0"},
+  {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":2}\t5\t6",
+  "warehouse/t/base_-9223372036854775808_v024/bucket_0"},
+  {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":3}\t7\t8",
+  "warehouse/t/base_-9223372036854775808_v024/bucket_0"},
+  {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":4}\t9\t10",
+  "warehouse/t/base_-9223372036854775808_v024/bucket_0"}
 };
 Assert.assertEquals("Unexpected row count after major compact", 
expected2.length, rs.size());
 for(int i = 0; i < expected2.length; i++) {

http://git-wip-us.apache.org/repos/asf/hive/blob/ddf3b6cd/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
index 9648645..beb36d7 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
@@ -713,15 +713,15 @@ public class TestCompactor {
   Path resultFile = null;
   for (int i = 0; i < names.length; i++) {
 names[i] = stat[i].getPath().getName();
-if (names[i].equals("delta_001_004")) {
+if (names[i].equals("delta_001_004_v009")) {
   resultFile = stat[i].getPath();
 }
   }
   Arrays.sort(names);
   String[] expected = new String[]{"delta_001_002",
-"delta_001_004", "delta_003_004", 
"delta_005_006"};
+"delta_001_004_v009", "delta_003_004", 
"delta_005_006"};
   if (!Arrays.deepEquals(expected, names)) {
-Assert.fail("Expected: " + Arrays.toString(expected) + ", found: " + 
Arrays.toString(names));
+Assert.fail("Expected: " + Arrays.toString(expected) + ", found: " + 
Arrays.toString(names) + ",stat=" + toString(stat));
   }
   checkExpectedTxnsPresent(null, new Path[]{resultFile}, 
columnNamesProperty, columnTypesProperty,
 0, 1L, 4L, 1);
@@ -767,7 +767,7 @@ public class TestCompactor {
 Assert.fail("Expecting 1 file \"base_004\" and found " + 
stat.length + " files " + Arrays.toString(stat));
   }
   String name = stat[0].getPath().getName();
-  Assert.assertEquals(name, "base_004");
+  Assert.assertEquals("base_004_v009", name);
   checkExpectedTxnsPresent(stat[0].getPath(), null, columnNamesProperty, 
columnTypesProperty, 0, 1L, 4L, 1);
 

[1/3] hive git commit: HIVE-20823: Make Compactor run in a transaction (Eugene Koifman, reviewed by Vaibhav Gumashta)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master f5b14fc04 -> ddf3b6cd0


http://git-wip-us.apache.org/repos/asf/hive/blob/ddf3b6cd/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
--
diff --git 
a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java 
b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index 77fe736..5e085f8 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.metastore.txn.AcidWriteSetService;
 import org.apache.hadoop.hive.metastore.txn.TxnStore;
 import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.ql.TestTxnCommands2;
 import org.junit.After;
 import org.junit.Assert;
 import org.apache.hadoop.hive.common.FileUtils;
@@ -87,7 +88,7 @@ public class TestDbTxnManager2 {
   private static HiveConf conf = new HiveConf(Driver.class);
   private HiveTxnManager txnMgr;
   private Context ctx;
-  private Driver driver;
+  private Driver driver, driver2;
   private TxnStore txnHandler;
 
   public TestDbTxnManager2() throws Exception {
@@ -103,6 +104,7 @@ public class TestDbTxnManager2 {
 SessionState.start(conf);
 ctx = new Context(conf);
 driver = new Driver(new 
QueryState.Builder().withHiveConf(conf).nonIsolated().build(), null);
+driver2 = new Driver(new QueryState.Builder().withHiveConf(conf).build(), 
null);
 TxnDbUtil.cleanDb(conf);
 TxnDbUtil.prepDb(conf);
 SessionState ss = SessionState.get();
@@ -115,6 +117,7 @@ public class TestDbTxnManager2 {
   @After
   public void tearDown() throws Exception {
 driver.close();
+driver2.close();
 if (txnMgr != null) {
   txnMgr.closeTxnManager();
 }
@@ -548,10 +551,10 @@ public class TestDbTxnManager2 {
 checkCmdOnDriver(cpr);
 count = TxnDbUtil.countQueryAgent(conf, "select count(*) from 
COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' 
and CQ_TYPE='i'");
 Assert.assertEquals(1, count);
-org.apache.hadoop.hive.ql.TestTxnCommands2.runWorker(conf);
+TestTxnCommands2.runWorker(conf);
 count = TxnDbUtil.countQueryAgent(conf, "select count(*) from 
COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='r' 
and CQ_TYPE='i'");
 Assert.assertEquals(1, count);
-org.apache.hadoop.hive.ql.TestTxnCommands2.runCleaner(conf);
+TestTxnCommands2.runCleaner(conf);
 count = TxnDbUtil.countQueryAgent(conf, "select count(*) from 
COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11'");
 Assert.assertEquals(0, count);
 count = TxnDbUtil.countQueryAgent(conf, "select count(*) from 
COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11' and 
CC_STATE='s' and CC_TYPE='i'");
@@ -561,10 +564,10 @@ public class TestDbTxnManager2 {
 checkCmdOnDriver(cpr);
 count = TxnDbUtil.countQueryAgent(conf, "select count(*) from 
COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and 
CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='i'");
 Assert.assertEquals(1, count);
-org.apache.hadoop.hive.ql.TestTxnCommands2.runWorker(conf);
+TestTxnCommands2.runWorker(conf);
 count = TxnDbUtil.countQueryAgent(conf, "select count(*) from 
COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and 
CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='r' and CQ_TYPE='i'");
 Assert.assertEquals(1, count);
-org.apache.hadoop.hive.ql.TestTxnCommands2.runCleaner(conf);
+TestTxnCommands2.runCleaner(conf);
 count = TxnDbUtil.countQueryAgent(conf, "select count(*) from 
COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p'");
 Assert.assertEquals(0, count);
 count = TxnDbUtil.countQueryAgent(conf, "select count(*) from 
COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p' and 
CC_STATE='s' and CC_TYPE='i'");
@@ -576,7 +579,7 @@ public class TestDbTxnManager2 {
 checkCmdOnDriver(cpr);
 count = TxnDbUtil.countQueryAgent(conf, "select count(*) from 
COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' 
and CQ_TYPE='a'");
 Assert.assertEquals(1, count);
-org.apache.hadoop.hive.ql.TestTxnCommands2.runWorker(conf); // will fail
+TestTxnCommands2.runWorker(conf); // will fail
 count = TxnDbUtil.countQueryAgent(conf, "select count(*) from 
COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' 
and CQ_TYPE='a'");
 Assert.assertEquals(0, count);
 count = TxnDbUtil.countQueryAgent(conf, "select count(*) from 
COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11' and 
CC_STATE='f' and CC_TYPE='a'");
@@ -586,7 +589,7 @@ public class TestDbTxnManager2 {
 checkCmdOnDriver(cpr);
 count = 

[2/3] hive git commit: HIVE-20823: Make Compactor run in a transaction (Eugene Koifman, reviewed by Vaibhav Gumashta)

[ekoifman]

http://git-wip-us.apache.org/repos/asf/hive/blob/ddf3b6cd/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java 
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
index d9f186c..939f8c7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
@@ -18,15 +18,22 @@
 package org.apache.hadoop.hive.ql.txn.compactor;
 
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hive.common.ValidCompactorWriteIdList;
+import org.apache.hadoop.hive.common.ValidTxnList;
+import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
+import org.apache.hadoop.hive.metastore.api.CommitTxnRequest;
 import org.apache.hadoop.hive.metastore.api.GetValidWriteIdsRequest;
+import org.apache.hadoop.hive.metastore.api.HeartbeatRequest;
 import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.OpenTxnRequest;
 import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils;
+import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
 import org.apache.hadoop.mapred.JobConf;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.common.ValidWriteIdList;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.Warehouse;
 import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
@@ -58,7 +65,7 @@ public class Worker extends CompactorThread {
   static final private long SLEEP_TIME = 5000;
   static final private int baseThreadNum = 10002;
 
-  private String name;
+  private String workerName;
   private JobConf mrJob; // the MR job for compaction
 
   /**
@@ -83,7 +90,7 @@ public class Worker extends CompactorThread {
   // Make sure nothing escapes this run method and kills the metastore at 
large,
   // so wrap it in a big catch Throwable statement.
   try {
-final CompactionInfo ci = txnHandler.findNextToCompact(name);
+final CompactionInfo ci = txnHandler.findNextToCompact(workerName);
 LOG.debug("Processing compaction request " + ci);
 
 if (ci == null && !stop.get()) {
@@ -144,14 +151,6 @@ public class Worker extends CompactorThread {
 // Compaction doesn't work under a transaction and hence pass 0 for 
current txn Id
 // The response will have one entry per table and hence we get only 
one OpenWriteIds
 String fullTableName = TxnUtils.getFullTableName(t.getDbName(), 
t.getTableName());
-GetValidWriteIdsRequest rqst = new 
GetValidWriteIdsRequest(Collections.singletonList(fullTableName));
-final ValidWriteIdList tblValidWriteIds =
-
TxnUtils.createValidCompactWriteIdList(txnHandler.getValidWriteIds(rqst).getTblValidWriteIds().get(0));
-LOG.debug("ValidCompactWriteIdList: " + 
tblValidWriteIds.writeToString());
-txnHandler.setCompactionHighestWriteId(ci, 
tblValidWriteIds.getHighWatermark());
-final StringBuilder jobName = new StringBuilder(name);
-jobName.append("-compactor-");
-jobName.append(ci.getFullPartitionName());
 
 // Determine who to run as
 String runAs;
@@ -162,9 +161,45 @@ public class Worker extends CompactorThread {
   runAs = ci.runAs;
 }
 
-LOG.info("Starting " + ci.type.toString() + " compaction for " +
-ci.getFullPartitionName());
+/**
+ * HIVE-20942: We need a transaction.  could call txnHandler directly 
but then we'd have to set up a hearbeat
+ * but using {@link HiveTxnManager} creates a Thrift connection to the 
HMS
+ * will this cause security checks that could fail?
+ * on the other hand we run SQL via Driver which certainly uses {@link 
HiveTxnManager}
+ final HiveTxnManager txnMgr = 
TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
+ * openTxn requires Context() which is set up based on query 
parse/plan
+ long txnid = txnMgr.openTxn(null, null);
+ */
+OpenTxnRequest otReq = new OpenTxnRequest(1, runAs, hostname());
+otReq.setAgentInfo(getName());//ThreadName
+long compactorTxnId = txnHandler.openTxns(otReq).getTxn_ids().get(0);
+//todo: now we can update compaction_queue entry with this id
+//also make sure to write to TXN_COMPONENTS so that if txn aborts, we 
don't delete the metadata about it from TXNS
+
+HeartbeatRequest heartbeatRequest = new HeartbeatRequest();
+heartbeatRequest.setTxnid(compactorTxnId);
+heartbeatRequest.setLockid(0);
+/**
+ * todo: now set up a thread to do the heartbeat
+  

[2/2] hive git commit: HIVE-20730: Do delete event filtering even if hive.acid.index is not there (Saurabh Seth via Eugene Koifman)

[ekoifman]

HIVE-20730: Do delete event filtering even if hive.acid.index is not there 
(Saurabh Seth via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6fbdf374
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6fbdf374
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6fbdf374

Branch: refs/heads/master
Commit: 6fbdf374888ee03ff73a4b04dfcde496f954c458
Parents: 232a028
Author: Saurabh Seth 
Authored: Wed Nov 14 14:57:59 2018 -0800
Committer: Eugene Koifman 
Committed: Wed Nov 14 14:57:59 2018 -0800

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  2 +
 .../hadoop/hive/ql/io/orc/OrcRecordUpdater.java |  8 ++-
 .../io/orc/VectorizedOrcAcidRowBatchReader.java | 54 
 .../hive/ql/io/orc/TestOrcRawRecordMerger.java  |  2 +
 .../TestVectorizedOrcAcidRowBatchReader.java| 24 +++--
 5 files changed, 74 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/6fbdf374/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 65264f3..6ca1315 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1905,6 +1905,8 @@ public class HiveConf extends Configuration {
 TESTMODE_BUCKET_CODEC_VERSION("hive.test.bucketcodec.version", 1,
   "For testing only.  Will make ACID subsystem write 
RecordIdentifier.bucketId in specified\n" +
 "format", false),
+HIVETESTMODEACIDKEYIDXSKIP("hive.test.acid.key.index.skip", false, "For 
testing only. OrcRecordUpdater will skip "
++ "generation of the hive.acid.key.index", false),
 
 HIVEMERGEMAPFILES("hive.merge.mapfiles", true,
 "Merge small files at the end of a map-only job"),

http://git-wip-us.apache.org/repos/asf/hive/blob/6fbdf374/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
index 4ebd69e..6d4578e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
@@ -347,7 +347,9 @@ public class OrcRecordUpdater implements RecordUpdater {
 
writerOptions.getConfiguration().set(OrcConf.DICTIONARY_KEY_SIZE_THRESHOLD.getAttribute(),
 "-1.0");
   }
 }
-writerOptions.fileSystem(fs).callback(indexBuilder);
+if(!HiveConf.getBoolVar(options.getConfiguration(), 
HiveConf.ConfVars.HIVETESTMODEACIDKEYIDXSKIP)) {
+  writerOptions.fileSystem(fs).callback(indexBuilder);
+}
 rowInspector = (StructObjectInspector)options.getInspector();
 
writerOptions.inspector(createEventObjectInspector(findRecId(options.getInspector(),
 options.getRecordIdColumn(;
@@ -621,6 +623,10 @@ public class OrcRecordUpdater implements RecordUpdater {
   static RecordIdentifier[] parseKeyIndex(Reader reader) {
 String[] stripes;
 try {
+  if (!reader.hasMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME)) {
+return null;
+  }
+
   ByteBuffer val =
   reader.getMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME)
   .duplicate();

http://git-wip-us.apache.org/repos/asf/hive/blob/6fbdf374/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
index 2f809de..6d1ca722 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
@@ -473,10 +473,16 @@ public class VectorizedOrcAcidRowBatchReader
   return new OrcRawRecordMerger.KeyInterval(null, null);
 }
 RecordIdentifier[] keyIndex = OrcRecordUpdater.parseKeyIndex(reader);
-if(keyIndex == null || keyIndex.length != stripes.size()) {
-  LOG.warn("Could not find keyIndex or length doesn't match (" +
-  firstStripeIndex + "," + lastStripeIndex + "," + stripes.size() + 
"," +
-  (keyIndex == null ? -1 : keyIndex.length) + ")");
+
+if(keyIndex == null) {
+  LOG.warn("Could not find keyIndex (" + firstStripeIndex + "," +
+  lastStripeIndex + "," + stripes.size() + ")");
+}
+
+if(keyIndex != null && 

[1/2] hive git commit: HIVE-20888: TxnHandler: sort() called on immutable lists (Igor Kryvenko via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 8fca02e58 -> 6fbdf3748


HIVE-20888: TxnHandler: sort() called on immutable lists (Igor Kryvenko via 
Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/232a0282
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/232a0282
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/232a0282

Branch: refs/heads/master
Commit: 232a0282d054f92b666355d0522d29becdef6e1b
Parents: 8fca02e
Author: Igor Kryvenko 
Authored: Wed Nov 14 14:51:47 2018 -0800
Committer: Eugene Koifman 
Committed: Wed Nov 14 14:51:47 2018 -0800

--
 .../java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java   | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/232a0282/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
--
diff --git 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index 0bb739f..25d79ad 100644
--- 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -1520,7 +1520,10 @@ abstract class TxnHandler implements TxnStore, 
TxnStore.MutexAPI {
   txnIds = rqst.getTxnIds();
 }
 
-Collections.sort(txnIds); //easier to read logs and for assumption 
done in replication flow
+//Easiest check since we can't differentiate do we handle singleton 
list or list with multiple txn ids.
+if(txnIds.size() > 1) {
+  Collections.sort(txnIds); //easier to read logs and for assumption 
done in replication flow
+}
 
 // Check if all the input txns are in open state. Write ID should be 
allocated only for open transactions.
 if (!isTxnsInOpenState(txnIds, stmt)) {

hive git commit: HIVE-20862: QueryId no longer shows up in the logs (Eugene Koifman reviewed by Vaibhav Gumashta)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 6d713b656 -> 840dd431f


HIVE-20862: QueryId no longer shows up in the logs (Eugene Koifman reviewed by 
Vaibhav Gumashta)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/840dd431
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/840dd431
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/840dd431

Branch: refs/heads/master
Commit: 840dd431f3772772fc57060e27e3f2bee72a8936
Parents: 6d713b6
Author: Eugene Koifman 
Authored: Wed Nov 7 17:11:14 2018 -0800
Committer: Eugene Koifman 
Committed: Wed Nov 7 17:11:14 2018 -0800

--
 ql/src/java/org/apache/hadoop/hive/ql/Driver.java  | 13 -
 .../apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java  |  2 +-
 .../apache/hadoop/hive/ql/session/SessionState.java|  1 +
 .../hadoop/hive/metastore/txn/TxnCommonUtils.java  |  1 +
 4 files changed, 11 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/840dd431/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java 
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index ff5086a..57f71a8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -543,7 +543,8 @@ public class Driver implements IDriver {
 
 LockedDriverState.setLockedDriverState(lDrvState);
 
-String queryId = queryState.getQueryId();
+final String queryId = Strings.isNullOrEmpty(queryState.getQueryId()) ?
+QueryPlan.makeQueryId() : queryState.getQueryId();
 
 SparkSession ss = SessionState.get().getSparkSession();
 if (ss != null) {
@@ -1484,7 +1485,7 @@ public class Driver implements IDriver {
 throw new LockException("Unexpected tables in compaction: " + 
txnTables);
   }
   String fullTableName = txnTables.get(0);
-  txnWriteIds = new ValidTxnWriteIdList(0L); // No transaction for the 
compaction for now.
+  txnWriteIds = new ValidTxnWriteIdList(0L); // No transaction for the 
compaction for now. todo: Since MM compaction is a query, a txn has been opened 
at this point
   txnWriteIds.addTableValidWriteIdList(compactionWriteIds);
 } else {
   txnWriteIds = txnMgr.getValidWriteIds(txnTables, txnString);
@@ -1672,8 +1673,10 @@ public class Driver implements IDriver {
   queryTxnMgr.acquireLocks(plan, ctx, userFromUGI, lDrvState);
   // This check is for controlling the correctness of the current state
   if (queryTxnMgr.recordSnapshot(plan) && !validTxnListsGenerated) {
-throw new IllegalStateException("calling recordValidTxn() more than 
once in the same " +
-JavaUtils.txnIdToString(queryTxnMgr.getCurrentTxnId()));
+throw new IllegalStateException(
+"Need to record valid WriteID list but there is no valid TxnID 
list (" +
+JavaUtils.txnIdToString(queryTxnMgr.getCurrentTxnId()) +
+", queryId:" + plan.getQueryId() + ")");
   }
 
   if (plan.hasAcidResourcesInQuery() || hasAcidDdl) {
@@ -2230,7 +2233,7 @@ public class Driver implements IDriver {
 }
 Metrics metrics = MetricsFactory.getInstance();
 
-String queryId = queryState.getQueryId();
+String queryId = plan.getQueryId();
 // Get the query string from the conf file as the compileInternal() method 
might
 // hide sensitive information during query redaction.
 String queryStr = conf.getQueryString();

http://git-wip-us.apache.org/repos/asf/hive/blob/840dd431/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java 
b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
index ba1f1ff..12c1027 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
@@ -262,7 +262,7 @@ public interface HiveTxnManager {
 
   /**
* For resources that support MVCC, the state of the DB must be recorded for 
the duration of the
-   * operation/transaction.  Returns {@code true} if current statment needs to 
do this.
+   * operation/transaction.  Returns {@code true} if current statement needs 
to do this.
*/
   boolean recordSnapshot(QueryPlan queryPlan);
 

http://git-wip-us.apache.org/repos/asf/hive/blob/840dd431/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java 

hive git commit: HIVE-14516: OrcInputFormat.SplitGenerator.callInternal() can be optimized (Igor Kryvenko via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master a99be34a0 -> a26d9d166


HIVE-14516: OrcInputFormat.SplitGenerator.callInternal() can be optimized (Igor 
Kryvenko via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a26d9d16
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a26d9d16
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a26d9d16

Branch: refs/heads/master
Commit: a26d9d166064117152779267e78c6b7fdd2ba6ee
Parents: a99be34
Author: Igor Kryvenko 
Authored: Thu Oct 25 17:00:14 2018 -0700
Committer: Eugene Koifman 
Committed: Thu Oct 25 17:00:14 2018 -0700

--
 .../java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java  | 6 ++
 1 file changed, 2 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/a26d9d16/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index 25b2d48..720dbe5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -1551,9 +1551,7 @@ public class OrcInputFormat implements 
InputFormat,
   } else {
 populateAndCacheStripeDetails();
 boolean[] includeStripe = null;
-// We can't eliminate stripes if there are deltas because the
-// deltas may change the rows making them match the predicate. todo: 
See HIVE-14516.
-if ((deltas == null || deltas.isEmpty()) && context.sarg != null) {
+if (context.sarg != null) {
   String[] colNames =
   extractNeededColNames((readerTypes == null ? fileTypes : 
readerTypes),
   context.conf, readerIncluded, isOriginal);
@@ -2212,7 +2210,7 @@ public class OrcInputFormat implements 
InputFormat,
OrcFile.WriterVersion writerVersion,
List stripeStats,
   int stripeCount, Path filePath, final SchemaEvolution evolution) {
-if (sarg == null || stripeStats == null || writerVersion == 
OrcFile.WriterVersion.ORIGINAL) {
+if (stripeStats == null || writerVersion == 
OrcFile.WriterVersion.ORIGINAL) {
   return null; // only do split pruning if HIVE-8732 has been fixed in the 
writer
 }
 // eliminate stripes that doesn't satisfy the predicate condition

hive git commit: HIVE-17231: ColumnizedDeleteEventRegistry.DeleteReaderValue optimization (Eugene Koifman, reviewed by Gopal V)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 0f2f999bb -> d7be4b9f2


HIVE-17231: ColumnizedDeleteEventRegistry.DeleteReaderValue optimization 
(Eugene Koifman, reviewed by Gopal V)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d7be4b9f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d7be4b9f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d7be4b9f

Branch: refs/heads/master
Commit: d7be4b9f26345439c472969461d3d2c81f7e5057
Parents: 0f2f999
Author: Eugene Koifman 
Authored: Tue Oct 16 11:23:11 2018 -0700
Committer: Eugene Koifman 
Committed: Tue Oct 16 11:23:11 2018 -0700

--
 .../hadoop/hive/ql/exec/FileSinkOperator.java   |  4 ++
 .../hive/ql/io/orc/OrcRawRecordMerger.java  | 55 +++-
 .../io/orc/VectorizedOrcAcidRowBatchReader.java | 13 +
 3 files changed, 12 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/d7be4b9f/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index 8c7a78b..79e41d9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -388,6 +388,10 @@ public class FileSinkOperator extends 
TerminalOperator implements
  *
  * A new FSP is created for each partition, so this only requires the 
bucket numbering and that
  * is mapped in directly as an index.
+ *
+ * This relies on ReduceSinkOperator to shuffle update/delete rows by
+ * UDFToInteger(RecordIdentifier), i.e. by writerId in ROW__ID.
+ * {@link 
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer#getPartitionColsFromBucketColsForUpdateDelete(Operator,
 boolean)}
  */
 public int createDynamicBucket(int bucketNum) {
   // this assumes all paths are bucket names (which means no lookup is 
needed)

http://git-wip-us.apache.org/repos/asf/hive/blob/d7be4b9f/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
index 6be0c74..8cabf96 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
@@ -1122,7 +1122,7 @@ public class OrcRawRecordMerger implements 
AcidInputFormat.RawReader{
   }
   continue;
 }
-for (Path deltaFile : getDeltaFiles(delta, bucket, conf, 
mergerOptions, isBucketed)) {
+for (Path deltaFile : getDeltaFiles(delta, bucket, mergerOptions)) {
   FileSystem fs = deltaFile.getFileSystem(conf);
   if(!fs.exists(deltaFile)) {
 /**
@@ -1262,53 +1262,12 @@ public class OrcRawRecordMerger implements 
AcidInputFormat.RawReader{
* This determines the set of {@link ReaderPairAcid} to create for a given 
delta/.
* For unbucketed tables {@code bucket} can be thought of as a write tranche.
*/
-  static Path[] getDeltaFiles(Path deltaDirectory, int bucket, Configuration 
conf,
-  Options mergerOptions, boolean isBucketed) 
throws IOException {
-if(isBucketed) {
-  /**
-   * for bucketed tables (for now) we always trust that the N in bucketN 
file name means that
-   * all records have {@link RecordIdentifier#getBucketProperty()} 
encoding bucketId = N.  This
-   * means that a delete event in bucketN can only modify an insert in 
another bucketN file for
-   * the same N. (Down the road we may trust it only in certain delta dirs)
-   *
-   * Compactor takes all types of deltas for a given bucket.  For regular 
read, any file that
-   * contains (only) insert events is treated as base and only
-   * delete_delta/ are treated as deltas.
-   */
-assert (!mergerOptions.isCompacting &&
-  deltaDirectory.getName().startsWith(AcidUtils.DELETE_DELTA_PREFIX)
-) || mergerOptions.isCompacting : "Unexpected delta: " + 
deltaDirectory;
-  Path deltaFile = AcidUtils.createBucketFile(deltaDirectory, bucket);
-  return new Path[]{deltaFile};
-}
-/**
- * For unbucketed tables insert events are also stored in bucketN files 
but here N is
- * the writer ID.  We can trust that N matches info in {@link 
RecordIdentifier#getBucketProperty()}
- * delta_x_y but it's not required since we can't trust N for 
delete_delta_x_x/bucketN.
- * Thus we always have to take all files in a 

hive git commit: HIVE-20538: Allow to store a key value together with a transaction. (Jaume M via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 20e595399 -> 69a7fe60f


HIVE-20538: Allow to store a key value together with a transaction. (Jaume M 
via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/69a7fe60
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/69a7fe60
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/69a7fe60

Branch: refs/heads/master
Commit: 69a7fe60f7e103ecb1dbfb3aa6e28024258011ef
Parents: 20e5953
Author: Jaume Marhuenda 
Authored: Mon Oct 15 11:12:37 2018 -0700
Committer: Eugene Koifman 
Committed: Mon Oct 15 11:12:37 2018 -0700

--
 .../hive/metastore/api/CommitTxnKeyValue.java   | 589 +++
 .../hive/metastore/api/CommitTxnRequest.java| 119 +++-
 .../src/gen/thrift/gen-php/metastore/Types.php  | 149 +
 .../gen/thrift/gen-py/hive_metastore/ttypes.py  | 113 +++-
 .../gen/thrift/gen-rb/hive_metastore_types.rb   |  27 +-
 .../hive/metastore/HiveMetaStoreClient.java |  15 +
 .../hadoop/hive/metastore/IMetaStoreClient.java |  26 +
 .../src/main/thrift/hive_metastore.thrift   |   9 +
 .../hadoop/hive/metastore/txn/TxnHandler.java   |  32 +
 .../hadoop/hive/metastore/txn/TxnStore.java |   4 +
 .../HiveMetaStoreClientPreCatalog.java  |  14 +
 .../hive/metastore/TestHiveMetaStoreTxns.java   | 107 
 12 files changed, 1198 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/69a7fe60/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnKeyValue.java
--
diff --git 
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnKeyValue.java
 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnKeyValue.java
new file mode 100644
index 000..297cf59
--- /dev/null
+++ 
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnKeyValue.java
@@ -0,0 +1,589 @@
+/**
+ * Autogenerated by Thrift Compiler (0.9.3)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ *  @generated
+ */
+package org.apache.hadoop.hive.metastore.api;
+
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+
+import org.apache.thrift.scheme.TupleScheme;
+import org.apache.thrift.protocol.TTupleProtocol;
+import org.apache.thrift.protocol.TProtocolException;
+import org.apache.thrift.EncodingUtils;
+import org.apache.thrift.TException;
+import org.apache.thrift.async.AsyncMethodCallback;
+import org.apache.thrift.server.AbstractNonblockingServer.*;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.EnumSet;
+import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import javax.annotation.Generated;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
+@org.apache.hadoop.classification.InterfaceAudience.Public 
@org.apache.hadoop.classification.InterfaceStability.Stable public class 
CommitTxnKeyValue implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, 
Comparable {
+  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new 
org.apache.thrift.protocol.TStruct("CommitTxnKeyValue");
+
+  private static final org.apache.thrift.protocol.TField TABLE_ID_FIELD_DESC = 
new org.apache.thrift.protocol.TField("tableId", 
org.apache.thrift.protocol.TType.I64, (short)1);
+  private static final org.apache.thrift.protocol.TField KEY_FIELD_DESC = new 
org.apache.thrift.protocol.TField("key", 
org.apache.thrift.protocol.TType.STRING, (short)2);
+  private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = 
new org.apache.thrift.protocol.TField("value", 
org.apache.thrift.protocol.TType.STRING, (short)3);
+
+  private static final Map, SchemeFactory> schemes = 
new HashMap, SchemeFactory>();
+  static {
+schemes.put(StandardScheme.class, new 
CommitTxnKeyValueStandardSchemeFactory());
+schemes.put(TupleScheme.class, new CommitTxnKeyValueTupleSchemeFactory());
+  }
+
+  private long tableId; // required
+  private String key; // required
+  private String value; // required
+
+  /** The set of fields this struct contains, along with convenience methods 
for finding and manipulating them. */
+  public enum _Fields implements 

[1/2] hive git commit: HIVE-20719: SELECT statement fails after UPDATE with hive.optimize.sort.dynamic.partition optimization and vectorization on (Eugene Koifman, reviewed by Gopal V)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 87414f37e -> 35278429d


HIVE-20719: SELECT statement fails after UPDATE with 
hive.optimize.sort.dynamic.partition optimization and vectorization on (Eugene 
Koifman, reviewed by Gopal V)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3c6a36b9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3c6a36b9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3c6a36b9

Branch: refs/heads/master
Commit: 3c6a36b99d609b1baf5023fc5e801f74486bbb54
Parents: 87414f3
Author: Eugene Koifman 
Authored: Sat Oct 13 16:38:13 2018 -0700
Committer: Eugene Koifman 
Committed: Sat Oct 13 16:38:13 2018 -0700

--
 .../optimizer/SortedDynPartitionOptimizer.java  |  5 +-
 .../apache/hadoop/hive/ql/TestTxnCommands3.java | 61 
 2 files changed, 64 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/3c6a36b9/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
index 2dc2351..314b8b4 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.hive.ql.lib.RuleRegExp;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
 import org.apache.hadoop.hive.ql.parse.ParseContext;
+import org.apache.hadoop.hive.ql.parse.ParseUtils;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx;
 import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
@@ -208,8 +209,8 @@ public class SortedDynPartitionOptimizer extends Transform {
 if(!VirtualColumn.ROWID.getTypeInfo().equals(ci.getType())) {
   throw new IllegalStateException("expected 1st column to be ROW__ID 
but got wrong type: " + ci.toString());
 }
-//HIVE-17328: not sure this is correct... I don't think is gets 
wrapped in UDFToInteger
-bucketColumns.add(new ExprNodeColumnDesc(ci));
+//add a cast(ROW__ID as int) to wrap in UDFToInteger()
+bucketColumns.add(ParseUtils.createConversionCast(new 
ExprNodeColumnDesc(ci), TypeInfoFactory.intTypeInfo));
   } else {
 if (!destTable.getSortCols().isEmpty()) {
   // Sort columns specified by table

http://git-wip-us.apache.org/repos/asf/hive/blob/3c6a36b9/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java
index a25406d..833e637 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java
@@ -222,4 +222,65 @@ public class TestTxnCommands3 extends 
TxnCommandsBaseForTests {
 rs = runStatementOnDriver("select a, b from T order by a, b");
 Assert.assertEquals(stringifyValues(dataAll), rs);
   }
+
+  /**
+   * Test that rows are routed to proper files based on bucket col/ROW__ID
+   * Only the Vectorized Acid Reader checks if bucketId in ROW__ID inside the 
file
+   * matches the file name and only for files in delete_delta
+   */
+  @Test
+  public void testSdpoBucketed() throws Exception {
+testSdpoBucketed(true, true, 1);
+testSdpoBucketed(true, false, 1);
+testSdpoBucketed(false, true, 1);
+testSdpoBucketed(false, false,1);
+
+testSdpoBucketed(true, true, 2);
+testSdpoBucketed(true, false, 2);
+testSdpoBucketed(false, true, 2);
+testSdpoBucketed(false, false,2);
+  }
+  private void testSdpoBucketed(boolean isVectorized, boolean isSdpo, int 
bucketing_version)
+  throws Exception {
+hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, 
isVectorized);
+hiveConf.setBoolVar(HiveConf.ConfVars.HIVEOPTSORTDYNAMICPARTITION, isSdpo);
+runStatementOnDriver("drop table if exists acid_uap");
+runStatementOnDriver("create transactional table acid_uap(a int, b 
varchar(128)) " +
+"partitioned by (ds string) clustered by (a) into 2 buckets stored as 
orc TBLPROPERTIES " +
+"('bucketing_version'='" + bucketing_version + "')");
+runStatementOnDriver("insert into table acid_uap partition (ds='tomorrow') 
" +
+"values (1, 'bah'),(2, 'yah')");
+runStatementOnDriver("insert into table acid_uap partition (ds='today') " +
+"values 

[2/2] hive git commit: HIVE-20723: Allow per table specification of compaction yarn queue (Saurabh Seth via Eugene Koifman)

[ekoifman]

HIVE-20723: Allow per table specification of compaction yarn queue (Saurabh 
Seth via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/35278429
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/35278429
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/35278429

Branch: refs/heads/master
Commit: 35278429d9677b0878a4523ed7b03a5016f81e1d
Parents: 3c6a36b
Author: Saurabh Seth 
Authored: Sat Oct 13 16:46:16 2018 -0700
Committer: Eugene Koifman 
Committed: Sat Oct 13 16:46:16 2018 -0700

--
 .../hadoop/hive/ql/txn/compactor/TestCompactor.java  | 10 --
 .../apache/hadoop/hive/ql/txn/compactor/CompactorMR.java | 11 ++-
 2 files changed, 14 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/35278429/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
index cffa21a..a9d7468 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
@@ -1584,6 +1584,7 @@ public class TestCompactor {
*/
   @Test
   public void testTableProperties() throws Exception {
+conf.setVar(HiveConf.ConfVars.COMPACTOR_JOB_QUEUE, "root.user1");
 String tblName1 = "ttp1"; // plain acid table
 String tblName2 = "ttp2"; // acid table with customized tblproperties
 executeStatementOnDriver("drop table if exists " + tblName1, driver);
@@ -1596,7 +1597,8 @@ public class TestCompactor {
   "'transactional'='true'," +
   "'compactor.mapreduce.map.memory.mb'='2048'," + // 2048 MB memory for 
compaction map job
   "'compactorthreshold.hive.compactor.delta.num.threshold'='4'," +  // 
minor compaction if more than 4 delta dirs
-  "'compactorthreshold.hive.compactor.delta.pct.threshold'='0.47'" + // 
major compaction if more than 47%
+  "'compactorthreshold.hive.compactor.delta.pct.threshold'='0.47'," + // 
major compaction if more than 47%
+  "'compactor.hive.compactor.job.queue'='root.user2'" + // Override the 
system wide compactor queue for this table
   ")", driver);
 
 // Insert 5 rows to both tables
@@ -1641,6 +1643,7 @@ public class TestCompactor {
 t.run();
 JobConf job = t.getMrJob();
 Assert.assertEquals(2048, job.getMemoryForMapTask());  // 2048 comes from 
tblproperties
+Assert.assertEquals("root.user2", job.getQueueName()); // Queue name comes 
from table properties
 // Compact ttp1
 stop = new AtomicBoolean(true);
 t = new Worker();
@@ -1651,6 +1654,7 @@ public class TestCompactor {
 t.run();
 job = t.getMrJob();
 Assert.assertEquals(1024, job.getMemoryForMapTask());  // 1024 is the 
default value
+Assert.assertEquals("root.user1", job.getQueueName()); // The system wide 
compaction queue name
 // Clean up
 runCleaner(conf);
 rsp = txnHandler.showCompact(new ShowCompactRequest());
@@ -1702,7 +1706,8 @@ public class TestCompactor {
 executeStatementOnDriver("alter table " + tblName2 + " compact 'major'" +
   " with overwrite tblproperties (" +
   "'compactor.mapreduce.map.memory.mb'='3072'," +
-  "'tblprops.orc.compress.size'='3141')", driver);
+  "'tblprops.orc.compress.size'='3141'," +
+  "'compactor.hive.compactor.job.queue'='root.user2')", driver);
 
 rsp = txnHandler.showCompact(new ShowCompactRequest());
 Assert.assertEquals(4, rsp.getCompacts().size());
@@ -1722,6 +1727,7 @@ public class TestCompactor {
 job = t.getMrJob();
 Assert.assertEquals(3072, job.getMemoryForMapTask());
 
Assert.assertTrue(job.get("hive.compactor.table.props").contains("orc.compress.size4:3141"));
+Assert.assertEquals("root.user2", job.getQueueName());
 /*createReader(FileSystem fs, Path path) throws IOException {
  */
 //we just ran Major compaction so we should have a base_x in tblName2 that 
has the new files

http://git-wip-us.apache.org/repos/asf/hive/blob/35278429/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java 
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
index 611f85a..92c74e1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
@@ -150,11 +150,6 @@ public class 

hive git commit: HIVE-20694: Additional unit tests for VectorizedOrcAcidRowBatchReader min max key evaluation (Saurabh Seth via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 8b7043626 -> b054174bb


HIVE-20694: Additional unit tests for VectorizedOrcAcidRowBatchReader min max 
key evaluation (Saurabh Seth via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b054174b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b054174b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b054174b

Branch: refs/heads/master
Commit: b054174bb0eb8b692cafbb30194236fc75486e60
Parents: 8b70436
Author: Saurabh Seth 
Authored: Tue Oct 9 16:02:25 2018 -0700
Committer: Eugene Koifman 
Committed: Tue Oct 9 16:02:25 2018 -0700

--
 .../TestVectorizedOrcAcidRowBatchReader.java| 380 +++
 1 file changed, 380 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/b054174b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedOrcAcidRowBatchReader.java
--
diff --git 
a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedOrcAcidRowBatchReader.java
 
b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedOrcAcidRowBatchReader.java
index 0a499b1..0b26879 100644
--- 
a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedOrcAcidRowBatchReader.java
+++ 
b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedOrcAcidRowBatchReader.java
@@ -18,9 +18,11 @@
 package org.apache.hadoop.hive.ql.io.orc;
 
 import java.io.File;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
 
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.ValidWriteIdList;
@@ -43,9 +45,12 @@ import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
 import org.apache.hadoop.hive.ql.plan.MapWork;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Reporter;
+import org.apache.orc.OrcConf;
+import org.apache.orc.StripeInformation;
 import org.apache.orc.TypeDescription;
 import org.junit.Before;
 import org.junit.Test;
@@ -67,6 +72,8 @@ public class TestVectorizedOrcAcidRowBatchReader {
   private Path root;
   private ObjectInspector inspector;
   private ObjectInspector originalInspector;
+  private ObjectInspector bigRowInspector;
+  private ObjectInspector bigOriginalRowInspector;
 
   public static class DummyRow {
 LongWritable field;
@@ -110,6 +117,49 @@ public class TestVectorizedOrcAcidRowBatchReader {
 }
   }
 
+  /**
+   * A larger Dummy row that can be used to write multiple stripes.
+   */
+  public static class BigRow {
+BytesWritable field;
+RecordIdentifier rowId;
+
+BigRow(byte[] val) {
+  field = new BytesWritable(val);
+}
+
+BigRow(byte[] val, long rowId, long origTxn, int bucket) {
+  field = new BytesWritable(val);
+  bucket = BucketCodec.V1.encode(new 
AcidOutputFormat.Options(null).bucket(bucket));
+  this.rowId = new RecordIdentifier(origTxn, bucket, rowId);
+}
+
+static String getColumnNamesProperty() {
+  return "field";
+}
+static String getColumnTypesProperty() {
+  return "binary";
+}
+  }
+
+  /**
+   * A larger Dummy row for original files that can be used to write multiple 
stripes.
+   */
+  public static class BigOriginalRow {
+BytesWritable field;
+
+BigOriginalRow(byte[] val) {
+  field = new BytesWritable(val);
+}
+
+static String getColumnNamesProperty() {
+  return "field";
+}
+static String getColumnTypesProperty() {
+  return "binary";
+}
+  }
+
   @Before
   public void setup() throws Exception {
 conf = new JobConf();
@@ -122,6 +172,7 @@ public class TestVectorizedOrcAcidRowBatchReader {
 conf.set(IOConstants.SCHEMA_EVOLUTION_COLUMNS_TYPES, 
DummyRow.getColumnTypesProperty());
 conf.setBoolean(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED.varname, 
true);
 conf.set(HiveConf.ConfVars.HIVE_ORC_SPLIT_STRATEGY.varname, "BI");
+OrcConf.ROWS_BETWEEN_CHECKS.setLong(conf, 1);
 
 Path workDir = new Path(System.getProperty("test.tmp.dir",
 "target" + File.separator + "test" + File.separator + "tmp"));
@@ -135,6 +186,11 @@ public class TestVectorizedOrcAcidRowBatchReader {
 
   originalInspector = 
ObjectInspectorFactory.getReflectionObjectInspector(DummyOriginalRow.class,
   ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+
+  bigRowInspector = 
ObjectInspectorFactory.getReflectionObjectInspector(BigRow.class,
+  

hive git commit: HIVE-20635: VectorizedOrcAcidRowBatchReader doesn't filter delete events for original files (Saurabh Seth via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master f122e258b -> 39ed52c48


HIVE-20635: VectorizedOrcAcidRowBatchReader doesn't filter delete events for 
original files (Saurabh Seth via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/39ed52c4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/39ed52c4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/39ed52c4

Branch: refs/heads/master
Commit: 39ed52c48d6970e4ae83d423fe6cf5ced914a69c
Parents: f122e25
Author: Saurabh Seth 
Authored: Sat Oct 6 14:10:48 2018 -0700
Committer: Eugene Koifman 
Committed: Sat Oct 6 14:10:48 2018 -0700

--
 .../io/orc/VectorizedOrcAcidRowBatchReader.java |  83 -
 .../TestVectorizedOrcAcidRowBatchReader.java| 177 +++
 2 files changed, 252 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/39ed52c4/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
index 0cefeee..66280b2 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
@@ -228,6 +228,8 @@ public class VectorizedOrcAcidRowBatchReader
 LOG.info("Read ValidWriteIdList: " + this.validWriteIdList.toString()
 + ":" + orcSplit);
 
+this.syntheticProps = orcSplit.getSyntheticAcidProps();
+
 // Clone readerOptions for deleteEvents.
 Reader.Options deleteEventReaderOptions = readerOptions.clone();
 // Set the range on the deleteEventReaderOptions to 0 to INTEGER_MAX 
because
@@ -257,7 +259,6 @@ public class VectorizedOrcAcidRowBatchReader
 }
 rowIdProjected = areRowIdsProjected(rbCtx);
 rootPath = orcSplit.getRootDir();
-syntheticProps = orcSplit.getSyntheticAcidProps();
 
 /**
  * This could be optimized by moving dir type/write id based checks are
@@ -393,6 +394,13 @@ public class VectorizedOrcAcidRowBatchReader
   LOG.debug("findMinMaxKeys() " + ConfVars.FILTER_DELETE_EVENTS + 
"=false");
   return new OrcRawRecordMerger.KeyInterval(null, null);
 }
+
+//todo: since we already have OrcSplit.orcTail, should somehow use it to
+// get the acid.index, stats, etc rather than fetching the footer again
+// though it seems that orcTail is mostly null
+Reader reader = OrcFile.createReader(orcSplit.getPath(),
+OrcFile.readerOptions(conf));
+
 if(orcSplit.isOriginal()) {
   /**
* Among originals we may have files with _copy_N suffix.  To properly
@@ -403,14 +411,11 @@ public class VectorizedOrcAcidRowBatchReader
* Kind of chicken-and-egg - deal with this later.
* See {@link OrcRawRecordMerger#discoverOriginalKeyBounds(Reader, int,
* Reader.Options, Configuration, OrcRawRecordMerger.Options)}*/
-  LOG.debug("findMinMaxKeys(original split) - ignoring");
-  return new OrcRawRecordMerger.KeyInterval(null, null);
+  LOG.debug("findMinMaxKeys(original split)");
+
+  return findOriginalMinMaxKeys(orcSplit, reader, 
deleteEventReaderOptions);
 }
-//todo: since we already have OrcSplit.orcTail, should somehow use it to
-// get the acid.index, stats, etc rather than fetching the footer again
-// though it seems that orcTail is mostly null
-Reader reader = OrcFile.createReader(orcSplit.getPath(),
-OrcFile.readerOptions(conf));
+
 List stripes = reader.getStripes();
 final long splitStart = orcSplit.getStart();
 final long splitEnd = splitStart + orcSplit.getLength();
@@ -578,6 +583,68 @@ public class VectorizedOrcAcidRowBatchReader
 return keyInterval;
   }
 
+  private OrcRawRecordMerger.KeyInterval findOriginalMinMaxKeys(OrcSplit 
orcSplit, Reader reader,
+  Reader.Options deleteEventReaderOptions) {
+
+// This method returns the minimum and maximum synthetic row ids that are 
present in this split
+// because min and max keys are both inclusive when filtering out the 
delete delta records.
+
+if (syntheticProps == null) {
+  // syntheticProps containing the synthetic rowid offset is computed if 
there are delete delta files.
+  // If there aren't any delete delta files, then we don't need this 
anyway.
+  return new OrcRawRecordMerger.KeyInterval(null, null);
+}
+
+long splitStart = orcSplit.getStart();
+long splitEnd = orcSplit.getStart() + orcSplit.getLength();
+
+long minRowId = syntheticProps.getRowIdOffset();
+long maxRowId = 

hive git commit: HIVE-20664: Potential ArrayIndexOutOfBoundsException in VectorizedOrcAcidRowBatchReader.findMinMaxKeys (Saurabh Seth via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master f0434c5b5 -> 9593e46e6


HIVE-20664: Potential ArrayIndexOutOfBoundsException in 
VectorizedOrcAcidRowBatchReader.findMinMaxKeys (Saurabh Seth via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9593e46e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9593e46e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9593e46e

Branch: refs/heads/master
Commit: 9593e46e63aa8f41217a5eeafd557cc6c95e473e
Parents: f0434c5
Author: Saurabh Seth 
Authored: Fri Oct 5 08:35:47 2018 -0700
Committer: Eugene Koifman 
Committed: Fri Oct 5 08:35:47 2018 -0700

--
 .../io/orc/VectorizedOrcAcidRowBatchReader.java | 33 +---
 1 file changed, 29 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/9593e46e/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
index 1509bba..0cefeee 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
@@ -422,10 +422,7 @@ public class VectorizedOrcAcidRowBatchReader
   if(firstStripeIndex == -1 && stripe.getOffset() >= splitStart) {
 firstStripeIndex = i;
   }
-  if(lastStripeIndex == -1 && splitEnd <= stripeEnd &&
-  stripes.get(firstStripeIndex).getOffset() <= stripe.getOffset() ) {
-//the last condition is for when both splitStart and splitEnd are in
-// the same stripe
+  if(lastStripeIndex == -1 && splitEnd <= stripeEnd) {
 lastStripeIndex = i;
   }
 }
@@ -435,6 +432,34 @@ public class VectorizedOrcAcidRowBatchReader
   stripes.get(stripes.size() - 1).getLength() < splitEnd;
   lastStripeIndex = stripes.size() - 1;
 }
+
+if (firstStripeIndex > lastStripeIndex || firstStripeIndex == -1) {
+  /**
+   * If the firstStripeIndex was set after the lastStripeIndex the split 
lies entirely within a single stripe.
+   * In case the split lies entirely within the last stripe, the 
firstStripeIndex will never be found, hence the
+   * second condition.
+   * In this case, the reader for this split will not read any data.
+   * See {@link org.apache.orc.impl.RecordReaderImpl#RecordReaderImpl
+   * Create a KeyInterval such that no delete delta records are loaded 
into memory in the deleteEventRegistry.
+   */
+
+  long minRowId = 1;
+  long maxRowId = 0;
+  int minBucketProp = 1;
+  int maxBucketProp = 0;
+
+  OrcRawRecordMerger.KeyInterval keyIntervalTmp =
+  new OrcRawRecordMerger.KeyInterval(new RecordIdentifier(1, 
minBucketProp, minRowId),
+  new RecordIdentifier(0, maxBucketProp, maxRowId));
+
+  setSARG(keyIntervalTmp, deleteEventReaderOptions, minBucketProp, 
maxBucketProp,
+  minRowId, maxRowId);
+  LOG.info("findMinMaxKeys(): " + keyIntervalTmp +
+  " stripes(" + firstStripeIndex + "," + lastStripeIndex + ")");
+
+  return keyIntervalTmp;
+}
+
 if(firstStripeIndex == -1 || lastStripeIndex == -1) {
   //this should not happen but... if we don't know which stripe(s) are
   //involved we can't figure out min/max bounds

[1/2] hive git commit: HIVE-20556: Expose an API to retrieve the TBL_ID from TBLS in the metastore tables (Jaume Marhuenda via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 97f0513c4 -> fe3a457d7


http://git-wip-us.apache.org/repos/asf/hive/blob/fe3a457d/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
--
diff --git 
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
 
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index 4937d9d..d6f0d8c 100644
--- 
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ 
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -222,6 +222,9 @@ public abstract class TestHiveMetaStore {
 tbl = client.getTable(dbName, tblName);
   }
 
+  Assert.assertTrue(tbl.isSetId());
+  tbl.unsetId();
+
   Partition part = makePartitionObject(dbName, tblName, vals, tbl, 
"/part1");
   Partition part2 = makePartitionObject(dbName, tblName, vals2, tbl, 
"/part2");
   Partition part3 = makePartitionObject(dbName, tblName, vals3, tbl, 
"/part3");
@@ -1272,6 +1275,7 @@ public abstract class TestHiveMetaStore {
 
   Table tbl2 = client.getTable(dbName, tblName);
   assertNotNull(tbl2);
+  Assert.assertTrue(tbl2.isSetId());
   assertEquals(tbl2.getDbName(), dbName);
   assertEquals(tbl2.getTableName(), tblName);
   assertEquals(tbl2.getSd().getCols().size(), typ1.getFields().size());
@@ -1305,6 +1309,7 @@ public abstract class TestHiveMetaStore {
 assertTrue(fieldSchemasFull.contains(fs));
   }
 
+  tbl2.unsetId();
   client.createTable(tbl2);
   if (isThriftClient) {
 tbl2 = client.getTable(tbl2.getDbName(), tbl2.getTableName());
@@ -1664,6 +1669,56 @@ public abstract class TestHiveMetaStore {
   }
 
   @Test
+  public void testCreateAndGetTableWithDriver() throws Exception {
+String dbName = "createDb";
+String tblName = "createTbl";
+
+client.dropTable(dbName, tblName);
+silentDropDatabase(dbName);
+new DatabaseBuilder()
+.setName(dbName)
+.create(client, conf);
+
+createTable(dbName, tblName);
+Table tblRead = client.getTable(dbName, tblName);
+Assert.assertTrue(tblRead.isSetId());
+long firstTableId = tblRead.getId();
+
+createTable(dbName, tblName + "_2");
+Table tblRead2 = client.getTable(dbName, tblName + "_2");
+Assert.assertTrue(tblRead2.isSetId());
+Assert.assertNotEquals(firstTableId, tblRead2.getId());
+  }
+
+  @Test
+  public void testCreateTableSettingId() throws Exception {
+String dbName = "createDb";
+String tblName = "createTbl";
+
+client.dropTable(dbName, tblName);
+silentDropDatabase(dbName);
+new DatabaseBuilder()
+.setName(dbName)
+.create(client, conf);
+
+Table table = new TableBuilder()
+.setDbName(dbName)
+.setTableName(tblName)
+.addCol("foo", "string")
+.addCol("bar", "string")
+.build(conf);
+table.setId(1);
+try {
+  client.createTable(table);
+  Assert.fail("An error should happen when setting the id"
+  + " to create a table");
+} catch (InvalidObjectException e) {
+  Assert.assertTrue(e.getMessage().contains("Id shouldn't be set"));
+  Assert.assertTrue(e.getMessage().contains(tblName));
+}
+  }
+
+  @Test
   public void testAlterTable() throws Exception {
 String dbName = "alterdb";
 String invTblName = "alter-tbl";

http://git-wip-us.apache.org/repos/asf/hive/blob/fe3a457d/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java
--
diff --git 
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java
 
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java
index df83171..ebbd1c7 100644
--- 
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java
+++ 
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java
@@ -222,6 +222,8 @@ public class TestHiveMetaStorePartitionSpecs {
   clearAndRecreateDB(hmsc);
   createTable(hmsc, true);
   Table table = hmsc.getTable(dbName, tableName);
+  Assert.assertTrue(table.isSetId());
+  table.unsetId();
   populatePartitions(hmsc, table, Arrays.asList("isLocatedInTablePath", 
"isLocatedOutsideTablePath"));
 
   // Clone the table,

[2/2] hive git commit: HIVE-20556: Expose an API to retrieve the TBL_ID from TBLS in the metastore tables (Jaume Marhuenda via Eugene Koifman)

[ekoifman]

HIVE-20556: Expose an API to retrieve the TBL_ID from TBLS in the metastore 
tables (Jaume Marhuenda via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fe3a457d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fe3a457d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fe3a457d

Branch: refs/heads/master
Commit: fe3a457d7e6b8b9387b8fc0165d2e9efd9c6b364
Parents: 97f0513
Author: Jaume Marhuenda 
Authored: Thu Oct 4 14:39:04 2018 -0700
Committer: Eugene Koifman 
Committed: Thu Oct 4 14:39:04 2018 -0700

--
 data/files/exported_table/_metadata |   2 +-
 .../TestAuthorizationPreEventListener.java  |   3 +
 .../TestMetastoreAuthorizationProvider.java |   3 +
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |   1 +
 .../hadoop/hive/ql/metadata/TestHive.java   |   5 +
 .../ql/metadata/TestHiveMetaStoreChecker.java   |   5 +
 .../apache/hadoop/hive/metastore/api/Table.java | 357 ---
 .../src/gen/thrift/gen-php/metastore/Types.php  | 139 +---
 .../gen/thrift/gen-py/hive_metastore/ttypes.py  | 133 +++
 .../gen/thrift/gen-rb/hive_metastore_types.rb   |  46 +--
 .../src/main/thrift/hive_metastore.thrift   |  42 +--
 .../hadoop/hive/metastore/HiveMetaStore.java|   5 +
 .../hadoop/hive/metastore/ObjectStore.java  |   2 +
 .../hadoop/hive/metastore/model/MTable.java |  10 +-
 .../src/main/resources/package.jdo  |   8 +-
 .../hive/metastore/TestHiveMetaStore.java   |  55 +++
 .../TestHiveMetaStorePartitionSpecs.java|   2 +
 .../TestTablesCreateDropAlterTruncate.java  |  11 +
 18 files changed, 534 insertions(+), 295 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/fe3a457d/data/files/exported_table/_metadata
--
diff --git a/data/files/exported_table/_metadata 
b/data/files/exported_table/_metadata
index 81fbf63..8d8f8bb 100644
--- a/data/files/exported_table/_metadata
+++ b/data/files/exported_table/_metadata
@@ -1 +1 @@
-{"partitions":[],"table":"{\"1\":{\"str\":\"j1_41\"},\"2\":{\"str\":\"default\"},\"3\":{\"str\":\"johndee\"},\"4\":{\"i32\":1371900915},\"5\":{\"i32\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"lst\":[\"rec\",2,{\"1\":{\"str\":\"a\"},\"2\":{\"str\":\"string\"}},{\"1\":{\"str\":\"b\"},\"2\":{\"str\":\"int\"}}]},\"2\":{\"str\":\"hdfs://hivebase01:8020/user/hive/warehouse/j1_41\"},\"3\":{\"str\":\"org.apache.hadoop.mapred.TextInputFormat\"},\"4\":{\"str\":\"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat\"},\"5\":{\"tf\":0},\"6\":{\"i32\":-1},\"7\":{\"rec\":{\"2\":{\"str\":\"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe\"},\"3\":{\"map\":[\"str\",\"str\",2,{\"serialization.format\":\",\",\"field.delim\":\",\"}]}}},\"8\":{\"lst\":[\"str\",0]},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"rec\",0]},\"9\":{\"map\":[\"str\",\"str\",1,{\"transient_lastDdlTime\":\"1371900931\"}]},\"12\":{\"str\":\"MANAGED_TABLE\"}}","version":"0.1"
 }
\ No newline at end of file
+{"partitions":[],"table":"{\"2\":{\"str\":\"j1_41\"},\"3\":{\"str\":\"default\"},\"4\":{\"str\":\"johndee\"},\"5\":{\"i32\":1371900915},\"6\":{\"i32\":0},\"7\":{\"i32\":0},\"8\":{\"rec\":{\"1\":{\"lst\":[\"rec\",2,{\"1\":{\"str\":\"a\"},\"2\":{\"str\":\"string\"}},{\"1\":{\"str\":\"b\"},\"2\":{\"str\":\"int\"}}]},\"2\":{\"str\":\"hdfs://hivebase01:8020/user/hive/warehouse/j1_41\"},\"3\":{\"str\":\"org.apache.hadoop.mapred.TextInputFormat\"},\"4\":{\"str\":\"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat\"},\"5\":{\"tf\":0},\"6\":{\"i32\":-1},\"7\":{\"rec\":{\"2\":{\"str\":\"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe\"},\"3\":{\"map\":[\"str\",\"str\",2,{\"serialization.format\":\",\",\"field.delim\":\",\"}]}}},\"8\":{\"lst\":[\"str\",0]},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",0,{}]}}},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",1,{\"transient_lastDdlTime\":\"1371900931\"}]},\"13\":{\"str\":\"MANAGED_TABLE\"}}","version":"0.1
 "}

http://git-wip-us.apache.org/repos/asf/hive/blob/fe3a457d/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java
index 05c0009..1f6ec27 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java

hive git commit: HIVE-19985: ACID: Skip decoding the ROW__ID sections for read-only queries (Eugene Koifman, reviewed by Gopal V)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 05ddd21c7 -> e133ec5c2


HIVE-19985: ACID: Skip decoding the ROW__ID sections for read-only queries 
(Eugene Koifman, reviewed by Gopal V)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e133ec5c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e133ec5c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e133ec5c

Branch: refs/heads/master
Commit: e133ec5c28e0ed5082773f00b5bf4d55d2697db9
Parents: 05ddd21
Author: Eugene Koifman 
Authored: Sat Sep 29 09:56:44 2018 -0700
Committer: Eugene Koifman 
Committed: Sat Sep 29 09:56:44 2018 -0700

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |   3 +
 .../test/resources/testconfiguration.properties |   3 +-
 .../hive/llap/io/api/impl/LlapRecordReader.java |  61 +++--
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java |  47 +--
 .../hadoop/hive/ql/io/orc/OrcRecordUpdater.java |   2 +-
 .../io/orc/VectorizedOrcAcidRowBatchReader.java | 129 ++-
 .../apache/hadoop/hive/ql/TestTxnCommands3.java |  63 +
 .../clientpositive/acid_meta_columns_decode.q   |  24 
 .../llap/acid_meta_columns_decode.q.out | 128 ++
 9 files changed, 407 insertions(+), 53 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/e133ec5c/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 0cecae5..d1e6631 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2698,6 +2698,9 @@ public class HiveConf extends Configuration {
 MERGE_CARDINALITY_VIOLATION_CHECK("hive.merge.cardinality.check", true,
   "Set to true to ensure that each SQL Merge statement ensures that for 
each row in the target\n" +
 "table there is at most 1 matching row in the source table per SQL 
Specification."),
+OPTIMIZE_ACID_META_COLUMNS("hive.optimize.acid.meta.columns", true,
+"If true, don't decode Acid metadata columns from storage unless" +
+" they are needed."),
 
 // For Arrow SerDe
 HIVE_ARROW_ROOT_ALLOCATOR_LIMIT("hive.arrow.root.allocator.limit", 
Long.MAX_VALUE,

http://git-wip-us.apache.org/repos/asf/hive/blob/e133ec5c/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties 
b/itests/src/test/resources/testconfiguration.properties
index def3561..fdd8ecc 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -426,8 +426,9 @@ minillaplocal.query.files=\
   dec_str.q,\
   dp_counter_non_mm.q,\
   dp_counter_mm.q,\
-  acid_no_buckets.q, \
   acid_globallimit.q,\
+  acid_meta_columns_decode.q,\
+  acid_no_buckets.q, \
   acid_vectorization_missing_cols.q,\
   acid_vectorization_original.q,\
   alter_merge_stats_orc.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/e133ec5c/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java
--
diff --git 
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java
 
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java
index 3455d16..27a5b0f 100644
--- 
a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java
+++ 
b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java
@@ -160,7 +160,6 @@ class LlapRecordReader
 TypeDescription schema = OrcInputFormat.getDesiredRowTypeDescr(
 job, isAcidScan, Integer.MAX_VALUE);
 
-this.includes = new IncludesImpl(tableIncludedCols, isAcidScan, rbCtx, 
schema, job);
 
 int queueLimitBase = getQueueVar(ConfVars.LLAP_IO_VRB_QUEUE_LIMIT_BASE, 
job, daemonConf);
 int queueLimitMin =  getQueueVar(ConfVars.LLAP_IO_VRB_QUEUE_LIMIT_MIN, 
job, daemonConf);
@@ -184,6 +183,8 @@ class LlapRecordReader
   this.acidReader = new VectorizedOrcAcidRowBatchReader(
   (OrcSplit) split, jobConf, Reporter.NULL, null, rbCtx, true);
 }
+this.includes = new IncludesImpl(tableIncludedCols, isAcidScan, rbCtx,
+schema, job, isAcidScan && acidReader.includeAcidColumns());
 
 // Create the consumer of encoded data; it will coordinate decoding to 
CVBs.
 feedback = rp = cvp.createReadPipeline(this, split, includes, sarg, 
counters, includes,
@@ -341,19 +342,27 @@ class LlapRecordReader
   return false;
 }
 if 

hive git commit: HIVE-17917: VectorizedOrcAcidRowBatchReader.computeOffsetAndBucket optimization (Saurabh Seth via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 37fd22e6a -> 34331f3c7


HIVE-17917: VectorizedOrcAcidRowBatchReader.computeOffsetAndBucket optimization 
(Saurabh Seth via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/34331f3c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/34331f3c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/34331f3c

Branch: refs/heads/master
Commit: 34331f3c7b69200a0177f5446f1f15c8ed69ee86
Parents: 37fd22e
Author: Saurabh Seth 
Authored: Thu Sep 27 19:14:21 2018 -0700
Committer: Eugene Koifman 
Committed: Thu Sep 27 19:14:21 2018 -0700

--
 .../hadoop/hive/ql/io/orc/OrcInputFormat.java   |  23 ++-
 .../apache/hadoop/hive/ql/io/orc/OrcSplit.java  |  59 +++-
 .../io/orc/VectorizedOrcAcidRowBatchReader.java |  69 -
 .../hive/ql/io/orc/TestInputOutputFormat.java   |   6 +-
 .../acid_vectorization_original.q   |  29 +++-
 .../llap/acid_vectorization_original.q.out  | 146 +++
 6 files changed, 285 insertions(+), 47 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/34331f3c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index f34f393..728bf50 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -1036,6 +1036,7 @@ public class OrcInputFormat implements 
InputFormat,
 private final Path dir;
 private final boolean allowSyntheticFileIds;
 private final boolean isDefaultFs;
+private final Configuration conf;
 
 /**
  * @param dir - root of partition dir
@@ -1051,12 +1052,21 @@ public class OrcInputFormat implements 
InputFormat,
   this.dir = dir;
   this.allowSyntheticFileIds = allowSyntheticFileIds;
   this.isDefaultFs = isDefaultFs;
+  this.conf = context.conf;
 }
 
 @Override
 public List getSplits() throws IOException {
   List splits = Lists.newArrayList();
+  boolean isAcid = AcidUtils.isFullAcidScan(conf);
+  boolean vectorMode = Utilities.getIsVectorized(conf);
+  OrcSplit.OffsetAndBucketProperty offsetAndBucket = null;
   for (HdfsFileStatusWithId file : fileStatuses) {
+if (isOriginal && isAcid && vectorMode) {
+  offsetAndBucket = 
VectorizedOrcAcidRowBatchReader.computeOffsetAndBucket(file.getFileStatus(), 
dir,
+  isOriginal, !deltas.isEmpty(), conf);
+}
+
 FileStatus fileStatus = file.getFileStatus();
 long logicalLen = AcidUtils.getLogicalLength(fs, fileStatus);
 if (logicalLen != 0) {
@@ -1072,7 +1082,7 @@ public class OrcInputFormat implements 
InputFormat,
 }
 OrcSplit orcSplit = new OrcSplit(fileStatus.getPath(), fileKey, 
entry.getKey(),
 entry.getValue().getLength(), entry.getValue().getHosts(), 
null, isOriginal, true,
-deltas, -1, logicalLen, dir);
+deltas, -1, logicalLen, dir, offsetAndBucket);
 splits.add(orcSplit);
   }
 }
@@ -1352,6 +1362,7 @@ public class OrcInputFormat implements 
InputFormat,
 private SchemaEvolution evolution;
 //this is the root of the partition in which the 'file' is located
 private final Path rootDir;
+OrcSplit.OffsetAndBucketProperty offsetAndBucket = null;
 
 public SplitGenerator(SplitInfo splitInfo, UserGroupInformation ugi,
 boolean allowSyntheticFileIds, boolean isDefaultFs) throws IOException 
{
@@ -1480,7 +1491,7 @@ public class OrcInputFormat implements 
InputFormat,
 fileKey = new SyntheticFileId(file);
   }
   return new OrcSplit(file.getPath(), fileKey, offset, length, hosts,
-  orcTail, isOriginal, hasBase, deltas, scaledProjSize, fileLen, 
rootDir);
+  orcTail, isOriginal, hasBase, deltas, scaledProjSize, fileLen, 
rootDir, offsetAndBucket);
 }
 
 private static final class OffsetAndLength { // Java cruft; pair of long.
@@ -1519,6 +1530,14 @@ public class OrcInputFormat implements 
InputFormat,
 }
 
 private List callInternal() throws IOException {
+  boolean isAcid = AcidUtils.isFullAcidScan(context.conf);
+  boolean vectorMode = Utilities.getIsVectorized(context.conf);
+
+  if (isOriginal && isAcid && vectorMode) {
+offsetAndBucket = 
VectorizedOrcAcidRowBatchReader.computeOffsetAndBucket(file, rootDir, 
isOriginal,
+!deltas.isEmpty(), context.conf);
+  }
+
   // Figure out which stripes we need to read.
   if (ppdResult != null) {
 assert 

hive git commit: HIVE-16812: VectorizedOrcAcidRowBatchReader doesn't filter delete events (Eugene Koifman, reviewed by Sergey Shelukhin)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 1ab23e3b7 -> 37fd22e6a


HIVE-16812: VectorizedOrcAcidRowBatchReader doesn't filter delete events 
(Eugene Koifman, reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/37fd22e6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/37fd22e6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/37fd22e6

Branch: refs/heads/master
Commit: 37fd22e6a6a3c49b77fbdb1983d27b335e03129d
Parents: 1ab23e3
Author: Eugene Koifman 
Authored: Thu Sep 27 13:25:43 2018 -0700
Committer: Eugene Koifman 
Committed: Thu Sep 27 13:27:24 2018 -0700

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |   5 +-
 .../hadoop/hive/ql/exec/FileSinkOperator.java   |   2 +-
 .../hadoop/hive/ql/io/RecordIdentifier.java |  17 +-
 .../hive/ql/io/orc/OrcRawRecordMerger.java  |  33 +-
 .../io/orc/VectorizedOrcAcidRowBatchReader.java | 463 ---
 .../apache/hadoop/hive/ql/TestTxnCommands.java  |   9 +
 .../apache/hadoop/hive/ql/TestTxnCommands2.java |   5 +-
 .../apache/hadoop/hive/ql/TestTxnCommands3.java | 162 +++
 .../hadoop/hive/ql/TestTxnConcatenate.java  |  63 ---
 .../apache/hadoop/hive/ql/TestTxnLoadData.java  |   6 +
 .../apache/hadoop/hive/ql/TestTxnNoBuckets.java |   8 +
 .../hadoop/hive/ql/TxnCommandsBaseForTests.java |   5 +
 .../hive/ql/io/orc/TestInputOutputFormat.java   |   4 +-
 .../TestVectorizedOrcAcidRowBatchReader.java| 318 +++--
 14 files changed, 931 insertions(+), 169 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/37fd22e6/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 3b8e820..0cecae5 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1858,7 +1858,10 @@ public class HiveConf extends Configuration {
 + "into memory to optimize for performance. To prevent out-of-memory 
errors, this is a rough heuristic\n"
 + "that limits the total number of delete events that can be loaded 
into memory at once.\n"
 + "Roughly it has been set to 10 million delete events per bucket 
(~160 MB).\n"),
-
+FILTER_DELETE_EVENTS("hive.txn.filter.delete.events", true,
+"If true, VectorizedOrcAcidRowBatchReader will compute min/max " +
+"ROW__ID for the split and only load delete events in that 
range.\n"
+),
 HIVESAMPLERANDOMNUM("hive.sample.seednumber", 0,
 "A number used to percentage sampling. By changing this number, user 
will change the subsets of data sampled."),
 

http://git-wip-us.apache.org/repos/asf/hive/blob/37fd22e6/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index d82faf3..8c7a78b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -983,7 +983,7 @@ public class FileSinkOperator extends 
TerminalOperator implements
   int writerOffset;
   // This if/else chain looks ugly in the inner loop, but given that it 
will be 100% the same
   // for a given operator branch prediction should work quite nicely on it.
-  // RecordUpdateer expects to get the actual row, not a serialized 
version of it.  Thus we
+  // RecordUpdater expects to get the actual row, not a serialized version 
of it.  Thus we
   // pass the row rather than recordValue.
   if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID || 
conf.isMmTable()) {
 rowOutWriters[findWriterOffset(row)].write(recordValue);

http://git-wip-us.apache.org/repos/asf/hive/blob/37fd22e6/ql/src/java/org/apache/hadoop/hive/ql/io/RecordIdentifier.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/RecordIdentifier.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/RecordIdentifier.java
index ea7ba53..e38e21a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/RecordIdentifier.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/RecordIdentifier.java
@@ -176,6 +176,7 @@ public class RecordIdentifier implements 
WritableComparable {
   @Override
   public int compareTo(RecordIdentifier other) {
 if (other.getClass() != RecordIdentifier.class) {
+  //WTF?  assumes that other instanceof OrcRawRecordMerger.ReaderKey???
   return 

hive git commit: HIVE-20640: Upgrade Hive to use ORC 1.5.3(Eugene Koifman, reviewed by Gopal V)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master fb7291ac4 -> 00dc4c72e


HIVE-20640: Upgrade Hive to use ORC 1.5.3(Eugene Koifman, reviewed by Gopal V)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/00dc4c72
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/00dc4c72
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/00dc4c72

Branch: refs/heads/master
Commit: 00dc4c72e745c4a0deb9e82cfef2c795ed19541b
Parents: fb7291a
Author: Eugene Koifman 
Authored: Thu Sep 27 11:45:22 2018 -0700
Committer: Eugene Koifman 
Committed: Thu Sep 27 11:45:22 2018 -0700

--
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/00dc4c72/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 5e100fd..5008923 100644
--- a/pom.xml
+++ b/pom.xml
@@ -186,7 +186,7 @@
 0.9.3
 2.10.0
 2.3
-1.5.2
+1.5.3
 1.10.19
 2.0.0-M5
 4.1.17.Final

hive git commit: HIVE-20604: Minor compaction disables ORC column stats (Eugene Koifman, reviewed by Prasanth Jayachandran)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master bef6c9fd2 -> dfc926284


HIVE-20604: Minor compaction disables ORC column stats (Eugene Koifman, 
reviewed by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dfc92628
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dfc92628
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dfc92628

Branch: refs/heads/master
Commit: dfc9262842c7405543a7f92efb18a6eead92414e
Parents: bef6c9f
Author: Eugene Koifman 
Authored: Wed Sep 26 11:31:14 2018 -0700
Committer: Eugene Koifman 
Committed: Wed Sep 26 11:31:14 2018 -0700

--
 .../org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java   | 9 ++---
 .../apache/hadoop/hive/metastore/conf/MetastoreConf.java| 4 
 2 files changed, 10 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/dfc92628/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
index 835f0e8..c4c56f8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
@@ -24,6 +24,7 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Properties;
 
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.orc.CompressionKind;
 import org.apache.orc.TypeDescription;
 import org.slf4j.Logger;
@@ -288,9 +289,11 @@ public class OrcOutputFormat extends 
FileOutputFormat
 if (!options.isWritingBase()) {
   opts.bufferSize(OrcRecordUpdater.DELTA_BUFFER_SIZE)
   .stripeSize(OrcRecordUpdater.DELTA_STRIPE_SIZE)
-  .blockPadding(false)
-  .compress(CompressionKind.NONE)
-  .rowIndexStride(0);
+  .blockPadding(false);
+  if(!MetastoreConf.getBoolVar(options.getConfiguration(),
+  MetastoreConf.ConfVars.COMPACTOR_MINOR_STATS_COMPRESSION)) {
+opts.compress(CompressionKind.NONE).rowIndexStride(0);
+  }
 }
 final OrcRecordUpdater.KeyIndexBuilder watcher =
 new OrcRecordUpdater.KeyIndexBuilder("compactor");

http://git-wip-us.apache.org/repos/asf/hive/blob/dfc92628/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
--
diff --git 
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
 
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 30ea7f8..946f644 100644
--- 
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ 
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -392,6 +392,10 @@ public class MetastoreConf {
 "tables or partitions to be compacted once they are determined to 
need compaction.\n" +
 "It will also increase the background load on the Hadoop cluster 
as more MapReduce jobs\n" +
 "will be running in the background."),
+COMPACTOR_MINOR_STATS_COMPRESSION(
+"metastore.compactor.enable.stats.compression",
+"metastore.compactor.enable.stats.compression", true,
+"Can be used to disable compression and ORC indexes for files produced 
by minor compaction."),
 CONNECTION_DRIVER("javax.jdo.option.ConnectionDriverName",
 "javax.jdo.option.ConnectionDriverName", 
"org.apache.derby.jdbc.EmbeddedDriver",
 "Driver class name for a JDBC metastore"),

hive git commit: HIVE-18453: ACID: Add "CREATE TRANSACTIONAL TABLE" syntax to unify ACID ORC & Parquet support (Igor Kryvenko via Eugene Koifman

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 8ebde0441 -> 9c907769a


HIVE-18453: ACID: Add "CREATE TRANSACTIONAL TABLE" syntax to unify ACID ORC & 
Parquet support (Igor Kryvenko via Eugene Koifman


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9c907769
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9c907769
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9c907769

Branch: refs/heads/master
Commit: 9c907769a63a6b23c91fdf0b3f3d0aa6387035dc
Parents: 8ebde04
Author: Igor Kryvenko 
Authored: Tue Sep 18 19:16:38 2018 -0700
Committer: Eugene Koifman 
Committed: Tue Sep 18 19:16:38 2018 -0700

--
 .../org/apache/hadoop/hive/ql/parse/HiveLexer.g |   1 +
 .../apache/hadoop/hive/ql/parse/HiveParser.g|   5 +-
 .../hadoop/hive/ql/parse/IdentifiersParser.g|   2 +-
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  46 -
 .../create_external_transactional.q |   5 +
 .../create_transactional_full_acid.q|  28 +++
 .../create_transactional_insert_only.q  |  13 ++
 .../create_external_transactional.q.out |   1 +
 .../create_transactional_full_acid.q.out| 197 +++
 .../create_transactional_insert_only.q.out  |  75 +++
 10 files changed, 362 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/9c907769/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
index 3caa51f..8bf9cc0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
@@ -304,6 +304,7 @@ KW_PRINCIPALS: 'PRINCIPALS';
 KW_COMPACT: 'COMPACT';
 KW_COMPACTIONS: 'COMPACTIONS';
 KW_TRANSACTIONS: 'TRANSACTIONS';
+KW_TRANSACTIONAL: 'TRANSACTIONAL';
 KW_REWRITE : 'REWRITE';
 KW_AUTHORIZATION: 'AUTHORIZATION';
 KW_REOPTIMIZATION: 'REOPTIMIZATION';

http://git-wip-us.apache.org/repos/asf/hive/blob/9c907769/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
index 48f7303..78bc87c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
@@ -618,6 +618,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
 xlateMap.put("KW_SCHEDULING_POLICY", "SCHEDULING_POLICY");
 xlateMap.put("KW_PATH", "PATH");
 xlateMap.put("KW_AST", "AST");
+xlateMap.put("KW_TRANSACTIONAL", "TRANSACTIONAL");
 
 // Operators
 xlateMap.put("DOT", ".");
@@ -1091,7 +1092,7 @@ databaseComment
 createTableStatement
 @init { pushMsg("create table statement", state); }
 @after { popMsg(state); }
-: KW_CREATE (temp=KW_TEMPORARY)? (ext=KW_EXTERNAL)? KW_TABLE ifNotExists? 
name=tableName
+: KW_CREATE (temp=KW_TEMPORARY)? (trans=KW_TRANSACTIONAL)? 
(ext=KW_EXTERNAL)? KW_TABLE ifNotExists? name=tableName
   (  like=KW_LIKE likeName=tableName
  tableRowFormat?
  tableFileFormat?
@@ -1108,7 +1109,7 @@ createTableStatement
  tablePropertiesPrefixed?
  (KW_AS selectStatementWithCTE)?
   )
--> ^(TOK_CREATETABLE $name $temp? $ext? ifNotExists?
+-> ^(TOK_CREATETABLE $name $temp? $trans? $ext? ifNotExists?
  ^(TOK_LIKETABLE $likeName?)
  columnNameTypeOrConstraintList?
  tableComment?

http://git-wip-us.apache.org/repos/asf/hive/blob/9c907769/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
index f9c97e0..fa033d7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
@@ -809,7 +809,7 @@ nonReserved
 | KW_ROLE | KW_ROLES | KW_SCHEMA | KW_SCHEMAS | KW_SECOND | KW_SEMI | 
KW_SERDE | KW_SERDEPROPERTIES | KW_SERVER | KW_SETS | KW_SHARED
 | KW_SHOW | KW_SHOW_DATABASE | KW_SKEWED | KW_SORT | KW_SORTED | KW_SSL | 
KW_STATISTICS | KW_STORED | KW_AST
 | KW_STREAMTABLE | KW_STRING | KW_STRUCT | KW_TABLES | KW_TBLPROPERTIES | 
KW_TEMPORARY | KW_TERMINATED
-| KW_TINYINT | KW_TOUCH | KW_TRANSACTIONS | KW_UNARCHIVE | KW_UNDO | 
KW_UNIONTYPE | KW_UNLOCK | KW_UNSET
+| KW_TINYINT | KW_TOUCH | KW_TRANSACTIONAL | KW_TRANSACTIONS | 
KW_UNARCHIVE | KW_UNDO | KW_UNIONTYPE | KW_UNLOCK | KW_UNSET
 | KW_UNSIGNED | KW_URI 

hive git commit: HIVE-20553: more acid stats tests (Eugene Koifman, reviewed by Sergey Shelukhin)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 1b459beab -> 5eaf0ddf9


HIVE-20553: more acid stats tests (Eugene Koifman, reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5eaf0ddf
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5eaf0ddf
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5eaf0ddf

Branch: refs/heads/master
Commit: 5eaf0ddf98d9320a54916962e1f04c83f3f6f13e
Parents: 1b459be
Author: Eugene Koifman 
Authored: Fri Sep 14 11:29:21 2018 -0700
Committer: Eugene Koifman 
Committed: Fri Sep 14 11:29:21 2018 -0700

--
 .../test/queries/clientpositive/acid_stats2.q   |  29 +-
 .../results/clientpositive/acid_stats2.q.out| 512 +++
 2 files changed, 540 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/5eaf0ddf/ql/src/test/queries/clientpositive/acid_stats2.q
--
diff --git a/ql/src/test/queries/clientpositive/acid_stats2.q 
b/ql/src/test/queries/clientpositive/acid_stats2.q
index cf96731..8c40fa7 100644
--- a/ql/src/test/queries/clientpositive/acid_stats2.q
+++ b/ql/src/test/queries/clientpositive/acid_stats2.q
@@ -16,6 +16,8 @@ set hive.support.concurrency=true;
 set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
 set hive.query.results.cache.enabled=false;
 
+set hive.exec.dynamic.partition.mode=nonstrict;
+
 create table stats3(key int,value string) stored as orc tblproperties 
("transactional"="true");
 insert into table stats3  values (1, "foo");
 explain select count(*) from stats3;
@@ -39,4 +41,29 @@ delete from stats3 where key = 2;
 explain select count(*) from stats3;
 select count(*) from stats3;
 
-drop table stats3;
\ No newline at end of file
+drop table stats3;
+
+create table stats4(key int,value string) partitioned by (ds string) clustered 
by (value) into 2 buckets stored as orc tblproperties ("transactional"="true");
+insert into table stats4 partition (ds) values (12341234, 'bob', 
'today'),(123471234871239847, 'bob', 'today'),(431, 'tracy', 'tomorrow');
+desc formatted stats4;
+desc formatted stats4 partition(ds='tomorrow');
+desc formatted stats4 partition(ds='today');
+explain select count(*) from stats4;
+select count(*) from stats4;
+delete from stats4 where value = 'tracy' and ds = 'tomorrow';
+desc formatted stats4;
+desc formatted stats4 partition(ds='tomorrow');
+desc formatted stats4 partition(ds='today');
+explain select count(*) from stats4;
+select count(*) from stats4;
+explain select count(*) from stats4 where ds = 'tomorrow';
+select count(*) from stats4 where ds = 'tomorrow';
+delete from stats4 where key > 12341234 and ds = 'today';
+desc formatted stats4;
+desc formatted stats4 partition(ds='tomorrow');
+desc formatted stats4 partition(ds='today');
+explain select count(*) from stats4;
+select count(*) from stats4;
+explain select count(*) from stats4 where ds = 'tomorrow';
+select count(*) from stats4 where ds = 'tomorrow';
+drop table stats4;

http://git-wip-us.apache.org/repos/asf/hive/blob/5eaf0ddf/ql/src/test/results/clientpositive/acid_stats2.q.out
--
diff --git a/ql/src/test/results/clientpositive/acid_stats2.q.out 
b/ql/src/test/results/clientpositive/acid_stats2.q.out
index 5fc0505..ccd8999 100644
--- a/ql/src/test/results/clientpositive/acid_stats2.q.out
+++ b/ql/src/test/results/clientpositive/acid_stats2.q.out
@@ -235,3 +235,515 @@ POSTHOOK: query: drop table stats3
 POSTHOOK: type: DROPTABLE
 POSTHOOK: Input: default@stats3
 POSTHOOK: Output: default@stats3
+PREHOOK: query: create table stats4(key int,value string) partitioned by (ds 
string) clustered by (value) into 2 buckets stored as orc tblproperties 
("transactional"="true")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@stats4
+POSTHOOK: query: create table stats4(key int,value string) partitioned by (ds 
string) clustered by (value) into 2 buckets stored as orc tblproperties 
("transactional"="true")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@stats4
+PREHOOK: query: insert into table stats4 partition (ds) values (12341234, 
'bob', 'today'),(123471234871239847, 'bob', 'today'),(431, 'tracy', 'tomorrow')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@stats4
+POSTHOOK: query: insert into table stats4 partition (ds) values (12341234, 
'bob', 'today'),(123471234871239847, 'bob', 'today'),(431, 'tracy', 'tomorrow')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@stats4@ds=today
+POSTHOOK: Output: default@stats4@ds=tomorrow
+POSTHOOK: Lineage: stats4 

hive git commit: HIVE-17921 Aggregation with struct in LLAP produces wrong result (Saurabh Seth via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 346c0ce44 -> ad93cfd75


HIVE-17921 Aggregation with struct in LLAP produces wrong result (Saurabh Seth 
via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ad93cfd7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ad93cfd7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ad93cfd7

Branch: refs/heads/master
Commit: ad93cfd753276ade6ec346470a83b7b05c60339b
Parents: 346c0ce
Author: Eugene Koifman 
Authored: Mon Sep 10 13:38:00 2018 -0700
Committer: Eugene Koifman 
Committed: Mon Sep 10 13:38:27 2018 -0700

--
 ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java| 3 ++-
 .../results/clientpositive/llap/acid_vectorization_original.q.out | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/ad93cfd7/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
index 64428f0..bce7977 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.io.AcidInputFormat;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.io.ColumnarSplit;
@@ -243,7 +244,7 @@ public class OrcSplit extends FileSplit implements 
ColumnarSplit, LlapAwareSplit
   public boolean canUseLlapIo(Configuration conf) {
 final boolean hasDelta = deltas != null && !deltas.isEmpty();
 final boolean isAcidRead = AcidUtils.isFullAcidScan(conf);
-final boolean isVectorized = HiveConf.getBoolVar(conf, 
ConfVars.HIVE_VECTORIZATION_ENABLED);
+final boolean isVectorized = Utilities.getIsVectorized(conf);
 Boolean isSplitUpdate = null;
 if (isAcidRead) {
   final AcidUtils.AcidOperationalProperties acidOperationalProperties

http://git-wip-us.apache.org/repos/asf/hive/blob/ad93cfd7/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out 
b/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out
index 57ff575..00be86c 100644
--- a/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out
+++ b/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out
@@ -718,7 +718,6 @@ POSTHOOK: query: select ROW__ID, count(*) from 
over10k_orc_bucketed group by ROW
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@over10k_orc_bucketed
  A masked pattern was here 
-NULL   6
 PREHOOK: query: select ROW__ID, * from over10k_orc_bucketed where ROW__ID is 
null
 PREHOOK: type: QUERY
 PREHOOK: Input: default@over10k_orc_bucketed

hive git commit: HIVE-20410: aborted Insert Overwrite on transactional table causes "Not enough history available for..." error (Eugene Koifman, reviewed by Sergey Shelukhin)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master b43b4d2a2 -> 468907eab


HIVE-20410: aborted Insert Overwrite on transactional table causes "Not enough 
history available for..." error (Eugene Koifman, reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/468907ea
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/468907ea
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/468907ea

Branch: refs/heads/master
Commit: 468907eab36f78df3e14a24005153c9a23d62555
Parents: b43b4d2
Author: Eugene Koifman 
Authored: Fri Aug 17 17:08:33 2018 -0700
Committer: Eugene Koifman 
Committed: Fri Aug 17 17:08:33 2018 -0700

--
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java | 11 -
 .../apache/hadoop/hive/ql/TestTxnCommands.java  |  2 +-
 .../mm_insert_overwrite_aborted.q   | 20 
 .../mm_insert_overwrite_aborted.q.out   | 50 
 4 files changed, 80 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/468907ea/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index cd47a63..4d71eb4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -1127,12 +1127,19 @@ public class AcidUtils {
   }
 }
 
-if(bestBase.oldestBase != null && bestBase.status == null) {
+if(bestBase.oldestBase != null && bestBase.status == null &&
+MetaDataFile.isCompacted(bestBase.oldestBase, fs)) {
   /**
* If here, it means there was a base_x (> 1 perhaps) but none were 
suitable for given
* {@link writeIdList}.  Note that 'original' files are logically a 
base_Long.MIN_VALUE and thus
* cannot have any data for an open txn.  We could check {@link deltas} 
has files to cover
-   * [1,n] w/o gaps but this would almost never happen...*/
+   * [1,n] w/o gaps but this would almost never happen...
+   *
+   * We only throw for base_x produced by Compactor since that base erases 
all history and
+   * cannot be used for a client that has a snapshot in which something 
inside this base is
+   * open.  (Nor can we ignore this base of course)  But base_x which is a 
result of IOW,
+   * contains all history so we treat it just like delta wrt visibility.  
Imagine, IOW which
+   * aborts. It creates a base_x, which can and should just be ignored.*/
   long[] exceptions = writeIdList.getInvalidWriteIds();
   String minOpenWriteId = exceptions != null && exceptions.length > 0 ?
 Long.toString(exceptions[0]) : "x";

http://git-wip-us.apache.org/repos/asf/hive/blob/468907ea/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index 536281d..9a4322d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@ -108,7 +108,7 @@ public class TestTxnCommands extends 
TxnCommandsBaseForTests {
 Assert.assertEquals(1, rs.size());
 Assert.assertEquals("1", rs.get(0));
 hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, true);
-runStatementOnDriver("insert into " + Table.ACIDTBL + " values(3,2)");
+runStatementOnDriver("insert overwrite table " + Table.ACIDTBL + " 
values(3,2)");
 hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, false);
 runStatementOnDriver("insert into " + Table.ACIDTBL + " values(5,6)");
 rs = runStatementOnDriver("select a from " + Table.ACIDTBL + " order by 
a");

http://git-wip-us.apache.org/repos/asf/hive/blob/468907ea/ql/src/test/queries/clientpositive/mm_insert_overwrite_aborted.q
--
diff --git a/ql/src/test/queries/clientpositive/mm_insert_overwrite_aborted.q 
b/ql/src/test/queries/clientpositive/mm_insert_overwrite_aborted.q
new file mode 100644
index 000..938e1f4
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/mm_insert_overwrite_aborted.q
@@ -0,0 +1,20 @@
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.exec.dynamic.partition=true;
+set hive.vectorized.execution.enabled=true;
+set hive.create.as.insert.only=true;
+
+drop table if exists studentparttab30k;
+create table studentparttab30k (name string) row format delimited 

[1/3] hive git commit: HIVE-19800 Create separate submodules for pre and post upgrade and add rename file logic (Eugene Koifman, reviewed by Jason Dere)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 6fd4d64db -> 28b24dbf5


http://git-wip-us.apache.org/repos/asf/hive/blob/28b24dbf/upgrade-acid/src/main/java/org/apache/hadoop/hive/upgrade/acid/UpgradeTool.java
--
diff --git 
a/upgrade-acid/src/main/java/org/apache/hadoop/hive/upgrade/acid/UpgradeTool.java
 
b/upgrade-acid/src/main/java/org/apache/hadoop/hive/upgrade/acid/UpgradeTool.java
deleted file mode 100644
index 78c0843..000
--- 
a/upgrade-acid/src/main/java/org/apache/hadoop/hive/upgrade/acid/UpgradeTool.java
+++ /dev/null
@@ -1,808 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.upgrade.acid;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.fs.ContentSummary;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.hive.common.ValidTxnList;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.TableType;
-import org.apache.hadoop.hive.metastore.Warehouse;
-import org.apache.hadoop.hive.metastore.api.CompactionResponse;
-import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
-import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
-import org.apache.hadoop.hive.metastore.txn.TxnStore;
-import org.apache.hadoop.hive.metastore.txn.TxnUtils;
-import org.apache.hadoop.hive.ql.io.AcidUtils;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.shims.HadoopShims;
-import org.apache.hive.common.util.HiveVersionInfo;
-import org.apache.orc.OrcFile;
-import org.apache.orc.Reader;
-import org.apache.orc.impl.AcidStats;
-import org.apache.orc.impl.OrcAcidUtils;
-import org.apache.thrift.TException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import static 
org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.escapeSQLString;
-
-/**
- * This utility is designed to help with upgrading to Hive 3.0.  On-disk 
layout for transactional
- * tables has changed in 3.0 and require pre-processing before upgrade to 
ensure they are readable
- * by Hive 3.0.  Some transactional tables (identified by this utility) 
require Major compaction
- * to be run on them before upgrading to 3.0.  Once this compaction starts, no 
more
- * update/delete/merge statements may be executed on these tables until 
upgrade is finished.
- *
- * Additionally, a new type of transactional tables was added in 3.0 - 
insert-only tables.  These
- * tables support ACID semantics and work with any Input/OutputFormat.  Any 
Managed tables may
- * be made insert-only transactional table. These tables don't support 
Update/Delete/Merge commands.
- *
- * This utility works in 2 modes: preUpgrade and postUpgrade.
- * In preUpgrade mode it has to have 2.x Hive jars on the classpath.  It will 
perform analysis on
- * existing transactional tables, determine which require compaction and 
generate a set of SQL
- * commands to launch all of these compactions.
- *
- 

[3/3] hive git commit: HIVE-19800 Create separate submodules for pre and post upgrade and add rename file logic (Eugene Koifman, reviewed by Jason Dere)

[ekoifman]

HIVE-19800 Create separate submodules for pre and post upgrade and add rename 
file logic (Eugene Koifman, reviewed by Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/28b24dbf
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/28b24dbf
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/28b24dbf

Branch: refs/heads/master
Commit: 28b24dbf5ee6f47fbde501f2c9ba6796159b03f6
Parents: 6fd4d64
Author: Eugene Koifman 
Authored: Sat Aug 11 11:01:06 2018 -0700
Committer: Eugene Koifman 
Committed: Sat Aug 11 11:03:03 2018 -0700

--
 packaging/pom.xml   |   5 -
 pom.xml |   5 -
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java |  44 +-
 .../apache/hadoop/hive/ql/io/BucketCodec.java   |   2 +-
 .../apache/hadoop/hive/ql/util/UpgradeTool.java | 677 
 .../hadoop/hive/ql/TestTxnAddPartition.java |   2 +-
 .../apache/hadoop/hive/ql/TestTxnCommands.java  |   2 +-
 .../hive/ql/TestTxnCommandsForMmTable.java  |   2 +-
 .../hadoop/hive/ql/TestTxnConcatenate.java  |   2 +-
 .../org/apache/hadoop/hive/ql/TestTxnExIm.java  |   2 +-
 .../apache/hadoop/hive/ql/TestTxnLoadData.java  |   2 +-
 .../apache/hadoop/hive/ql/TestTxnNoBuckets.java |   2 +-
 .../hadoop/hive/ql/TxnCommandsBaseForTests.java |  12 +-
 .../hadoop/hive/ql/util/TestUpgradeTool.java| 248 ++
 .../TransactionalValidationListener.java|   5 +-
 .../hadoop/hive/metastore/utils/FileUtils.java  |  60 ++
 upgrade-acid/pom.xml| 255 +-
 upgrade-acid/pre-upgrade/pom.xml| 284 +++
 .../hive/upgrade/acid/PreUpgradeTool.java   | 646 +++
 .../hive/upgrade/acid/TestPreUpgradeTool.java   | 237 ++
 .../hadoop/hive/upgrade/acid/UpgradeTool.java   | 808 ---
 .../hive/upgrade/acid/TestUpgradeTool.java  | 291 ---
 22 files changed, 2221 insertions(+), 1372 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/28b24dbf/packaging/pom.xml
--
diff --git a/packaging/pom.xml b/packaging/pom.xml
index cae1fa9..5c859ac 100644
--- a/packaging/pom.xml
+++ b/packaging/pom.xml
@@ -283,11 +283,6 @@
   ${project.version}
 
 
-  org.apache.hive
-  hive-upgrade-acid
-  ${project.version}
-
-
   org.apache.hive.hcatalog
   hive-webhcat
   ${project.version}

http://git-wip-us.apache.org/repos/asf/hive/blob/28b24dbf/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 908b68c..7503cff 100644
--- a/pom.xml
+++ b/pom.xml
@@ -997,11 +997,6 @@
   ${slf4j.version}
 
 
-  org.apache.hive
-  hive-upgrade-acid
-  ${project.version}
-
-
   org.mockito
   mockito-all
   test

http://git-wip-us.apache.org/repos/asf/hive/blob/28b24dbf/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index 7818efb..cd47a63 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -296,7 +296,49 @@ public class AcidUtils {
 }
 return createBucketFile(new Path(directory, subdir), 
options.getBucketId());
   }
-
+  /**
+   * Represents bucketId and copy_N suffix
+   */
+  public static final class BucketMetaData {
+private static final BucketMetaData INVALID = new BucketMetaData(-1, 0);
+/**
+ * @param bucketFileName {@link #ORIGINAL_PATTERN} or {@link 
#ORIGINAL_PATTERN_COPY}
+ */
+public static BucketMetaData parse(String bucketFileName) {
+  if (ORIGINAL_PATTERN.matcher(bucketFileName).matches()) {
+int bucketId = Integer
+.parseInt(bucketFileName.substring(0, 
bucketFileName.indexOf('_')));
+return new BucketMetaData(bucketId, 0);
+  }
+  else if(ORIGINAL_PATTERN_COPY.matcher(bucketFileName).matches()) {
+int copyNumber = Integer.parseInt(
+bucketFileName.substring(bucketFileName.lastIndexOf('_') + 1));
+int bucketId = Integer
+.parseInt(bucketFileName.substring(0, 
bucketFileName.indexOf('_')));
+return new BucketMetaData(bucketId, copyNumber);
+  }
+  else if (bucketFileName.startsWith(BUCKET_PREFIX)) {
+return new BucketMetaData(Integer
+.parseInt(bucketFileName.substring(bucketFileName.indexOf('_') + 
1)), 0);
+  }
+  return INVALID;
+}
+public static BucketMetaData parse(Path bucketFile) {
+  return parse(bucketFile.getName());
+}
+  

[2/3] hive git commit: HIVE-19800 Create separate submodules for pre and post upgrade and add rename file logic (Eugene Koifman, reviewed by Jason Dere)

[ekoifman]

http://git-wip-us.apache.org/repos/asf/hive/blob/28b24dbf/upgrade-acid/pre-upgrade/pom.xml
--
diff --git a/upgrade-acid/pre-upgrade/pom.xml b/upgrade-acid/pre-upgrade/pom.xml
new file mode 100644
index 000..da73d31
--- /dev/null
+++ b/upgrade-acid/pre-upgrade/pom.xml
@@ -0,0 +1,284 @@
+
+
+http://maven.apache.org/POM/4.0.0;
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+
+
+
+org.apache.hive
+hive-upgrade-acid
+4.0.0-SNAPSHOT
+../pom.xml
+
+
+
+4.0.0
+
+hive-pre-upgrade
+Hive Pre Upgrade Acid
+jar
+
+
+../..
+
+
+${project.build.directory}/testconf
+file://
+${project.basedir}/src/test/resources
+${project.build.directory}/tmp
+
${project.build.directory}/warehouse
+file://
+1
+true
+2.3.3
+2.7.2
+
+
+
+
+commons-cli
+commons-cli
+1.2
+provided
+
+
+org.apache.hive
+hive-metastore
+${hdp.hive.version}
+provided
+
+
+org.apache.hive
+hive-exec
+${hdp.hive.version}
+provided
+
+
+org.apache.hadoop
+hadoop-common
+${hdp.hadoop.version}
+provided
+
+
+
+org.apache.hadoop
+hadoop-mapreduce-client-common
+2.7.2
+provided
+
+
+org.apache.orc
+orc-core
+1.3.3
+provided
+
+
+
+
+
+
+
+${basedir}/src/main/resources
+
+package.jdo
+
+
+
+
+
+
+
+org.apache.maven.plugins
+maven-antrun-plugin
+${maven.antrun.plugin.version}
+
+
+ant-contrib
+ant-contrib
+${ant.contrib.version}
+
+
+ant
+ant
+
+
+
+
+
+
+org.apache.maven.plugins
+maven-checkstyle-plugin
+${maven.checkstyle.plugin.version}
+
+
+org.codehaus.mojo
+exec-maven-plugin
+${maven.exec.plugin.version}
+
+
+
+
+
+
+org.apache.maven.plugins
+maven-antrun-plugin
+
+
+setup-test-dirs
+process-test-resources
+
+run
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+setup-metastore-scripts
+process-test-resources
+
+run
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+org.apache.maven.plugins
+maven-failsafe-plugin
+2.20.1
+
+
+
+integration-test
+verify
+
+
+
+
+true
+false
+-Xmx2048m
+false
+
+  

hive git commit: HIVE-17683: Add explain locks command (Igor Kryvenko via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/branch-3 9f9b1e9b2 -> 96e7c4726


HIVE-17683: Add explain locks  command (Igor Kryvenko via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/96e7c472
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/96e7c472
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/96e7c472

Branch: refs/heads/branch-3
Commit: 96e7c4726778998c4c4ed31a21a7d7134f941b73
Parents: 9f9b1e9
Author: Igor Kryvenko 
Authored: Fri Aug 3 17:18:41 2018 -0700
Committer: Eugene Koifman 
Committed: Fri Aug 3 17:18:41 2018 -0700

--
 .../apache/hadoop/hive/ql/exec/ExplainTask.java |  46 +
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java | 207 +++
 .../hadoop/hive/ql/lockmgr/DbTxnManager.java| 176 +---
 .../apache/hadoop/hive/ql/metadata/Table.java   |   4 +
 .../hive/ql/parse/ExplainConfiguration.java |   8 +
 .../hive/ql/parse/ExplainSemanticAnalyzer.java  |   2 +
 .../apache/hadoop/hive/ql/parse/HiveParser.g|   1 +
 .../hadoop/hive/ql/plan/ExplainLockDesc.java| 116 +++
 .../apache/hadoop/hive/ql/plan/ExplainWork.java |  17 ++
 .../test/queries/clientpositive/explain_locks.q |  22 ++
 .../results/clientpositive/explain_locks.q.out  |  91 
 .../hive/metastore/LockComponentBuilder.java|   6 +
 .../hive/metastore/LockRequestBuilder.java  |  17 ++
 13 files changed, 544 insertions(+), 169 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/96e7c472/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
index cbdeb33..5fa5e9e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
@@ -45,9 +45,11 @@ import 
org.apache.hadoop.hive.common.jsonexplain.JsonParserFactory;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
 import org.apache.hadoop.hive.ql.Driver;
 import org.apache.hadoop.hive.ql.DriverContext;
 import org.apache.hadoop.hive.ql.hooks.ReadEntity;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.optimizer.physical.StageIDsRearranger;
 import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
@@ -55,6 +57,7 @@ import 
org.apache.hadoop.hive.ql.parse.ExplainConfiguration.VectorizationDetailL
 import org.apache.hadoop.hive.ql.plan.Explain;
 import org.apache.hadoop.hive.ql.plan.Explain.Level;
 import org.apache.hadoop.hive.ql.plan.Explain.Vectorization;
+import org.apache.hadoop.hive.ql.plan.ExplainLockDesc;
 import org.apache.hadoop.hive.ql.plan.ExplainWork;
 import org.apache.hadoop.hive.ql.plan.HiveOperation;
 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
@@ -296,6 +299,44 @@ public class ExplainTask extends Task 
implements Serializable {
 return jsonOutput ? outJSONObject : null;
   }
 
+  private JSONObject getLocks(PrintStream out, ExplainWork work) {
+
+JSONObject jsonObject = new JSONObject(new LinkedHashMap<>());
+
+boolean jsonOutput = work.isFormatted();
+if (jsonOutput) {
+  out = null;
+}
+if (work.getParseContext() != null) {
+  List lockComponents = 
AcidUtils.makeLockComponents(work.getOutputs(), work.getInputs(), conf);
+  if (null != out) {
+out.print("LOCK INFORMATION:\n");
+  }
+  List locks = new ArrayList<>(lockComponents.size());
+
+  for (LockComponent component : lockComponents) {
+ExplainLockDesc lockDesc = new ExplainLockDesc(component);
+
+if (null != out) {
+  out.print(lockDesc.getFullName());
+  out.print(" -> ");
+  out.print(lockDesc.getLockType());
+  out.print('\n');
+} else {
+  locks.add(lockDesc);
+}
+
+  }
+
+  if (jsonOutput) {
+jsonObject.put("LOCK INFORMATION:", locks);
+  }
+} else {
+  System.err.println("No parse context!");
+}
+return jsonObject;
+  }
+
   private List toString(Collection objects) {
 List list = new ArrayList();
 for (Object object : objects) {
@@ -353,6 +394,11 @@ public class ExplainTask extends Task 
implements Serializable {
   } else if (work.getDependency()) {
 JSONObject jsonDependencies = getJSONDependencies(work);
 out.print(jsonDependencies);
+  }  else if (work.isLocks()) {
+JSONObject jsonLocks = getLocks(out, work);
+if (work.isFormatted()) {
+  

hive git commit: ADDENDUM: HIVE-17683: Add explain locks command (Igor Kryvenko via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 90d19acd2 -> 5e7aa0986


ADDENDUM: HIVE-17683: Add explain locks  command (Igor Kryvenko via Eugene 
Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5e7aa098
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5e7aa098
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5e7aa098

Branch: refs/heads/master
Commit: 5e7aa0986b29edb3325378a83bf02a26e760f857
Parents: 90d19ac
Author: Igor Kryvenko 
Authored: Mon Jul 23 11:42:01 2018 -0700
Committer: Eugene Koifman 
Committed: Mon Jul 23 11:42:01 2018 -0700

--
 .../hadoop/hive/ql/plan/ExplainLockDesc.java| 116 +++
 1 file changed, 116 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/5e7aa098/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainLockDesc.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainLockDesc.java 
b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainLockDesc.java
new file mode 100644
index 000..1e7fb2e
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainLockDesc.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.plan;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+import org.apache.hadoop.hive.metastore.api.LockType;
+
+import java.util.ArrayList;
+
+/**
+ * ExplainLockDesc represents lock entity in query plan.
+ */
+public class ExplainLockDesc {
+  private String catalogName;
+  private String dbName;
+  private String tableName;
+  private String partitionName;
+  private LockType lockType;
+
+  public ExplainLockDesc() {
+  }
+
+  public ExplainLockDesc(LockComponent component) {
+this.dbName = component.getDbname();
+if (null != component.getTablename()) {
+  this.tableName = component.getTablename();
+}
+if (null != component.getPartitionname()) {
+  this.partitionName = component.getPartitionname();
+}
+this.lockType = component.getType();
+  }
+
+  public String getCatalogName() {
+return catalogName;
+  }
+
+  public ExplainLockDesc setCatalogName(String catalogName) {
+this.catalogName = catalogName;
+return this;
+  }
+
+  public String getDbName() {
+return dbName;
+  }
+
+  public ExplainLockDesc setDbName(String dbName) {
+this.dbName = dbName;
+return this;
+  }
+
+  public String getTableName() {
+return tableName;
+  }
+
+  public ExplainLockDesc setTableName(String tableName) {
+this.tableName = tableName;
+return this;
+  }
+
+  public String getPartitionName() {
+return partitionName;
+  }
+
+  public ExplainLockDesc setPartitionName(String partitionName) {
+this.partitionName = partitionName;
+return this;
+  }
+
+  public LockType getLockType() {
+return lockType;
+  }
+
+  public ExplainLockDesc setLockType(LockType lockType) {
+this.lockType = lockType;
+return this;
+  }
+
+  public String getFullName() {
+ArrayList list = new ArrayList();
+if (null != catalogName) {
+  list.add(catalogName);
+}
+if (null != dbName) {
+  list.add(dbName);
+}
+if (null != tableName) {
+  list.add(tableName);
+}
+if (null != partitionName) {
+  list.add(partitionName);
+}
+return StringUtils.join(list, '.');
+  }
+
+  @Override public String toString() {
+return getFullName() + " -> " + this.getLockType();
+  }
+}

hive git commit: HIVE-17683: Add explain locks command (Igor Kryvenko via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master bed17e54d -> 90d19acd2


HIVE-17683: Add explain locks  command (Igor Kryvenko via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/90d19acd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/90d19acd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/90d19acd

Branch: refs/heads/master
Commit: 90d19acd2b4f8301847ef13b4c8a91df3eafc65d
Parents: bed17e5
Author: Igor Kryvenko 
Authored: Mon Jul 23 11:02:17 2018 -0700
Committer: Eugene Koifman 
Committed: Mon Jul 23 11:02:17 2018 -0700

--
 .../apache/hadoop/hive/ql/exec/ExplainTask.java |  46 
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java | 206 ++
 .../hadoop/hive/ql/lockmgr/DbTxnManager.java| 218 +--
 .../apache/hadoop/hive/ql/metadata/Table.java   |   4 +
 .../hive/ql/parse/ExplainConfiguration.java |   8 +
 .../hive/ql/parse/ExplainSemanticAnalyzer.java  |   2 +
 .../apache/hadoop/hive/ql/parse/HiveParser.g|   1 +
 .../apache/hadoop/hive/ql/plan/ExplainWork.java |  17 ++
 .../test/queries/clientpositive/explain_locks.q |  22 ++
 .../results/clientpositive/explain_locks.q.out  |  91 
 .../hive/metastore/LockComponentBuilder.java|   5 +
 .../hive/metastore/LockRequestBuilder.java  |  17 ++
 12 files changed, 429 insertions(+), 208 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/90d19acd/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
index 752c3f3..f185d9d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
@@ -45,9 +45,11 @@ import 
org.apache.hadoop.hive.common.jsonexplain.JsonParserFactory;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
 import org.apache.hadoop.hive.ql.Driver;
 import org.apache.hadoop.hive.ql.DriverContext;
 import org.apache.hadoop.hive.ql.hooks.ReadEntity;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.optimizer.physical.StageIDsRearranger;
 import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
@@ -55,6 +57,7 @@ import 
org.apache.hadoop.hive.ql.parse.ExplainConfiguration.VectorizationDetailL
 import org.apache.hadoop.hive.ql.plan.Explain;
 import org.apache.hadoop.hive.ql.plan.Explain.Level;
 import org.apache.hadoop.hive.ql.plan.Explain.Vectorization;
+import org.apache.hadoop.hive.ql.plan.ExplainLockDesc;
 import org.apache.hadoop.hive.ql.plan.ExplainWork;
 import org.apache.hadoop.hive.ql.plan.HiveOperation;
 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
@@ -330,6 +333,44 @@ public class ExplainTask extends Task 
implements Serializable {
 return null;
   }
 
+  private JSONObject getLocks(PrintStream out, ExplainWork work) {
+
+JSONObject jsonObject = new JSONObject(new LinkedHashMap<>());
+
+boolean jsonOutput = work.isFormatted();
+if (jsonOutput) {
+  out = null;
+}
+if (work.getParseContext() != null) {
+  List lockComponents = 
AcidUtils.makeLockComponents(work.getOutputs(), work.getInputs(), conf);
+  if (null != out) {
+out.print("LOCK INFORMATION:\n");
+  }
+  List locks = new ArrayList<>(lockComponents.size());
+
+  for (LockComponent component : lockComponents) {
+ExplainLockDesc lockDesc = new ExplainLockDesc(component);
+
+if (null != out) {
+  out.print(lockDesc.getFullName());
+  out.print(" -> ");
+  out.print(lockDesc.getLockType());
+  out.print('\n');
+} else {
+  locks.add(lockDesc);
+}
+
+  }
+
+  if (jsonOutput) {
+jsonObject.put("LOCK INFORMATION:", locks);
+  }
+} else {
+  System.err.println("No parse context!");
+}
+return jsonObject;
+  }
+
   @Override
   public int execute(DriverContext driverContext) {
 
@@ -352,6 +393,11 @@ public class ExplainTask extends Task 
implements Serializable {
   } else if (work.getDependency()) {
 JSONObject jsonDependencies = getJSONDependencies(work);
 out.print(jsonDependencies);
+  } else if (work.isLocks()) {
+JSONObject jsonLocks = getLocks(out, work);
+if(work.isFormatted()) {
+  out.print(jsonLocks);
+}
   } else {
 if (work.isUserLevelExplain()) {
   // Because of the implementation of the 

hive git commit: HIVE-20218: make sure Statement.executeUpdate() returns number of rows affected (Eugene Koifman, reviewed by Ashutosh Chauhan)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master cce3a0521 -> 92ecdd97b


HIVE-20218: make sure Statement.executeUpdate() returns number of rows affected 
(Eugene Koifman, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/92ecdd97
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/92ecdd97
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/92ecdd97

Branch: refs/heads/master
Commit: 92ecdd97bc183b939d1e074f23f52f96e6fafe25
Parents: cce3a05
Author: Eugene Koifman 
Authored: Sun Jul 22 16:25:57 2018 -0700
Committer: Eugene Koifman 
Committed: Sun Jul 22 16:25:57 2018 -0700

--
 .../org/apache/hive/jdbc/TestJdbcDriver2.java   | 31 
 .../apache/hive/jdbc/HivePreparedStatement.java |  3 +-
 .../org/apache/hive/jdbc/HiveStatement.java |  3 +-
 3 files changed, 34 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/92ecdd97/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java 
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
index 850b2d5..8f552b0 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
@@ -209,6 +209,37 @@ public class TestJdbcDriver2 {
 stmt.close();
   }
 
+
+  @Test
+  public void testExceucteUpdateCounts() throws Exception {
+Statement stmt =  con.createStatement();
+stmt.execute("set " + ConfVars.HIVE_SUPPORT_CONCURRENCY.varname + "=true");
+stmt.execute("set " + ConfVars.HIVE_TXN_MANAGER.varname +
+"=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager");
+stmt.execute("create table transactional_crud (a int, b int) stored as orc 
" +
+"tblproperties('transactional'='true', 
'transactional_properties'='default')");
+int count = stmt.executeUpdate("insert into transactional_crud 
values(1,2),(3,4),(5,6)");
+assertEquals("Statement insert", 3, count);
+count = stmt.executeUpdate("update transactional_crud set b = 17 where a 
<= 3");
+assertEquals("Statement update", 2, count);
+count = stmt.executeUpdate("delete from transactional_crud where b = 6");
+assertEquals("Statement delete", 1, count);
+
+stmt.close();
+PreparedStatement pStmt =
+con.prepareStatement("update transactional_crud set b = ? where a = ? 
or a = ?");
+pStmt.setInt(1, 15);
+pStmt.setInt(2, 1);
+pStmt.setInt(3, 3);
+count = pStmt.executeUpdate();
+assertEquals("2 row PreparedStatement update", 2, count);
+pStmt.setInt(1, 19);
+pStmt.setInt(2, 3);
+pStmt.setInt(3, 3);
+count = pStmt.executeUpdate();
+assertEquals("1 row PreparedStatement update", 1, count);
+  }
+
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
 Statement stmt = con.createStatement();

http://git-wip-us.apache.org/repos/asf/hive/blob/92ecdd97/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java
--
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java 
b/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java
index 77a1797..f86b112 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java
@@ -119,8 +119,7 @@ public class HivePreparedStatement extends HiveStatement 
implements PreparedStat
*/
 
   public int executeUpdate() throws SQLException {
-super.executeUpdate(updateSql(sql, parameters));
-return 0;
+return super.executeUpdate(updateSql(sql, parameters));
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hive/blob/92ecdd97/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
--
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java 
b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
index 0b38f9c..89ff514 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
@@ -511,7 +511,8 @@ public class HiveStatement implements java.sql.Statement {
   @Override
   public int executeUpdate(String sql) throws SQLException {
 execute(sql);
-return 0;
+return getUpdateCount();
+//return getLargeUpdateCount(); - not currently implemented... wrong type
   }
 
   /*

hive git commit: HIVE-19375: Bad message: 'transactional'='false' is no longer a valid property and will be ignored (Eugene Koifman, reviewed by Jason Dere)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/branch-3 f22412e0d -> e827e0382


HIVE-19375: Bad message: 'transactional'='false' is no longer a valid property 
and will be ignored (Eugene Koifman, reviewed by Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e827e038
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e827e038
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e827e038

Branch: refs/heads/branch-3
Commit: e827e0382e26af72fc543d87e1fdbaf0cf395d19
Parents: f22412e
Author: Eugene Koifman 
Authored: Thu Jul 12 11:34:29 2018 -0700
Committer: Eugene Koifman 
Committed: Thu Jul 12 11:34:29 2018 -0700

--
 .../hadoop/hive/metastore/TransactionalValidationListener.java | 6 ++
 1 file changed, 2 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/e827e038/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
--
diff --git 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
index d9fdd29..76a8261 100644
--- 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
+++ 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
@@ -316,10 +316,8 @@ public final class TransactionalValidationListener extends 
MetaStorePreEventList
 }
 
 if ("false".equalsIgnoreCase(transactional)) {
-  // just drop transactional=false.  For backward compatibility in case 
someone has scripts
-  // with transactional=false
-  LOG.info("'transactional'='false' is no longer a valid property and will 
be ignored: " +
-Warehouse.getQualifiedName(newTable));
+  // just drop transactional=false - absence of 'transactional' property 
is equivalent to
+  // transactional=false
   return;
 }
 

hive git commit: HIVE-19375: Bad message: 'transactional'='false' is no longer a valid property and will be ignored (Eugene Koifman, reviewed by Jason Dere)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master b3923255a -> e0c2b9d97


HIVE-19375: Bad message: 'transactional'='false' is no longer a valid property 
and will be ignored (Eugene Koifman, reviewed by Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e0c2b9d9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e0c2b9d9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e0c2b9d9

Branch: refs/heads/master
Commit: e0c2b9d97de579df7a7c1d368bb1c4ff9b185bf5
Parents: b392325
Author: Eugene Koifman 
Authored: Thu Jul 12 11:33:33 2018 -0700
Committer: Eugene Koifman 
Committed: Thu Jul 12 11:34:01 2018 -0700

--
 .../hadoop/hive/metastore/TransactionalValidationListener.java | 6 ++
 1 file changed, 2 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/e0c2b9d9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
--
diff --git 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
index d9fdd29..76a8261 100644
--- 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
+++ 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
@@ -316,10 +316,8 @@ public final class TransactionalValidationListener extends 
MetaStorePreEventList
 }
 
 if ("false".equalsIgnoreCase(transactional)) {
-  // just drop transactional=false.  For backward compatibility in case 
someone has scripts
-  // with transactional=false
-  LOG.info("'transactional'='false' is no longer a valid property and will 
be ignored: " +
-Warehouse.getQualifiedName(newTable));
+  // just drop transactional=false - absence of 'transactional' property 
is equivalent to
+  // transactional=false
   return;
 }
 

hive git commit: HIVE-19387: Truncate table for Acid tables conflicts with ResultSet cache (Eugene Koifman, reviewed by Jason Dere)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/branch-3 3c71687d1 -> f22412e0d


HIVE-19387: Truncate table for Acid tables conflicts with ResultSet cache 
(Eugene Koifman, reviewed by Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f22412e0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f22412e0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f22412e0

Branch: refs/heads/branch-3
Commit: f22412e0def2e82fb3c79c03de295dc46ec25d9e
Parents: 3c71687
Author: Eugene Koifman 
Authored: Thu Jul 12 11:08:03 2018 -0700
Committer: Eugene Koifman 
Committed: Thu Jul 12 11:08:03 2018 -0700

--
 .../test/resources/testconfiguration.properties |   1 +
 .../hive/ql/parse/DDLSemanticAnalyzer.java  |  11 +-
 .../hadoop/hive/ql/plan/AlterTableDesc.java |   5 +
 .../hadoop/hive/ql/plan/TruncateTableDesc.java  |  33 +-
 .../clientpositive/results_cache_truncate.q |  61 +++
 .../llap/results_cache_truncate.q.out   | 470 +++
 6 files changed, 579 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/f22412e0/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties 
b/itests/src/test/resources/testconfiguration.properties
index 6d8a406..a0037d8 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -625,6 +625,7 @@ minillaplocal.query.files=\
   results_cache_quoted_identifiers.q,\
   results_cache_temptable.q,\
   results_cache_transactional.q,\
+  results_cache_truncate.q,\
   results_cache_with_masking.q,\
   sample10.q,\
   sample10_mm.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/f22412e0/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index f9d6d41..b14648a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -1480,7 +1480,13 @@ public class DDLSemanticAnalyzer extends 
BaseSemanticAnalyzer {
   }
 }
 
-TruncateTableDesc truncateTblDesc = new TruncateTableDesc(tableName, 
partSpec, null);
+TruncateTableDesc truncateTblDesc = new TruncateTableDesc(tableName, 
partSpec, null, table);
+if(truncateTblDesc.mayNeedWriteId()) {
+  if(this.ddlDescWithWriteId != null) {
+throw new IllegalStateException("ddlDescWithWriteId is already set: " 
+ this.ddlDescWithWriteId);
+  }
+  this.ddlDescWithWriteId = truncateTblDesc;
+}
 
 DDLWork ddlWork = new DDLWork(getInputs(), getOutputs(), truncateTblDesc);
 Task truncateTask = TaskFactory.get(ddlWork);
@@ -1759,6 +1765,9 @@ public class DDLSemanticAnalyzer extends 
BaseSemanticAnalyzer {
 
 DDLWork ddlWork = new DDLWork(getInputs(), getOutputs(), alterTblDesc);
 if (isPotentialMmSwitch) {
+  if(this.ddlDescWithWriteId != null) {
+throw new IllegalStateException("ddlDescWithWriteId is already set: " 
+ this.ddlDescWithWriteId);
+  }
   this.ddlDescWithWriteId = alterTblDesc;
   ddlWork.setNeedLock(true); // Hmm... why don't many other operations 
here need locks?
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/f22412e0/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java 
b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
index 0b04c0c..ec04a01 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
@@ -967,4 +967,9 @@ public class AlterTableDesc extends DDLDesc implements 
Serializable, DDLDesc.DDL
   public Long getWriteId() {
 return this.writeId;
   }
+
+  @Override
+  public String toString() {
+return this.getClass().getSimpleName() + " for " + getFullTableName();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/f22412e0/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java 
b/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java
index ed68f1a..8c3d852 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java
+++ 

hive git commit: HIVE-19387: Truncate table for Acid tables conflicts with ResultSet cache (Eugene Koifman, reviewed by Jason Dere)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 385a26ad8 -> 733c4f316


HIVE-19387: Truncate table for Acid tables conflicts with ResultSet cache 
(Eugene Koifman, reviewed by Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/733c4f31
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/733c4f31
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/733c4f31

Branch: refs/heads/master
Commit: 733c4f3169142d426995f8e12961c209688d29d0
Parents: 385a26a
Author: Eugene Koifman 
Authored: Wed Jul 11 14:02:31 2018 -0700
Committer: Eugene Koifman 
Committed: Wed Jul 11 14:02:31 2018 -0700

--
 .../test/resources/testconfiguration.properties |   1 +
 .../hive/ql/parse/DDLSemanticAnalyzer.java  |  11 +-
 .../hadoop/hive/ql/plan/AlterTableDesc.java |   5 +
 .../hadoop/hive/ql/plan/TruncateTableDesc.java  |  33 +-
 .../clientpositive/results_cache_truncate.q |  61 +++
 .../llap/results_cache_truncate.q.out   | 470 +++
 6 files changed, 579 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/733c4f31/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties 
b/itests/src/test/resources/testconfiguration.properties
index 9e012ce..72dd144 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -610,6 +610,7 @@ minillaplocal.query.files=\
   results_cache_quoted_identifiers.q,\
   results_cache_temptable.q,\
   results_cache_transactional.q,\
+  results_cache_truncate.q,\
   results_cache_with_masking.q,\
   sample10.q,\
   sample10_mm.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/733c4f31/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index 9ad4689..b6825ae 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -1478,7 +1478,13 @@ public class DDLSemanticAnalyzer extends 
BaseSemanticAnalyzer {
   }
 }
 
-TruncateTableDesc truncateTblDesc = new TruncateTableDesc(tableName, 
partSpec, null);
+TruncateTableDesc truncateTblDesc = new TruncateTableDesc(tableName, 
partSpec, null, table);
+if(truncateTblDesc.mayNeedWriteId()) {
+  if(this.ddlDescWithWriteId != null) {
+throw new IllegalStateException("ddlDescWithWriteId is already set: " 
+ this.ddlDescWithWriteId);
+  }
+  this.ddlDescWithWriteId = truncateTblDesc;
+}
 
 DDLWork ddlWork = new DDLWork(getInputs(), getOutputs(), truncateTblDesc);
 Task truncateTask = TaskFactory.get(ddlWork);
@@ -1757,6 +1763,9 @@ public class DDLSemanticAnalyzer extends 
BaseSemanticAnalyzer {
 
 DDLWork ddlWork = new DDLWork(getInputs(), getOutputs(), alterTblDesc);
 if (isPotentialMmSwitch) {
+  if(this.ddlDescWithWriteId != null) {
+throw new IllegalStateException("ddlDescWithWriteId is already set: " 
+ this.ddlDescWithWriteId);
+  }
   this.ddlDescWithWriteId = alterTblDesc;
   ddlWork.setNeedLock(true); // Hmm... why don't many other operations 
here need locks?
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/733c4f31/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java 
b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
index 0b04c0c..ec04a01 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
@@ -967,4 +967,9 @@ public class AlterTableDesc extends DDLDesc implements 
Serializable, DDLDesc.DDL
   public Long getWriteId() {
 return this.writeId;
   }
+
+  @Override
+  public String toString() {
+return this.getClass().getSimpleName() + " for " + getFullTableName();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/733c4f31/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java 
b/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java
index ed68f1a..8c3d852 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java
+++ 

hive git commit: HIVE-19965: Make HiveEndPoint use IMetaStoreClient.add_partition (Eugene Koifman, reviewed by Prasanth Jayachandran)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master ba210c904 -> be1955f61


HIVE-19965: Make HiveEndPoint use IMetaStoreClient.add_partition (Eugene 
Koifman, reviewed by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/be1955f6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/be1955f6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/be1955f6

Branch: refs/heads/master
Commit: be1955f61496925a5e452228b46d8d9a3f08ddbd
Parents: ba210c9
Author: Eugene Koifman 
Authored: Sun Jun 24 15:27:12 2018 -0700
Committer: Eugene Koifman 
Committed: Sun Jun 24 15:27:12 2018 -0700

--
 .../hive/hcatalog/streaming/HiveEndPoint.java   | 142 +--
 .../hive/hcatalog/streaming/TestStreaming.java  |  12 ++
 2 files changed, 78 insertions(+), 76 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/be1955f6/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
--
diff --git 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
index 3ee19dd..3604630 100644
--- 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
+++ 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
@@ -19,9 +19,15 @@
 package org.apache.hive.hcatalog.streaming;
 
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.JavaUtils;
+import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
 import org.apache.hadoop.hive.metastore.api.DataOperationType;
+import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.hive.cli.CliSessionState;
@@ -55,6 +61,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
 /**
  * Information about the hive end point (i.e. table or partition) to write to.
@@ -294,42 +301,42 @@ public class HiveEndPoint {
 private final String agentInfo;
 
 /**
- * @param endPoint end point to connect to
- * @param ugi on behalf of whom streaming is done. cannot be null
- * @param conf HiveConf object
+ * @param endPoint   end point to connect to
+ * @param ugion behalf of whom streaming is done. cannot be null
+ * @param conf   HiveConf object
  * @param createPart create the partition if it does not exist
- * @throws ConnectionError if there is trouble connecting
- * @throws InvalidPartition if specified partition does not exist (and 
createPart=false)
- * @throws InvalidTable if specified table does not exist
+ * @throws ConnectionError if there is trouble connecting
+ * @throws InvalidPartitionif specified partition does not exist 
(and createPart=false)
+ * @throws InvalidTableif specified table does not exist
  * @throws PartitionCreationFailed if createPart=true and not able to 
create partition
  */
 private ConnectionImpl(HiveEndPoint endPoint, UserGroupInformation ugi,
-   HiveConf conf, boolean createPart, String agentInfo)
-throws ConnectionError, InvalidPartition, InvalidTable
-   , PartitionCreationFailed {
+HiveConf conf, boolean createPart, String agentInfo)
+throws ConnectionError, InvalidPartition, InvalidTable
+, PartitionCreationFailed {
   this.endPt = endPoint;
   this.ugi = ugi;
   this.agentInfo = agentInfo;
-  this.username = ugi==null ? System.getProperty("user.name") : 
ugi.getShortUserName();
-  if (conf==null) {
+  this.username = ugi == null ? System.getProperty("user.name") : 
ugi.getShortUserName();
+  if (conf == null) {
 conf = HiveEndPoint.createHiveConf(this.getClass(), 
endPoint.metaStoreUri);
+  } else {
+overrideConfSettings(conf);
   }
-  else {
-  overrideConfSettings(conf);
-  }
-  this.secureMode = ugi==null ? false : ugi.hasKerberosCredentials();
+  this.secureMode = ugi == null ? false : ugi.hasKerberosCredentials();
   this.msClient = getMetaStoreClient(endPoint, conf, secureMode);
   // We use a separate metastore client for heartbeat calls to ensure 
heartbeat RPC calls are
   // isolated from the other transaction related RPC calls.
   

hive git commit: HIVE-19965: Make HiveEndPoint use IMetaStoreClient.add_partition (Eugene Koifman, reviewed by Prasanth Jayachandran)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/branch-3 0e1de2e5b -> bf39dcb4d


HIVE-19965: Make HiveEndPoint use IMetaStoreClient.add_partition (Eugene 
Koifman, reviewed by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bf39dcb4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bf39dcb4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bf39dcb4

Branch: refs/heads/branch-3
Commit: bf39dcb4d4da2fa1fc5258f6204f0ec0213ffb52
Parents: 0e1de2e
Author: Eugene Koifman 
Authored: Sun Jun 24 15:27:50 2018 -0700
Committer: Eugene Koifman 
Committed: Sun Jun 24 15:27:50 2018 -0700

--
 .../hive/hcatalog/streaming/HiveEndPoint.java   | 142 +--
 .../hive/hcatalog/streaming/TestStreaming.java  |  12 ++
 2 files changed, 78 insertions(+), 76 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/bf39dcb4/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
--
diff --git 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
index 3ee19dd..3604630 100644
--- 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
+++ 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
@@ -19,9 +19,15 @@
 package org.apache.hive.hcatalog.streaming;
 
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.JavaUtils;
+import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
 import org.apache.hadoop.hive.metastore.api.DataOperationType;
+import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.hive.cli.CliSessionState;
@@ -55,6 +61,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
 /**
  * Information about the hive end point (i.e. table or partition) to write to.
@@ -294,42 +301,42 @@ public class HiveEndPoint {
 private final String agentInfo;
 
 /**
- * @param endPoint end point to connect to
- * @param ugi on behalf of whom streaming is done. cannot be null
- * @param conf HiveConf object
+ * @param endPoint   end point to connect to
+ * @param ugion behalf of whom streaming is done. cannot be null
+ * @param conf   HiveConf object
  * @param createPart create the partition if it does not exist
- * @throws ConnectionError if there is trouble connecting
- * @throws InvalidPartition if specified partition does not exist (and 
createPart=false)
- * @throws InvalidTable if specified table does not exist
+ * @throws ConnectionError if there is trouble connecting
+ * @throws InvalidPartitionif specified partition does not exist 
(and createPart=false)
+ * @throws InvalidTableif specified table does not exist
  * @throws PartitionCreationFailed if createPart=true and not able to 
create partition
  */
 private ConnectionImpl(HiveEndPoint endPoint, UserGroupInformation ugi,
-   HiveConf conf, boolean createPart, String agentInfo)
-throws ConnectionError, InvalidPartition, InvalidTable
-   , PartitionCreationFailed {
+HiveConf conf, boolean createPart, String agentInfo)
+throws ConnectionError, InvalidPartition, InvalidTable
+, PartitionCreationFailed {
   this.endPt = endPoint;
   this.ugi = ugi;
   this.agentInfo = agentInfo;
-  this.username = ugi==null ? System.getProperty("user.name") : 
ugi.getShortUserName();
-  if (conf==null) {
+  this.username = ugi == null ? System.getProperty("user.name") : 
ugi.getShortUserName();
+  if (conf == null) {
 conf = HiveEndPoint.createHiveConf(this.getClass(), 
endPoint.metaStoreUri);
+  } else {
+overrideConfSettings(conf);
   }
-  else {
-  overrideConfSettings(conf);
-  }
-  this.secureMode = ugi==null ? false : ugi.hasKerberosCredentials();
+  this.secureMode = ugi == null ? false : ugi.hasKerberosCredentials();
   this.msClient = getMetaStoreClient(endPoint, conf, secureMode);
   // We use a separate metastore client for heartbeat calls to ensure 
heartbeat RPC calls are
   // isolated from the other transaction related RPC calls.
   

hive git commit: HIVE-19908 Block Insert Overwrite with Union All on full CRUD ACID tables using HIVE_UNION_SUBDIR_ (Eugene Koifman, reviewed by Prasanth Jayachandran)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/branch-3 665198c4c -> 40fe6c397


HIVE-19908 Block Insert Overwrite with Union All on full CRUD ACID tables using 
HIVE_UNION_SUBDIR_ (Eugene Koifman, reviewed by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/40fe6c39
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/40fe6c39
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/40fe6c39

Branch: refs/heads/branch-3
Commit: 40fe6c397ecaee0793db70187bd7ce84b871d4fe
Parents: 665198c
Author: Eugene Koifman 
Authored: Wed Jun 20 09:07:45 2018 -0700
Committer: Eugene Koifman 
Committed: Wed Jun 20 09:07:45 2018 -0700

--
 ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 15 +--
 .../apache/hadoop/hive/ql/plan/FileSinkDesc.java  |  9 +
 .../apache/hadoop/hive/ql/TestTxnNoBuckets.java   | 18 ++
 3 files changed, 40 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/40fe6c39/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java 
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index e43d1af..4a7131a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -71,6 +71,7 @@ import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
 import org.apache.hadoop.hive.ql.cache.results.CacheUsage;
 import org.apache.hadoop.hive.ql.cache.results.QueryResultsCache;
 import org.apache.hadoop.hive.ql.cache.results.QueryResultsCache.CacheEntry;
+import org.apache.hadoop.hive.ql.exec.AbstractFileMergeOperator;
 import org.apache.hadoop.hive.ql.exec.ConditionalTask;
 import org.apache.hadoop.hive.ql.exec.DagUtils;
 import org.apache.hadoop.hive.ql.exec.ExplainTask;
@@ -1578,9 +1579,19 @@ public class Driver implements IDriver {
   Utilities.getTableName(tableInfo.getTableName()));
   desc.setTableWriteId(writeId);
 
-  //it's possible to have > 1 FileSink writing to the same 
table/partition
-  //e.g. Merge stmt, multi-insert stmt when mixing DP and SP writes
+  /**
+   * it's possible to have > 1 FileSink writing to the same 
table/partition
+   * e.g. Merge stmt, multi-insert stmt when mixing DP and SP writes
+   * Insert ... Select ... Union All Select ... using
+   * {@link 
org.apache.hadoop.hive.ql.exec.AbstractFileMergeOperator#UNION_SUDBIR_PREFIX}
+   */
   desc.setStatementId(queryTxnMgr.getStmtIdAndIncrement());
+  String unionAllSubdir = "/" + 
AbstractFileMergeOperator.UNION_SUDBIR_PREFIX;
+  if(desc.getInsertOverwrite() && 
desc.getDirName().toString().contains(unionAllSubdir) &&
+  desc.isFullAcidTable()) {
+throw new UnsupportedOperationException("QueryId=" + 
plan.getQueryId() +
+" is not supported due to OVERWRITE and UNION ALL.  Please use 
truncate + insert");
+  }
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/40fe6c39/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java 
b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
index 1d05468..42b8f40 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
@@ -300,6 +300,15 @@ public class FileSinkDesc extends AbstractOperatorDesc 
implements IStatsGatherDe
   return AcidUtils.isInsertOnlyTable(getTableInfo().getProperties());
 }
   }
+  public boolean isFullAcidTable() {
+if(getTable() != null) {
+  return AcidUtils.isFullAcidTable(table);
+}
+else {
+  return 
AcidUtils.isTablePropertyTransactional(getTableInfo().getProperties()) &&
+  !AcidUtils.isInsertOnlyTable(getTableInfo().getProperties());
+}
+  }
 
   public boolean isMaterialization() {
 return materialization;

http://git-wip-us.apache.org/repos/asf/hive/blob/40fe6c39/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
index f071531..7ab76b3 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
@@ -275,6 +275,24 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree 
/U

hive git commit: HIVE-19908 Block Insert Overwrite with Union All on full CRUD ACID tables using HIVE_UNION_SUBDIR_ (Eugene Koifman, reviewed by Prasanth Jayachandran)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 52f190e15 -> 6c4d7a9b9


HIVE-19908 Block Insert Overwrite with Union All on full CRUD ACID tables using 
HIVE_UNION_SUBDIR_ (Eugene Koifman, reviewed by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6c4d7a9b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6c4d7a9b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6c4d7a9b

Branch: refs/heads/master
Commit: 6c4d7a9b945f1dad27ef1a6193a689040fee7b66
Parents: 52f190e
Author: Eugene Koifman 
Authored: Wed Jun 20 09:01:52 2018 -0700
Committer: Eugene Koifman 
Committed: Wed Jun 20 09:01:52 2018 -0700

--
 ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 15 +--
 .../apache/hadoop/hive/ql/plan/FileSinkDesc.java  |  9 +
 .../apache/hadoop/hive/ql/TestTxnNoBuckets.java   | 18 ++
 3 files changed, 40 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/6c4d7a9b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java 
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 43a78ca..762e57c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -71,6 +71,7 @@ import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
 import org.apache.hadoop.hive.ql.cache.results.CacheUsage;
 import org.apache.hadoop.hive.ql.cache.results.QueryResultsCache;
 import org.apache.hadoop.hive.ql.cache.results.QueryResultsCache.CacheEntry;
+import org.apache.hadoop.hive.ql.exec.AbstractFileMergeOperator;
 import org.apache.hadoop.hive.ql.exec.ConditionalTask;
 import org.apache.hadoop.hive.ql.exec.DagUtils;
 import org.apache.hadoop.hive.ql.exec.ExplainTask;
@@ -1581,9 +1582,19 @@ public class Driver implements IDriver {
   Utilities.getTableName(tableInfo.getTableName()));
   desc.setTableWriteId(writeId);
 
-  //it's possible to have > 1 FileSink writing to the same 
table/partition
-  //e.g. Merge stmt, multi-insert stmt when mixing DP and SP writes
+  /**
+   * it's possible to have > 1 FileSink writing to the same 
table/partition
+   * e.g. Merge stmt, multi-insert stmt when mixing DP and SP writes
+   * Insert ... Select ... Union All Select ... using
+   * {@link 
org.apache.hadoop.hive.ql.exec.AbstractFileMergeOperator#UNION_SUDBIR_PREFIX}
+   */
   desc.setStatementId(queryTxnMgr.getStmtIdAndIncrement());
+  String unionAllSubdir = "/" + 
AbstractFileMergeOperator.UNION_SUDBIR_PREFIX;
+  if(desc.getInsertOverwrite() && 
desc.getDirName().toString().contains(unionAllSubdir) &&
+  desc.isFullAcidTable()) {
+throw new UnsupportedOperationException("QueryId=" + 
plan.getQueryId() +
+" is not supported due to OVERWRITE and UNION ALL.  Please use 
truncate + insert");
+  }
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/6c4d7a9b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java 
b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
index 1d05468..42b8f40 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
@@ -300,6 +300,15 @@ public class FileSinkDesc extends AbstractOperatorDesc 
implements IStatsGatherDe
   return AcidUtils.isInsertOnlyTable(getTableInfo().getProperties());
 }
   }
+  public boolean isFullAcidTable() {
+if(getTable() != null) {
+  return AcidUtils.isFullAcidTable(table);
+}
+else {
+  return 
AcidUtils.isTablePropertyTransactional(getTableInfo().getProperties()) &&
+  !AcidUtils.isInsertOnlyTable(getTableInfo().getProperties());
+}
+  }
 
   public boolean isMaterialization() {
 return materialization;

http://git-wip-us.apache.org/repos/asf/hive/blob/6c4d7a9b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
index f071531..7ab76b3 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
@@ -275,6 +275,24 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree 
/Users/ekoifman

hive git commit: HIVE-19917: Export of full CRUD transactional table fails if table is not in default database (Eugene Koifman, reviewed by Jason Dere)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 73ee8a138 -> 97da974fa


HIVE-19917: Export of full CRUD transactional table fails if table is not in 
default database (Eugene Koifman, reviewed by Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/97da974f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/97da974f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/97da974f

Branch: refs/heads/master
Commit: 97da974fa5db6c9de3f4841b745693c7c92892e9
Parents: 73ee8a1
Author: Eugene Koifman 
Authored: Fri Jun 15 16:59:39 2018 -0700
Committer: Eugene Koifman 
Committed: Fri Jun 15 16:59:39 2018 -0700

--
 .../org/apache/hadoop/hive/ql/TestTxnExIm.java  | 23 +++-
 1 file changed, 17 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/97da974f/ql/src/test/org/apache/hadoop/hive/ql/TestTxnExIm.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnExIm.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnExIm.java
index 8d3ab77..ad4ed76 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnExIm.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnExIm.java
@@ -53,18 +53,29 @@ public class TestTxnExIm extends TxnCommandsBaseForTests {
* simplest export test.
*/
   @Test
-  public void testExport() throws Exception {
+  public void testExportDefaultDb() throws Exception {
+testExport(true);
+  }
+  @Test
+  public void testExportCustomDb() throws Exception {
+testExport(false);
+  }
+  private void testExport(boolean useDefualtDb) throws Exception {
 int[][] rows1 = {{1, 2}, {3, 4}};
-runStatementOnDriver("drop table if exists T");
+final String tblName = useDefualtDb ? "T" : "foo.T";
+runStatementOnDriver("drop table if exists " + tblName);
 runStatementOnDriver("drop table if exists TImport ");
-runStatementOnDriver("create table T (a int, b int) stored as ORC");
+if(!useDefualtDb) {
+  runStatementOnDriver("create database foo");
+}
+runStatementOnDriver("create table " + tblName + " (a int, b int) stored 
as ORC");
 runStatementOnDriver("create table TImport (a int, b int) stored as ORC 
TBLPROPERTIES " +
 "('transactional'='false')");
-runStatementOnDriver("insert into T(a,b) " + makeValuesClause(rows1));
-List rs = runStatementOnDriver("select * from T order by a,b");
+runStatementOnDriver("insert into " + tblName + "(a,b) " + 
makeValuesClause(rows1));
+List rs = runStatementOnDriver("select * from " + tblName + " 
order by a,b");
 Assert.assertEquals("Content didn't match rs", stringifyValues(rows1), rs);
 
-String exportStmt = "export table T to '" + getTestDataDir() + "/export'";
+String exportStmt = "export table " + tblName + " to '" + getTestDataDir() 
+ "/export'";
 rs = runStatementOnDriver("explain " + exportStmt);
 StringBuilder sb = new StringBuilder("*** " + exportStmt);
 for (String r : rs) {

hive git commit: HIVE-19917: Export of full CRUD transactional table fails if table is not in default database (Eugene Koifman, reviewed by Jason Dere)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/branch-3 2b4868746 -> fb1e5ba7a


HIVE-19917: Export of full CRUD transactional table fails if table is not in 
default database (Eugene Koifman, reviewed by Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fb1e5ba7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fb1e5ba7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fb1e5ba7

Branch: refs/heads/branch-3
Commit: fb1e5ba7ae381ba8e10cd004e8a7297269217672
Parents: 2b48687
Author: Eugene Koifman 
Authored: Fri Jun 15 20:48:13 2018 -0700
Committer: Eugene Koifman 
Committed: Fri Jun 15 20:48:13 2018 -0700

--
 .../org/apache/hadoop/hive/ql/TestTxnExIm.java  | 23 +++-
 1 file changed, 17 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/fb1e5ba7/ql/src/test/org/apache/hadoop/hive/ql/TestTxnExIm.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnExIm.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnExIm.java
index 8d3ab77..ad4ed76 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnExIm.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnExIm.java
@@ -53,18 +53,29 @@ public class TestTxnExIm extends TxnCommandsBaseForTests {
* simplest export test.
*/
   @Test
-  public void testExport() throws Exception {
+  public void testExportDefaultDb() throws Exception {
+testExport(true);
+  }
+  @Test
+  public void testExportCustomDb() throws Exception {
+testExport(false);
+  }
+  private void testExport(boolean useDefualtDb) throws Exception {
 int[][] rows1 = {{1, 2}, {3, 4}};
-runStatementOnDriver("drop table if exists T");
+final String tblName = useDefualtDb ? "T" : "foo.T";
+runStatementOnDriver("drop table if exists " + tblName);
 runStatementOnDriver("drop table if exists TImport ");
-runStatementOnDriver("create table T (a int, b int) stored as ORC");
+if(!useDefualtDb) {
+  runStatementOnDriver("create database foo");
+}
+runStatementOnDriver("create table " + tblName + " (a int, b int) stored 
as ORC");
 runStatementOnDriver("create table TImport (a int, b int) stored as ORC 
TBLPROPERTIES " +
 "('transactional'='false')");
-runStatementOnDriver("insert into T(a,b) " + makeValuesClause(rows1));
-List rs = runStatementOnDriver("select * from T order by a,b");
+runStatementOnDriver("insert into " + tblName + "(a,b) " + 
makeValuesClause(rows1));
+List rs = runStatementOnDriver("select * from " + tblName + " 
order by a,b");
 Assert.assertEquals("Content didn't match rs", stringifyValues(rows1), rs);
 
-String exportStmt = "export table T to '" + getTestDataDir() + "/export'";
+String exportStmt = "export table " + tblName + " to '" + getTestDataDir() 
+ "/export'";
 rs = runStatementOnDriver("explain " + exportStmt);
 StringBuilder sb = new StringBuilder("*** " + exportStmt);
 for (String r : rs) {

hive git commit: HIVE-19826: update errata.txt

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 9578edb50 -> 0a5b3b4d3


HIVE-19826: update errata.txt


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0a5b3b4d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0a5b3b4d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0a5b3b4d

Branch: refs/heads/master
Commit: 0a5b3b4d3632b2a973fa53282caea9175fb71f22
Parents: 9578edb
Author: Eugene Koifman 
Authored: Thu Jun 14 18:10:00 2018 -0700
Committer: Eugene Koifman 
Committed: Thu Jun 14 18:10:00 2018 -0700

--
 errata.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/0a5b3b4d/errata.txt
--
diff --git a/errata.txt b/errata.txt
index c6d259f..74d4c64 100644
--- a/errata.txt
+++ b/errata.txt
@@ -2,7 +2,7 @@ Commits with the wrong or no JIRA referenced:
 
 git commit   branch jira   url
 1aec6a39424942f5dc7974413ad5ea2829985a73 master HIVE-19826 
https://issues.apache.org/jira/browse/HIVE-19826
-7981d38cbf6ef43384f5fb51b1560d7cf5add729 master HIVE-19826 
https://issues.apache.org/jira/browse/HIVE-19826
+7981d38cbf6ef43384f5fb51b1560d7cf5add729 branch-3   HIVE-19826 
https://issues.apache.org/jira/browse/HIVE-19826
 233884620af67e6af72b60629f799a69f5823eb2 master HIVE-18627 
https://issues.apache.org/jira/browse/HIVE-18627
 eb0034c0cdcc5f10fd5d7382e2caf787a8003e7a master HIVE-17420 
https://issues.apache.org/jira/browse/HIVE-17420
 f1aae85f197de09d4b86143f7f13d5aa21d2eb85 master HIVE-16431 
https://issues.apache.org/jira/browse/HIVE-16431

[2/2] hive git commit: HIVE-19826: update errata.txt

[ekoifman]

HIVE-19826: update errata.txt


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/91e0d2b8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/91e0d2b8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/91e0d2b8

Branch: refs/heads/branch-3
Commit: 91e0d2b8cdc08b4aee7e1f4b53f6da3bc32e3d72
Parents: f7c060a
Author: Eugene Koifman 
Authored: Thu Jun 14 18:09:04 2018 -0700
Committer: Eugene Koifman 
Committed: Thu Jun 14 18:09:04 2018 -0700

--
 errata.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/91e0d2b8/errata.txt
--
diff --git a/errata.txt b/errata.txt
index 7b86da0..cfa01e2 100644
--- a/errata.txt
+++ b/errata.txt
@@ -1,7 +1,7 @@
 Commits with the wrong or no JIRA referenced:
 
 git commit   branch jira   url
-7981d38cbf6ef43384f5fb51b1560d7cf5add729 branch-3 HIVE-19826 
https://issues.apache.org/jira/browse/HIVE-19826
+7981d38cbf6ef43384f5fb51b1560d7cf5add729 branch-3   HIVE-19826 
https://issues.apache.org/jira/browse/HIVE-19826
 233884620af67e6af72b60629f799a69f5823eb2 master HIVE-18627 
https://issues.apache.org/jira/browse/HIVE-18627
 eb0034c0cdcc5f10fd5d7382e2caf787a8003e7a master HIVE-17420 
https://issues.apache.org/jira/browse/HIVE-17420
 f1aae85f197de09d4b86143f7f13d5aa21d2eb85 master HIVE-16431 
https://issues.apache.org/jira/browse/HIVE-16431

[1/2] hive git commit: HIVE-19826: update errata.txt

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/branch-3 88d0da45b -> 91e0d2b8c


HIVE-19826: update errata.txt


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f7c060a3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f7c060a3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f7c060a3

Branch: refs/heads/branch-3
Commit: f7c060a367d3444225bab94cf5d2666a31754544
Parents: 88d0da4
Author: Eugene Koifman 
Authored: Thu Jun 14 18:08:27 2018 -0700
Committer: Eugene Koifman 
Committed: Thu Jun 14 18:08:27 2018 -0700

--
 errata.txt | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/f7c060a3/errata.txt
--
diff --git a/errata.txt b/errata.txt
index 59c2d7d..7b86da0 100644
--- a/errata.txt
+++ b/errata.txt
@@ -1,6 +1,7 @@
 Commits with the wrong or no JIRA referenced:
 
 git commit   branch jira   url
+7981d38cbf6ef43384f5fb51b1560d7cf5add729 branch-3 HIVE-19826 
https://issues.apache.org/jira/browse/HIVE-19826
 233884620af67e6af72b60629f799a69f5823eb2 master HIVE-18627 
https://issues.apache.org/jira/browse/HIVE-18627
 eb0034c0cdcc5f10fd5d7382e2caf787a8003e7a master HIVE-17420 
https://issues.apache.org/jira/browse/HIVE-17420
 f1aae85f197de09d4b86143f7f13d5aa21d2eb85 master HIVE-16431 
https://issues.apache.org/jira/browse/HIVE-16431

hive git commit: HIVE-19826: update errata.txt

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master a06721e8e -> 9578edb50


HIVE-19826: update errata.txt


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9578edb5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9578edb5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9578edb5

Branch: refs/heads/master
Commit: 9578edb502c5a17d489784e5d29674e84eb0bbe6
Parents: a06721e
Author: Eugene Koifman 
Authored: Thu Jun 14 17:45:47 2018 -0700
Committer: Eugene Koifman 
Committed: Thu Jun 14 17:45:47 2018 -0700

--
 errata.txt | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/9578edb5/errata.txt
--
diff --git a/errata.txt b/errata.txt
index 59c2d7d..c6d259f 100644
--- a/errata.txt
+++ b/errata.txt
@@ -1,6 +1,8 @@
 Commits with the wrong or no JIRA referenced:
 
 git commit   branch jira   url
+1aec6a39424942f5dc7974413ad5ea2829985a73 master HIVE-19826 
https://issues.apache.org/jira/browse/HIVE-19826
+7981d38cbf6ef43384f5fb51b1560d7cf5add729 master HIVE-19826 
https://issues.apache.org/jira/browse/HIVE-19826
 233884620af67e6af72b60629f799a69f5823eb2 master HIVE-18627 
https://issues.apache.org/jira/browse/HIVE-18627
 eb0034c0cdcc5f10fd5d7382e2caf787a8003e7a master HIVE-17420 
https://issues.apache.org/jira/browse/HIVE-17420
 f1aae85f197de09d4b86143f7f13d5aa21d2eb85 master HIVE-16431 
https://issues.apache.org/jira/browse/HIVE-16431

hive git commit: OrcRawRecordMerger doesn't work for more than one file in non vectorized case (Sergey Shelukhin, reviewed by Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 645458532 -> 1aec6a394


OrcRawRecordMerger doesn't work for more than one file in non vectorized case 
(Sergey Shelukhin, reviewed by Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1aec6a39
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1aec6a39
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1aec6a39

Branch: refs/heads/master
Commit: 1aec6a39424942f5dc7974413ad5ea2829985a73
Parents: 6454585
Author: sergey 
Authored: Sat Jun 9 09:31:16 2018 -0700
Committer: Eugene Koifman 
Committed: Sat Jun 9 09:31:16 2018 -0700

--
 .../hadoop/hive/ql/io/RecordIdentifier.java |  1 +
 .../hadoop/hive/ql/io/orc/OrcInputFormat.java   |  6 ++--
 .../hive/ql/io/orc/OrcRawRecordMerger.java  | 38 ++--
 3 files changed, 33 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/1aec6a39/ql/src/java/org/apache/hadoop/hive/ql/io/RecordIdentifier.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/RecordIdentifier.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/RecordIdentifier.java
index 607abfd..ea7ba53 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/RecordIdentifier.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/RecordIdentifier.java
@@ -226,6 +226,7 @@ public class RecordIdentifier implements 
WritableComparable {
 return "{originalWriteId: " + writeId + ", " + bucketToString() + ", row: 
" + getRowId() +"}";
   }
   protected String bucketToString() {
+if (bucketId == -1) return ("bucket: " + bucketId);
 BucketCodec codec =
   BucketCodec.determineVersion(bucketId);
 return  "bucket: " + bucketId + "(" + codec.getVersion() + "." +

http://git-wip-us.apache.org/repos/asf/hive/blob/1aec6a39/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index 31338d7..a789dd2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -2076,9 +2076,11 @@ public class OrcInputFormat implements 
InputFormat,
 String txnString = conf.get(ValidWriteIdList.VALID_WRITEIDS_KEY);
 ValidWriteIdList validWriteIdList
 = (txnString == null) ? new ValidReaderWriteIdList() : new 
ValidReaderWriteIdList(txnString);
-LOG.debug("getReader:: Read ValidWriteIdList: " + 
validWriteIdList.toString()
+if (LOG.isDebugEnabled()) {
+  LOG.debug("getReader:: Read ValidWriteIdList: " + 
validWriteIdList.toString()
 + " isTransactionalTable: " + HiveConf.getBoolVar(conf, 
ConfVars.HIVE_TRANSACTIONAL_TABLE_SCAN));
-
+  LOG.debug("Creating merger for {} and {}", split.getPath(), 
Arrays.toString(deltas));
+}
 final OrcRawRecordMerger records =
 new OrcRawRecordMerger(conf, true, reader, split.isOriginal(), bucket,
 validWriteIdList, readOptions, deltas, mergerOptions);

http://git-wip-us.apache.org/repos/asf/hive/blob/1aec6a39/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
index 8c7c72e..9d954ca 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
@@ -249,6 +249,10 @@ public class OrcRawRecordMerger implements 
AcidInputFormat.RawReader{
   } while (nextRecord() != null &&
 (minKey != null && key.compareRow(getMinKey()) <= 0));
 }
+@Override
+public String toString() {
+  return "[key=" + key + ", nextRecord=" + nextRecord + ", reader=" + 
reader + "]";
+}
 @Override public final OrcStruct nextRecord() {
   return nextRecord;
 }
@@ -281,7 +285,6 @@ public class OrcRawRecordMerger implements 
AcidInputFormat.RawReader{
 OrcRecordUpdater.getRowId(nextRecord()),
 OrcRecordUpdater.getCurrentTransaction(nextRecord()),
 OrcRecordUpdater.getOperation(nextRecord()) == 
OrcRecordUpdater.DELETE_OPERATION);
-
 // if this record is larger than maxKey, we need to stop
 if (getMaxKey() != null && getKey().compareRow(getMaxKey()) > 0) {
   LOG.debug("key " + getKey() + " > maxkey " + getMaxKey());
@@ -999,7 +1002,7 @@ public class OrcRawRecordMerger implements 
AcidInputFormat.RawReader{
   

hive git commit: OrcRawRecordMerger doesn't work for more than one file in non vectorized case (Sergey Shelukhin, reviewed by Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/branch-3 1c5ce3f61 -> 7981d38cb


OrcRawRecordMerger doesn't work for more than one file in non vectorized case 
(Sergey Shelukhin, reviewed by Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7981d38c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7981d38c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7981d38c

Branch: refs/heads/branch-3
Commit: 7981d38cbf6ef43384f5fb51b1560d7cf5add729
Parents: 1c5ce3f
Author: sergey 
Authored: Sat Jun 9 09:10:15 2018 -0700
Committer: Eugene Koifman 
Committed: Sat Jun 9 09:10:15 2018 -0700

--
 .../hadoop/hive/ql/io/RecordIdentifier.java |  1 +
 .../hadoop/hive/ql/io/orc/OrcInputFormat.java   |  6 ++--
 .../hive/ql/io/orc/OrcRawRecordMerger.java  | 38 ++--
 3 files changed, 33 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/7981d38c/ql/src/java/org/apache/hadoop/hive/ql/io/RecordIdentifier.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/RecordIdentifier.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/RecordIdentifier.java
index 607abfd..ea7ba53 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/RecordIdentifier.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/RecordIdentifier.java
@@ -226,6 +226,7 @@ public class RecordIdentifier implements 
WritableComparable {
 return "{originalWriteId: " + writeId + ", " + bucketToString() + ", row: 
" + getRowId() +"}";
   }
   protected String bucketToString() {
+if (bucketId == -1) return ("bucket: " + bucketId);
 BucketCodec codec =
   BucketCodec.determineVersion(bucketId);
 return  "bucket: " + bucketId + "(" + codec.getVersion() + "." +

http://git-wip-us.apache.org/repos/asf/hive/blob/7981d38c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index 694cf75..732e233 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -2076,9 +2076,11 @@ public class OrcInputFormat implements 
InputFormat,
 String txnString = conf.get(ValidWriteIdList.VALID_WRITEIDS_KEY);
 ValidWriteIdList validWriteIdList
 = (txnString == null) ? new ValidReaderWriteIdList() : new 
ValidReaderWriteIdList(txnString);
-LOG.debug("getReader:: Read ValidWriteIdList: " + 
validWriteIdList.toString()
+if (LOG.isDebugEnabled()) {
+  LOG.debug("getReader:: Read ValidWriteIdList: " + 
validWriteIdList.toString()
 + " isTransactionalTable: " + HiveConf.getBoolVar(conf, 
ConfVars.HIVE_TRANSACTIONAL_TABLE_SCAN));
-
+  LOG.debug("Creating merger for {} and {}", split.getPath(), 
Arrays.toString(deltas));
+}
 final OrcRawRecordMerger records =
 new OrcRawRecordMerger(conf, true, reader, split.isOriginal(), bucket,
 validWriteIdList, readOptions, deltas, mergerOptions);

http://git-wip-us.apache.org/repos/asf/hive/blob/7981d38c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
index 8c7c72e..9d954ca 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
@@ -249,6 +249,10 @@ public class OrcRawRecordMerger implements 
AcidInputFormat.RawReader{
   } while (nextRecord() != null &&
 (minKey != null && key.compareRow(getMinKey()) <= 0));
 }
+@Override
+public String toString() {
+  return "[key=" + key + ", nextRecord=" + nextRecord + ", reader=" + 
reader + "]";
+}
 @Override public final OrcStruct nextRecord() {
   return nextRecord;
 }
@@ -281,7 +285,6 @@ public class OrcRawRecordMerger implements 
AcidInputFormat.RawReader{
 OrcRecordUpdater.getRowId(nextRecord()),
 OrcRecordUpdater.getCurrentTransaction(nextRecord()),
 OrcRecordUpdater.getOperation(nextRecord()) == 
OrcRecordUpdater.DELETE_OPERATION);
-
 // if this record is larger than maxKey, we need to stop
 if (getMaxKey() != null && getKey().compareRow(getMaxKey()) > 0) {
   LOG.debug("key " + getKey() + " > maxkey " + getMaxKey());
@@ -999,7 +1002,7 @@ public class OrcRawRecordMerger implements 
AcidInputFormat.RawReader{
 

hive git commit: HIVE-19750: Initialize NEXT_WRITE_ID. NWI_NEXT on converting an existing table to full acid (Eugene Koifman, reviewed by Sankar Hariappan)

[ekoifman]

als("base_1002", status[i].getPath().getName());
   Assert.assertEquals(2, buckets.length);
   Assert.assertEquals("bucket_0", buckets[0].getPath().getName());
 }
@@ -859,7 +859,7 @@ public class TestTxnCommands2 {
 status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
   (Table.NONACIDORCTBL).toString().toLowerCase()), 
FileUtils.HIDDEN_FILES_PATH_FILTER);
 Assert.assertEquals(1, status.length);
-Assert.assertEquals("base_002", status[0].getPath().getName());
+Assert.assertEquals("base_1002", status[0].getPath().getName());
 FileStatus[] buckets = fs.listStatus(status[0].getPath(), 
FileUtils.HIDDEN_FILES_PATH_FILTER);
 Arrays.sort(buckets);
 Assert.assertEquals(2, buckets.length);

http://git-wip-us.apache.org/repos/asf/hive/blob/13fbae57/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
index 11c5930..fb88f25 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
@@ -263,12 +263,18 @@ public class TestTxnLoadData extends 
TxnCommandsBaseForTests {
 {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":0}\t0\t2", 
"t/00_0"},
 {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":1}\t0\t4", 
"t/00_0"},
 //from Load Data into acid converted table
-{"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2", 
"t/delta_001_001_/00_0"},
-{"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t3\t4", 
"t/delta_001_001_/00_0"},
-{"{\"writeid\":1,\"bucketid\":536936448,\"rowid\":0}\t2\t2", 
"t/delta_001_001_/01_0"},
-{"{\"writeid\":1,\"bucketid\":536936448,\"rowid\":1}\t3\t3", 
"t/delta_001_001_/01_0"},
-{"{\"writeid\":1,\"bucketid\":537001984,\"rowid\":0}\t4\t4", 
"t/delta_001_001_/02_0"},
-{"{\"writeid\":1,\"bucketid\":537001984,\"rowid\":1}\t5\t5", 
"t/delta_001_001_/02_0"},
+{"{\"writeid\":1001,\"bucketid\":536870912,\"rowid\":0}\t1\t2",
+"t/delta_1001_1001_/00_0"},
+{"{\"writeid\":1001,\"bucketid\":536870912,\"rowid\":1}\t3\t4",
+"t/delta_1001_1001_/00_0"},
+{"{\"writeid\":1001,\"bucketid\":536936448,\"rowid\":0}\t2\t2",
+"t/delta_1001_1001_/01_0"},
+{"{\"writeid\":1001,\"bucketid\":536936448,\"rowid\":1}\t3\t3",
+"t/delta_1001_1001_/01_0"},
+{"{\"writeid\":1001,\"bucketid\":537001984,\"rowid\":0}\t4\t4",
+"t/delta_1001_1001_/02_0"},
+{"{\"writeid\":1001,\"bucketid\":537001984,\"rowid\":1}\t5\t5",
+"t/delta_1001_1001_/02_0"},
 };
 checkResult(expected, testQuery, isVectorized, "load data inpath");
 
@@ -279,9 +285,12 @@ public class TestTxnLoadData extends 
TxnCommandsBaseForTests {
 runStatementOnDriver("load data local inpath '" + getWarehouseDir() + 
"/2/data' overwrite into table T");
 
 String[][] expected2 = new String[][] {
-{"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t5\t6", 
"t/base_002/00_0"},
-{"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":1}\t7\t8", 
"t/base_002/00_0"},
-{"{\"writeid\":2,\"bucketid\":536936448,\"rowid\":0}\t8\t8", 
"t/base_002/01_0"}
+{"{\"writeid\":1002,\"bucketid\":536870912,\"rowid\":0}\t5\t6",
+"t/base_1002/00_0"},
+{"{\"writeid\":1002,\"bucketid\":536870912,\"rowid\":1}\t7\t8", 
"t/base_1002/00_0"},
+{"{\"writeid\":1002,\"bucketid\":53693

hive git commit: HIVE-18748: Rename table impacts the ACID behavior as table names are not updated in meta-tables. (Eugene Koifman, reviewed by Sankar Hariappan)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/branch-3 bf951b7e5 -> c7f713d57


HIVE-18748: Rename table impacts the ACID behavior as table names are not 
updated in meta-tables. (Eugene Koifman, reviewed by Sankar Hariappan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c7f713d5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c7f713d5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c7f713d5

Branch: refs/heads/branch-3
Commit: c7f713d572ce95d7bbc98892190c1b5ebf2627f3
Parents: bf951b7
Author: Eugene Koifman 
Authored: Tue May 29 10:45:53 2018 -0700
Committer: Eugene Koifman 
Committed: Tue May 29 10:45:53 2018 -0700

--
 .../hadoop/hive/ql/TestTxnConcatenate.java  |  61 +++
 .../hive/metastore/AcidEventListener.java   |  51 ++
 .../hadoop/hive/metastore/HiveAlterHandler.java |   9 +
 .../hadoop/hive/metastore/txn/TxnHandler.java   | 177 +++
 .../hadoop/hive/metastore/txn/TxnStore.java |   5 +
 5 files changed, 303 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/c7f713d5/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java
index 2663fec..511198a 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java
@@ -21,8 +21,11 @@ package org.apache.hadoop.hive.ql;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
 import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
 import org.apache.hadoop.hive.metastore.txn.TxnStore;
 import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
@@ -174,4 +177,62 @@ public class TestTxnConcatenate extends 
TxnCommandsBaseForTests {
 "t/base_002/00_0"}};
 checkResult(expected2, testQuery, false, "check data after concatenate", 
LOG);
   }
+  @Test
+  public void testRenameTable() throws Exception {
+MetastoreConf.setBoolVar(hiveConf, 
MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID, true);
+runStatementOnDriver("drop database if exists mydb1 cascade");
+runStatementOnDriver("drop database if exists mydb2 cascade");
+runStatementOnDriver("create database mydb1");
+runStatementOnDriver("create database mydb2");
+runStatementOnDriver("create table mydb1.T(a int, b int) stored as orc");
+runStatementOnDriver("insert into mydb1.T values(1,2),(4,5)");
+//put something in WRITE_SET
+runStatementOnDriver("update mydb1.T set b = 6 where b = 5");
+runStatementOnDriver("alter table mydb1.T compact 'minor'");
+
+runStatementOnDriver("alter table mydb1.T RENAME TO mydb1.S");
+
+String testQuery = "select ROW__ID, a, b, INPUT__FILE__NAME from mydb1.S";
+String[][] expected = new String[][] {
+{"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2",
+"s/delta_001_001_/bucket_0"},
+{"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t4\t6",
+"s/delta_002_002_/bucket_0"}};
+checkResult(expected, testQuery, false, "check data", LOG);
+
+
+Assert.assertEquals(0, TxnDbUtil.countQueryAgent(hiveConf,
+"select count(*) from COMPLETED_TXN_COMPONENTS where CTC_TABLE='t'"));
+Assert.assertEquals(0, TxnDbUtil.countQueryAgent(hiveConf,
+"select count(*) from COMPACTION_QUEUE where CQ_TABLE='t'"));
+Assert.assertEquals(0, TxnDbUtil.countQueryAgent(hiveConf,
+"select count(*) from WRITE_SET where WS_TABLE='t'"));
+Assert.assertEquals(0, TxnDbUtil.countQueryAgent(hiveConf,
+"select count(*) from TXN_TO_WRITE_ID where T2W_TABLE='t'"));
+Assert.assertEquals(0, TxnDbUtil.countQueryAgent(hiveConf,
+"select count(*) from NEXT_WRITE_ID where NWI_TABLE='t'"));
+
+Assert.assertEquals(
+TxnDbUtil.queryToString(hiveConf, "select * from 
COMPLETED_TXN_COMPONENTS"), 2,
+TxnDbUtil.countQueryAgent(hiveConf,
+"select count(*) from COMPLETED_TXN_COMPONENTS where 
CTC_TABLE='s'"));
+Assert.assertEquals(1, TxnDbUtil.countQueryAgent(hiveConf,
+"select count(*) from COMPACTION_QUEUE where CQ_TABLE='s'"));
+Assert.assertEquals(1, TxnDbUtil.countQueryAgent(hiveConf,
+"select count(*) from WRITE_SET where WS_TABLE='s'"));
+Assert.assertEquals(2, 

hive git commit: HIVE-18748 - Rename table impacts the ACID behavior as table names are not updated in meta-tables. (Eugene Koifman, reviewed by Sankar Hariappan)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master b329afa0a -> 76f3ba36a


HIVE-18748 - Rename table impacts the ACID behavior as table names are not 
updated in meta-tables. (Eugene Koifman, reviewed by Sankar Hariappan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/76f3ba36
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/76f3ba36
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/76f3ba36

Branch: refs/heads/master
Commit: 76f3ba36abc7ca2d6733a5279fc9702001fb3fdf
Parents: b329afa
Author: Eugene Koifman 
Authored: Thu May 17 09:30:47 2018 -0700
Committer: Eugene Koifman 
Committed: Thu May 17 09:30:47 2018 -0700

--
 .../hadoop/hive/ql/TestTxnConcatenate.java  |  61 +++
 .../hive/metastore/AcidEventListener.java   |  51 ++
 .../hadoop/hive/metastore/HiveAlterHandler.java |   9 +
 .../hadoop/hive/metastore/txn/TxnHandler.java   | 177 +++
 .../hadoop/hive/metastore/txn/TxnStore.java |   5 +
 5 files changed, 303 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/76f3ba36/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java
index 2663fec..511198a 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java
@@ -21,8 +21,11 @@ package org.apache.hadoop.hive.ql;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
 import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
 import org.apache.hadoop.hive.metastore.txn.TxnStore;
 import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
@@ -174,4 +177,62 @@ public class TestTxnConcatenate extends 
TxnCommandsBaseForTests {
 "t/base_002/00_0"}};
 checkResult(expected2, testQuery, false, "check data after concatenate", 
LOG);
   }
+  @Test
+  public void testRenameTable() throws Exception {
+MetastoreConf.setBoolVar(hiveConf, 
MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID, true);
+runStatementOnDriver("drop database if exists mydb1 cascade");
+runStatementOnDriver("drop database if exists mydb2 cascade");
+runStatementOnDriver("create database mydb1");
+runStatementOnDriver("create database mydb2");
+runStatementOnDriver("create table mydb1.T(a int, b int) stored as orc");
+runStatementOnDriver("insert into mydb1.T values(1,2),(4,5)");
+//put something in WRITE_SET
+runStatementOnDriver("update mydb1.T set b = 6 where b = 5");
+runStatementOnDriver("alter table mydb1.T compact 'minor'");
+
+runStatementOnDriver("alter table mydb1.T RENAME TO mydb1.S");
+
+String testQuery = "select ROW__ID, a, b, INPUT__FILE__NAME from mydb1.S";
+String[][] expected = new String[][] {
+{"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2",
+"s/delta_001_001_/bucket_0"},
+{"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t4\t6",
+"s/delta_002_002_/bucket_0"}};
+checkResult(expected, testQuery, false, "check data", LOG);
+
+
+Assert.assertEquals(0, TxnDbUtil.countQueryAgent(hiveConf,
+"select count(*) from COMPLETED_TXN_COMPONENTS where CTC_TABLE='t'"));
+Assert.assertEquals(0, TxnDbUtil.countQueryAgent(hiveConf,
+"select count(*) from COMPACTION_QUEUE where CQ_TABLE='t'"));
+Assert.assertEquals(0, TxnDbUtil.countQueryAgent(hiveConf,
+"select count(*) from WRITE_SET where WS_TABLE='t'"));
+Assert.assertEquals(0, TxnDbUtil.countQueryAgent(hiveConf,
+"select count(*) from TXN_TO_WRITE_ID where T2W_TABLE='t'"));
+Assert.assertEquals(0, TxnDbUtil.countQueryAgent(hiveConf,
+"select count(*) from NEXT_WRITE_ID where NWI_TABLE='t'"));
+
+Assert.assertEquals(
+TxnDbUtil.queryToString(hiveConf, "select * from 
COMPLETED_TXN_COMPONENTS"), 2,
+TxnDbUtil.countQueryAgent(hiveConf,
+"select count(*) from COMPLETED_TXN_COMPONENTS where 
CTC_TABLE='s'"));
+Assert.assertEquals(1, TxnDbUtil.countQueryAgent(hiveConf,
+"select count(*) from COMPACTION_QUEUE where CQ_TABLE='s'"));
+Assert.assertEquals(1, TxnDbUtil.countQueryAgent(hiveConf,
+"select count(*) from WRITE_SET where WS_TABLE='s'"));
+

[2/3] hive git commit: HIVE-18288 - merge/concat not supported on Acid table (Eugene Koifman, reviewed by Sergey Shelukhin)

[ekoifman]

HIVE-18288 - merge/concat not supported on Acid table (Eugene Koifman, reviewed 
by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cb61ed63
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cb61ed63
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cb61ed63

Branch: refs/heads/branch-3
Commit: cb61ed631c28811fc169e3a33280edd1f41f2186
Parents: a7d2fb8
Author: Eugene Koifman 
Authored: Mon May 7 12:06:59 2018 -0700
Committer: Eugene Koifman 
Committed: Mon May 7 12:06:59 2018 -0700

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |   3 +
 .../apache/hadoop/hive/ql/metadata/Hive.java|   4 +-
 .../hive/ql/parse/DDLSemanticAnalyzer.java  |  21 ++-
 .../hadoop/hive/ql/TestTxnConcatenate.java  | 159 +++
 4 files changed, 177 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/cb61ed63/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 4b8dc19..090c255 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2565,6 +2565,9 @@ public class HiveConf extends Configuration {
 COMPACTOR_JOB_QUEUE("hive.compactor.job.queue", "", "Used to specify name 
of Hadoop queue to which\n" +
   "Compaction jobs will be submitted.  Set to empty string to let Hadoop 
choose the queue."),
 
+
TRANSACTIONAL_CONCATENATE_NOBLOCK("hive.transactional.concatenate.noblock", 
false,
+"Will cause 'alter table T concatenate' to be non-blocking"),
+
 HIVE_COMPACTOR_COMPACT_MM("hive.compactor.compact.insert.only", true,
 "Whether the compactor should compact insert-only tables. A safety 
switch."),
 /**

http://git-wip-us.apache.org/repos/asf/hive/blob/cb61ed63/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java 
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 5ab86b4..3218f96 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -4405,9 +4405,9 @@ private void constructOneLBLocationMap(FileStatus fSta,
   throws HiveException {
 try {
   CompactionType cr = null;
-  if ("major".equals(compactType)) {
+  if ("major".equalsIgnoreCase(compactType)) {
 cr = CompactionType.MAJOR;
-  } else if ("minor".equals(compactType)) {
+  } else if ("minor".equalsIgnoreCase(compactType)) {
 cr = CompactionType.MINOR;
   } else {
 throw new RuntimeException("Unknown compaction type " + compactType);

http://git-wip-us.apache.org/repos/asf/hive/blob/cb61ed63/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index defb8be..f0b9eda 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -1964,9 +1964,19 @@ public class DDLSemanticAnalyzer extends 
BaseSemanticAnalyzer {
 
 try {
   tblObj = getTable(tableName);
-  // TODO: we should probably block all ACID tables here.
-  if (AcidUtils.isInsertOnlyTable(tblObj.getParameters())) {
-throw new SemanticException("Merge is not supported for MM tables");
+  if(AcidUtils.isTransactionalTable(tblObj)) {
+LinkedHashMap newPartSpec = null;
+if (partSpec != null) {
+  newPartSpec = new LinkedHashMap<>(partSpec);
+}
+
+boolean isBlocking = !HiveConf.getBoolVar(conf,
+ConfVars.TRANSACTIONAL_CONCATENATE_NOBLOCK, false);
+AlterTableSimpleDesc desc = new AlterTableSimpleDesc(
+tableName, newPartSpec, "MAJOR", isBlocking);
+
+rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), 
desc)));
+return;
   }
   mergeDesc.setTableDesc(Utilities.getTableDesc(tblObj));
 
@@ -2039,11 +2049,6 @@ public class DDLSemanticAnalyzer extends 
BaseSemanticAnalyzer {
 throw new 
SemanticException(ErrorMsg.CONCATENATE_UNSUPPORTED_TABLE_NOT_MANAGED.getMsg());
   }
 
-  // transactional tables are compacted and no longer needs to be 
bucketed, so not safe for merge/concatenation
-

[1/3] hive git commit: HIVE-18131 - Truncate table for Acid tables (Eugene Koifman, reviewed by Prasanth Jayachandran)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/branch-3 f93f76d18 -> 63abf9be0


HIVE-18131 - Truncate table for Acid tables (Eugene Koifman, reviewed by 
Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a7d2fb80
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a7d2fb80
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a7d2fb80

Branch: refs/heads/branch-3
Commit: a7d2fb805ce8725525651c54804511ffe5dfbaee
Parents: f93f76d
Author: Eugene Koifman 
Authored: Mon May 7 12:05:19 2018 -0700
Committer: Eugene Koifman 
Committed: Mon May 7 12:05:19 2018 -0700

--
 .../hadoop/hive/ql/lockmgr/TestDbTxnManager2.java| 15 +++
 1 file changed, 15 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/a7d2fb80/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
--
diff --git 
a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java 
b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index 1d79242..e06f0a4 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -2512,4 +2512,19 @@ public class TestDbTxnManager2 {
 Assert.assertEquals("Unexpected lock count", 1, locks.size());
 checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T", null, 
locks);
   }
+  @Test
+  public void testTruncate() throws Exception {
+dropTable(new String[] {"T"});
+CommandProcessorResponse cpr = driver.run("create table T (a int, b int) 
stored as" +
+" orc tblproperties('transactional'='true')");
+checkCmdOnDriver(cpr);
+checkCmdOnDriver(driver.run("insert into T values(0,2),(1,4)"));
+checkCmdOnDriver(driver.run("truncate table T"));
+cpr = driver.compileAndRespond("truncate table T");
+checkCmdOnDriver(cpr);
+txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");//gets X lock on T
+List locks = getLocks();
+Assert.assertEquals("Unexpected lock count", 1, locks.size());
+checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T", null, 
locks);
+  }
 }

[3/3] hive git commit: HIVE-19322 - broken test: TestNegativeMinimrCliDriver#testCliDriver[minimr_broken_pipe] (Eugene Koifman, reviewed by Jason Dere)

[ekoifman]

HIVE-19322 - broken test: 
TestNegativeMinimrCliDriver#testCliDriver[minimr_broken_pipe] (Eugene Koifman, 
reviewed by Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/63abf9be
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/63abf9be
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/63abf9be

Branch: refs/heads/branch-3
Commit: 63abf9be053c2f3a4ff524b7ab2b1c17248decb5
Parents: cb61ed6
Author: Eugene Koifman 
Authored: Mon May 7 12:08:35 2018 -0700
Committer: Eugene Koifman 
Committed: Mon May 7 12:08:35 2018 -0700

--
 .../ql/cache/results/QueryResultsCache.java |  2 +-
 .../ql/metadata/SessionHiveMetaStoreClient.java | 66 +---
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  3 +-
 .../ql/parse/repl/dump/PartitionExport.java |  9 ++-
 .../hadoop/hive/ql/session/SessionState.java| 14 +++--
 .../check_constraint_tbl_level.q.out|  1 -
 .../check_constraint_violation.q.out|  1 -
 .../merge_constraint_notnull.q.out  |  1 -
 8 files changed, 63 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/63abf9be/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java 
b/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
index 56a9faa..6734ac5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
@@ -687,7 +687,7 @@ public final class QueryResultsCache {
   if (readEntity.getType() == Type.TABLE) {
 Table tableUsed = readEntity.getTable();
 Map tempTables =
-
SessionHiveMetaStoreClient.getTempTablesForDatabase(tableUsed.getDbName());
+
SessionHiveMetaStoreClient.getTempTablesForDatabase(tableUsed.getDbName(), 
tableUsed.getTableName());
 if (tempTables != null && 
tempTables.containsKey(tableUsed.getTableName())) {
   LOG.info("{} resolves to a temporary table in the current session. 
This query cannot use the cache.",
   tableUsed.getTableName());

http://git-wip-us.apache.org/repos/asf/hive/blob/63abf9be/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
index d89df48..071756d 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
@@ -68,10 +68,22 @@ import org.apache.hadoop.hive.shims.HadoopShims;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.hive.ql.stats.StatsUtils;
 import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
 
+/**
+ * todo: This need review re: thread safety.  Various places (see callsers of
+ * {@link SessionState#setCurrentSessionState(SessionState)}) pass 
SessionState to forked threads.
+ * Currently it looks like those threads only read metadata but this is 
fragile.
+ * Also, maps (in SessionState) where tempt table metadata is stored are 
concurrent and so
+ * any put/get crosses a memory barrier and so does using most {@code 
java.util.concurrent.*}
+ * so the readers of the objects in these maps should have the most recent 
view of the object.
+ * But again, could be fragile.
+ */
 public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements 
IMetaStoreClient {
+  private static final Logger LOG = 
LoggerFactory.getLogger(SessionHiveMetaStoreClient.class);
 
   SessionHiveMetaStoreClient(Configuration conf, Boolean allowEmbedded) throws 
MetaException {
 super(conf, null, allowEmbedded);
@@ -174,7 +186,7 @@ public class SessionHiveMetaStoreClient extends 
HiveMetaStoreClient implements I
 List tableNames = super.getAllTables(dbName);
 
 // May need to merge with list of temp tables
-Map tables = getTempTablesForDatabase(dbName);
+Map tables = getTempTablesForDatabase(dbName, "?");
 if (tables == null || tables.size() == 0) {
   return tableNames;
 }
@@ -198,7 +210,7 @@ public class SessionHiveMetaStoreClient extends 
HiveMetaStoreClient implements I
 // May need to merge with list of temp tables
 dbName 

[1/2] hive git commit: HIVE-18131 - Truncate table for Acid tables (Eugene Koifman, reviewed by Prasanth Jayachandran)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master c38b7f88e -> 4a2bfb8be


HIVE-18131 - Truncate table for Acid tables (Eugene Koifman, reviewed by 
Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ab7affea
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ab7affea
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ab7affea

Branch: refs/heads/master
Commit: ab7affea11d6583aabeb471939771e215ac44f48
Parents: c38b7f8
Author: Eugene Koifman 
Authored: Mon May 7 11:52:33 2018 -0700
Committer: Eugene Koifman 
Committed: Mon May 7 11:52:33 2018 -0700

--
 .../hadoop/hive/ql/lockmgr/TestDbTxnManager2.java| 15 +++
 1 file changed, 15 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/ab7affea/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
--
diff --git 
a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java 
b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index 1d79242..e06f0a4 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -2512,4 +2512,19 @@ public class TestDbTxnManager2 {
 Assert.assertEquals("Unexpected lock count", 1, locks.size());
 checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T", null, 
locks);
   }
+  @Test
+  public void testTruncate() throws Exception {
+dropTable(new String[] {"T"});
+CommandProcessorResponse cpr = driver.run("create table T (a int, b int) 
stored as" +
+" orc tblproperties('transactional'='true')");
+checkCmdOnDriver(cpr);
+checkCmdOnDriver(driver.run("insert into T values(0,2),(1,4)"));
+checkCmdOnDriver(driver.run("truncate table T"));
+cpr = driver.compileAndRespond("truncate table T");
+checkCmdOnDriver(cpr);
+txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");//gets X lock on T
+List locks = getLocks();
+Assert.assertEquals("Unexpected lock count", 1, locks.size());
+checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T", null, 
locks);
+  }
 }

[2/2] hive git commit: HIVE-18288 - merge/concat not supported on Acid table (Eugene Koifman, reviewed by Sergey Shelukhin)

[ekoifman]

HIVE-18288 - merge/concat not supported on Acid table (Eugene Koifman, reviewed 
by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4a2bfb8b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4a2bfb8b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4a2bfb8b

Branch: refs/heads/master
Commit: 4a2bfb8bed36b53b1bcc4eee0e4f916820f335f4
Parents: ab7affe
Author: Eugene Koifman 
Authored: Mon May 7 11:55:16 2018 -0700
Committer: Eugene Koifman 
Committed: Mon May 7 11:55:16 2018 -0700

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |   3 +
 .../apache/hadoop/hive/ql/metadata/Hive.java|   4 +-
 .../hive/ql/parse/DDLSemanticAnalyzer.java  |  21 ++-
 .../hadoop/hive/ql/TestTxnConcatenate.java  | 159 +++
 4 files changed, 177 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/4a2bfb8b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index b872827..23a9c74 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2565,6 +2565,9 @@ public class HiveConf extends Configuration {
 COMPACTOR_JOB_QUEUE("hive.compactor.job.queue", "", "Used to specify name 
of Hadoop queue to which\n" +
   "Compaction jobs will be submitted.  Set to empty string to let Hadoop 
choose the queue."),
 
+
TRANSACTIONAL_CONCATENATE_NOBLOCK("hive.transactional.concatenate.noblock", 
false,
+"Will cause 'alter table T concatenate' to be non-blocking"),
+
 HIVE_COMPACTOR_COMPACT_MM("hive.compactor.compact.insert.only", true,
 "Whether the compactor should compact insert-only tables. A safety 
switch."),
 /**

http://git-wip-us.apache.org/repos/asf/hive/blob/4a2bfb8b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java 
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 5dbc478..903470a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -4407,9 +4407,9 @@ private void constructOneLBLocationMap(FileStatus fSta,
   throws HiveException {
 try {
   CompactionType cr = null;
-  if ("major".equals(compactType)) {
+  if ("major".equalsIgnoreCase(compactType)) {
 cr = CompactionType.MAJOR;
-  } else if ("minor".equals(compactType)) {
+  } else if ("minor".equalsIgnoreCase(compactType)) {
 cr = CompactionType.MINOR;
   } else {
 throw new RuntimeException("Unknown compaction type " + compactType);

http://git-wip-us.apache.org/repos/asf/hive/blob/4a2bfb8b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index defb8be..f0b9eda 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -1964,9 +1964,19 @@ public class DDLSemanticAnalyzer extends 
BaseSemanticAnalyzer {
 
 try {
   tblObj = getTable(tableName);
-  // TODO: we should probably block all ACID tables here.
-  if (AcidUtils.isInsertOnlyTable(tblObj.getParameters())) {
-throw new SemanticException("Merge is not supported for MM tables");
+  if(AcidUtils.isTransactionalTable(tblObj)) {
+LinkedHashMap newPartSpec = null;
+if (partSpec != null) {
+  newPartSpec = new LinkedHashMap<>(partSpec);
+}
+
+boolean isBlocking = !HiveConf.getBoolVar(conf,
+ConfVars.TRANSACTIONAL_CONCATENATE_NOBLOCK, false);
+AlterTableSimpleDesc desc = new AlterTableSimpleDesc(
+tableName, newPartSpec, "MAJOR", isBlocking);
+
+rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), 
desc)));
+return;
   }
   mergeDesc.setTableDesc(Utilities.getTableDesc(tblObj));
 
@@ -2039,11 +2049,6 @@ public class DDLSemanticAnalyzer extends 
BaseSemanticAnalyzer {
 throw new 
SemanticException(ErrorMsg.CONCATENATE_UNSUPPORTED_TABLE_NOT_MANAGED.getMsg());
   }
 
-  // transactional tables are compacted and no longer needs to be 
bucketed, so not safe for merge/concatenation
-  

hive git commit: HIVE-18570 - ACID IOW implemented using base may delete too much data (Eugene Koifman, reviewed by Sergey Shelukhin)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 552dbaf33 -> 8158f8848


HIVE-18570 - ACID IOW implemented using base may delete too much data (Eugene 
Koifman, reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8158f884
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8158f884
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8158f884

Branch: refs/heads/master
Commit: 8158f884854a4294d7444a393f28c19d2860
Parents: 552dbaf
Author: Eugene Koifman 
Authored: Fri May 4 13:22:07 2018 -0700
Committer: Eugene Koifman 
Committed: Fri May 4 13:22:07 2018 -0700

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  4 ++
 .../hadoop/hive/ql/hooks/WriteEntity.java   | 13 +++--
 .../hadoop/hive/ql/lockmgr/DbTxnManager.java|  6 +-
 .../apache/hadoop/hive/ql/metadata/Table.java   | 12 ++--
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  2 +-
 .../hive/ql/lockmgr/TestDbTxnManager2.java  | 58 +---
 .../alter_rename_partition_failure3.q.out   |  2 +-
 .../alter_table_add_partition.q.out |  2 +-
 .../clientnegative/alter_view_failure5.q.out|  2 +-
 .../clientnegative/alter_view_failure7.q.out|  2 +-
 .../test/results/clientnegative/analyze1.q.out  |  2 +-
 .../clientnegative/archive_partspec1.q.out  |  2 +-
 .../test/results/clientnegative/dyn_part1.q.out |  2 +-
 ql/src/test/results/clientnegative/touch2.q.out |  2 +-
 .../truncate_partition_column.q.out |  2 +-
 15 files changed, 84 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/8158f884/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 844a200..7628c91 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2436,6 +2436,10 @@ public class HiveConf extends Configuration {
 "In nonstrict mode, for non-ACID resources, INSERT will only acquire 
shared lock, which\n" +
 "allows two concurrent writes to the same partition but still lets 
lock manager prevent\n" +
 "DROP TABLE etc. when the table is being written to"),
+TXN_OVERWRITE_X_LOCK("hive.txn.xlock.iow", true,
+"Ensures commands with OVERWRITE (such as INSERT OVERWRITE) acquire 
Exclusive locks for\b" +
+"transactional tables.  This ensures that inserts (w/o overwrite) 
running concurrently\n" +
+"are not hidden by the INSERT OVERWRITE."),
 /**
  * @deprecated Use MetastoreConf.TXN_TIMEOUT
  */

http://git-wip-us.apache.org/repos/asf/hive/blob/8158f884/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java 
b/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
index c465adf..f1cf113 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
@@ -63,7 +63,7 @@ public class WriteEntity extends Entity implements 
Serializable {
 
   public WriteEntity(Database database, WriteType type) {
 super(database, true);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   /**
@@ -74,12 +74,12 @@ public class WriteEntity extends Entity implements 
Serializable {
*/
   public WriteEntity(Table t, WriteType type) {
 super(t, true);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   public WriteEntity(Table t, WriteType type, boolean complete) {
 super(t, complete);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   /**
@@ -104,12 +104,12 @@ public class WriteEntity extends Entity implements 
Serializable {
*/
   public WriteEntity(Partition p, WriteType type) {
 super(p, true);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   public WriteEntity(DummyPartition p, WriteType type, boolean complete) {
 super(p, complete);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   /**
@@ -161,6 +161,9 @@ public class WriteEntity extends Entity implements 
Serializable {
* @param type new operation type
*/
   public void setWriteType(WriteType type) {
+setWriteTypeInternal(type);
+  }
+  private void setWriteTypeInternal(WriteType type) {
 writeType = type;
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/8158f884/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java

hive git commit: HIVE-18570 - ACID IOW implemented using base may delete too much data (Eugene Koifman, reviewed by Sergey Shelukhin)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/branch-3 1ad644c60 -> 812ae7388


HIVE-18570 - ACID IOW implemented using base may delete too much data (Eugene 
Koifman, reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/812ae738
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/812ae738
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/812ae738

Branch: refs/heads/branch-3
Commit: 812ae7388936d3a8861304a66eb1f448b126e408
Parents: 1ad644c
Author: Eugene Koifman 
Authored: Fri May 4 13:21:00 2018 -0700
Committer: Eugene Koifman 
Committed: Fri May 4 13:21:00 2018 -0700

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  4 ++
 .../hadoop/hive/ql/hooks/WriteEntity.java   | 13 +++--
 .../hadoop/hive/ql/lockmgr/DbTxnManager.java|  6 +-
 .../apache/hadoop/hive/ql/metadata/Table.java   | 12 ++--
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  2 +-
 .../hive/ql/lockmgr/TestDbTxnManager2.java  | 58 +---
 .../alter_rename_partition_failure3.q.out   |  2 +-
 .../alter_table_add_partition.q.out |  2 +-
 .../clientnegative/alter_view_failure5.q.out|  2 +-
 .../clientnegative/alter_view_failure7.q.out|  2 +-
 .../test/results/clientnegative/analyze1.q.out  |  2 +-
 .../clientnegative/archive_partspec1.q.out  |  2 +-
 .../test/results/clientnegative/dyn_part1.q.out |  2 +-
 ql/src/test/results/clientnegative/touch2.q.out |  2 +-
 .../truncate_partition_column.q.out |  2 +-
 15 files changed, 84 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/812ae738/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 9a0d8a9..8c6cb61 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2436,6 +2436,10 @@ public class HiveConf extends Configuration {
 "In nonstrict mode, for non-ACID resources, INSERT will only acquire 
shared lock, which\n" +
 "allows two concurrent writes to the same partition but still lets 
lock manager prevent\n" +
 "DROP TABLE etc. when the table is being written to"),
+TXN_OVERWRITE_X_LOCK("hive.txn.xlock.iow", true,
+"Ensures commands with OVERWRITE (such as INSERT OVERWRITE) acquire 
Exclusive locks for\b" +
+"transactional tables.  This ensures that inserts (w/o overwrite) 
running concurrently\n" +
+"are not hidden by the INSERT OVERWRITE."),
 /**
  * @deprecated Use MetastoreConf.TXN_TIMEOUT
  */

http://git-wip-us.apache.org/repos/asf/hive/blob/812ae738/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java 
b/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
index c465adf..f1cf113 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
@@ -63,7 +63,7 @@ public class WriteEntity extends Entity implements 
Serializable {
 
   public WriteEntity(Database database, WriteType type) {
 super(database, true);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   /**
@@ -74,12 +74,12 @@ public class WriteEntity extends Entity implements 
Serializable {
*/
   public WriteEntity(Table t, WriteType type) {
 super(t, true);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   public WriteEntity(Table t, WriteType type, boolean complete) {
 super(t, complete);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   /**
@@ -104,12 +104,12 @@ public class WriteEntity extends Entity implements 
Serializable {
*/
   public WriteEntity(Partition p, WriteType type) {
 super(p, true);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   public WriteEntity(DummyPartition p, WriteType type, boolean complete) {
 super(p, complete);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   /**
@@ -161,6 +161,9 @@ public class WriteEntity extends Entity implements 
Serializable {
* @param type new operation type
*/
   public void setWriteType(WriteType type) {
+setWriteTypeInternal(type);
+  }
+  private void setWriteTypeInternal(WriteType type) {
 writeType = type;
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/812ae738/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java

hive git commit: HIVE-17457 - IOW Acid Insert Overwrite when the transaction fails (Eugene Koifman, reviewed by Sergey Shelukhin)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/branch-3 ae4df6279 -> 9adf5a950


HIVE-17457 - IOW Acid Insert Overwrite when the transaction fails (Eugene 
Koifman, reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9adf5a95
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9adf5a95
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9adf5a95

Branch: refs/heads/branch-3
Commit: 9adf5a95069ae5bbd65ce90b779c8dab180e0e70
Parents: ae4df62
Author: Eugene Koifman 
Authored: Thu May 3 09:23:30 2018 -0700
Committer: Eugene Koifman 
Committed: Thu May 3 09:23:30 2018 -0700

--
 .../hadoop/hive/ql/lockmgr/DbTxnManager.java |  2 +-
 .../apache/hadoop/hive/ql/TestTxnCommands.java   | 19 +--
 2 files changed, 18 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/9adf5a95/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java 
b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
index 68b4c3b..94f6b00 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
@@ -361,7 +361,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
 return true;
   case INSERT_OVERWRITE:
 //see HIVE-18154
-return false; // TODO: is this still relevant for insert-only 
tables?
+return false;
   default:
 //not relevant for LOAD
 return false;

http://git-wip-us.apache.org/repos/asf/hive/blob/9adf5a95/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index 6a3be39..6faba42 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@ -74,11 +74,26 @@ public class TestTxnCommands extends 
TxnCommandsBaseForTests {
 return TEST_DATA_DIR;
   }
 
-  @Test//todo: what is this for?
+  /**
+   * tests that a failing Insert Overwrite (which creates a new base_x) is 
properly marked as
+   * aborted.
+   */
+  @Test
   public void testInsertOverwrite() throws Exception {
 runStatementOnDriver("insert overwrite table " + Table.NONACIDORCTBL + " 
select a,b from " + Table.NONACIDORCTBL2);
 runStatementOnDriver("create table " + Table.NONACIDORCTBL2 + "3(a int, b 
int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc 
TBLPROPERTIES ('transactional'='false')");
-
+runStatementOnDriver("insert into " + Table.ACIDTBL + " values(1,2)");
+List rs = runStatementOnDriver("select a from " + Table.ACIDTBL + 
" where b = 2");
+Assert.assertEquals(1, rs.size());
+Assert.assertEquals("1", rs.get(0));
+hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, true);
+runStatementOnDriver("insert into " + Table.ACIDTBL + " values(3,2)");
+hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, false);
+runStatementOnDriver("insert into " + Table.ACIDTBL + " values(5,6)");
+rs = runStatementOnDriver("select a from " + Table.ACIDTBL + " order by 
a");
+Assert.assertEquals(2, rs.size());
+Assert.assertEquals("1", rs.get(0));
+Assert.assertEquals("5", rs.get(1));
   }
   @Ignore("not needed but useful for testing")
   @Test

hive git commit: HIVE-17457 - IOW Acid Insert Overwrite when the transaction fails (Eugene Koifman, reviewed by Sergey Shelukhin)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 2c7f9c26e -> dfaf90f2b


HIVE-17457 - IOW Acid Insert Overwrite when the transaction fails (Eugene 
Koifman, reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dfaf90f2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dfaf90f2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dfaf90f2

Branch: refs/heads/master
Commit: dfaf90f2b3a69477ea6b38144cf4ed55de9c4d95
Parents: 2c7f9c2
Author: Eugene Koifman 
Authored: Thu May 3 09:22:41 2018 -0700
Committer: Eugene Koifman 
Committed: Thu May 3 09:22:41 2018 -0700

--
 .../hadoop/hive/ql/lockmgr/DbTxnManager.java |  2 +-
 .../apache/hadoop/hive/ql/TestTxnCommands.java   | 19 +--
 2 files changed, 18 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/dfaf90f2/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java 
b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
index 68b4c3b..94f6b00 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
@@ -361,7 +361,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
 return true;
   case INSERT_OVERWRITE:
 //see HIVE-18154
-return false; // TODO: is this still relevant for insert-only 
tables?
+return false;
   default:
 //not relevant for LOAD
 return false;

http://git-wip-us.apache.org/repos/asf/hive/blob/dfaf90f2/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index 6a3be39..6faba42 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@ -74,11 +74,26 @@ public class TestTxnCommands extends 
TxnCommandsBaseForTests {
 return TEST_DATA_DIR;
   }
 
-  @Test//todo: what is this for?
+  /**
+   * tests that a failing Insert Overwrite (which creates a new base_x) is 
properly marked as
+   * aborted.
+   */
+  @Test
   public void testInsertOverwrite() throws Exception {
 runStatementOnDriver("insert overwrite table " + Table.NONACIDORCTBL + " 
select a,b from " + Table.NONACIDORCTBL2);
 runStatementOnDriver("create table " + Table.NONACIDORCTBL2 + "3(a int, b 
int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc 
TBLPROPERTIES ('transactional'='false')");
-
+runStatementOnDriver("insert into " + Table.ACIDTBL + " values(1,2)");
+List rs = runStatementOnDriver("select a from " + Table.ACIDTBL + 
" where b = 2");
+Assert.assertEquals(1, rs.size());
+Assert.assertEquals("1", rs.get(0));
+hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, true);
+runStatementOnDriver("insert into " + Table.ACIDTBL + " values(3,2)");
+hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, false);
+runStatementOnDriver("insert into " + Table.ACIDTBL + " values(5,6)");
+rs = runStatementOnDriver("select a from " + Table.ACIDTBL + " order by 
a");
+Assert.assertEquals(2, rs.size());
+Assert.assertEquals("1", rs.get(0));
+Assert.assertEquals("5", rs.get(1));
   }
   @Ignore("not needed but useful for testing")
   @Test

hive git commit: HIVE-19322 - broken test: TestNegativeMinimrCliDriver#testCliDriver[minimr_broken_pipe] (Eugene Koifman, reviewed by Jason Dere)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 3c4a9c6bc -> ee6a53268


HIVE-19322 - broken test: 
TestNegativeMinimrCliDriver#testCliDriver[minimr_broken_pipe] (Eugene Koifman, 
reviewed by Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ee6a5326
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ee6a5326
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ee6a5326

Branch: refs/heads/master
Commit: ee6a53268989a823bb4a40638d9c93c52658f76c
Parents: 3c4a9c6
Author: Eugene Koifman 
Authored: Mon Apr 30 17:53:45 2018 -0700
Committer: Eugene Koifman 
Committed: Mon Apr 30 17:53:45 2018 -0700

--
 .../ql/cache/results/QueryResultsCache.java |  2 +-
 .../ql/metadata/SessionHiveMetaStoreClient.java | 66 +---
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  3 +-
 .../ql/parse/repl/dump/PartitionExport.java |  9 ++-
 .../hadoop/hive/ql/session/SessionState.java| 14 +++--
 .../avro_non_nullable_union.q.out   |  1 -
 .../clientnegative/cachingprintstream.q.out |  2 -
 .../check_constraint_tbl_level.q.out|  1 -
 .../check_constraint_violation.q.out|  1 -
 .../clientnegative/compute_stats_long.q.out |  1 -
 .../test/results/clientnegative/dyn_part3.q.out |  1 -
 .../clientnegative/dyn_part_max_per_node.q.out  |  1 -
 .../dynamic_partitions_with_whitelist.q.out |  1 -
 .../results/clientnegative/insertsel_fail.q.out |  2 +-
 .../local_mapred_error_cache.q.out  |  1 -
 .../merge_constraint_notnull.q.out  |  1 -
 .../clientnegative/script_broken_pipe2.q.out|  1 -
 .../clientnegative/script_broken_pipe3.q.out|  1 -
 .../results/clientnegative/script_error.q.out   |  1 -
 .../results/clientnegative/serde_regex2.q.out   |  1 -
 .../stats_aggregator_error_2.q.out  |  1 -
 .../stats_publisher_error_1.q.out   |  1 -
 .../stats_publisher_error_2.q.out   |  1 -
 .../clientnegative/subquery_corr_in_agg.q.out   |  1 -
 .../subquery_in_implicit_gby.q.out  |  1 -
 .../subquery_notin_implicit_gby.q.out   |  1 -
 .../subquery_scalar_corr_multi_rows.q.out   |  1 -
 .../subquery_scalar_multi_rows.q.out|  1 -
 .../clientnegative/udf_assert_true.q.out|  1 -
 .../clientnegative/udf_assert_true2.q.out   |  1 -
 .../clientnegative/udf_reflect_neg.q.out|  1 -
 .../results/clientnegative/udf_test_error.q.out |  1 -
 .../clientnegative/udf_test_error_reduce.q.out  |  1 -
 33 files changed, 64 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/ee6a5326/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java 
b/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
index 56a9faa..6734ac5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
@@ -687,7 +687,7 @@ public final class QueryResultsCache {
   if (readEntity.getType() == Type.TABLE) {
 Table tableUsed = readEntity.getTable();
 Map tempTables =
-
SessionHiveMetaStoreClient.getTempTablesForDatabase(tableUsed.getDbName());
+
SessionHiveMetaStoreClient.getTempTablesForDatabase(tableUsed.getDbName(), 
tableUsed.getTableName());
 if (tempTables != null && 
tempTables.containsKey(tableUsed.getTableName())) {
   LOG.info("{} resolves to a temporary table in the current session. 
This query cannot use the cache.",
   tableUsed.getTableName());

http://git-wip-us.apache.org/repos/asf/hive/blob/ee6a5326/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
index d89df48..071756d 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
@@ -68,10 +68,22 @@ import org.apache.hadoop.hive.shims.HadoopShims;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.hive.ql.stats.StatsUtils;
 import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
 
+/**
+ * todo: This need review re: 

hive git commit: HIVE-19270 - TestAcidOnTez tests are failing (Eugene Koifman, reviewed by Sankar Hariappan)

[ekoifman]

t;/delta_001_001_0001/bucket_0"},
+  {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":2}\t1\t2", 
"/delta_001_001_0001/bucket_0"},
   {"{\"writeid\":1,\"bucketid\":536870914,\"rowid\":0}\t9\t10", 
"/delta_001_001_0002/bucket_0"},
   {"{\"writeid\":1,\"bucketid\":536870914,\"rowid\":1}\t7\t8", 
"/delta_001_001_0002/bucket_0"},
   {"{\"writeid\":1,\"bucketid\":536870914,\"rowid\":2}\t5\t6", 
"/delta_001_001_0002/bucket_0"},
@@ -476,8 +476,8 @@ public class TestAcidOnTez {
   LOG.warn(s);
 }
 String[][] expected2 = {
-  {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":0}\t1\t2", 
"/delta_001_001_0001/bucket_0"},
   {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":1}\t3\t4", 
"/delta_001_001_0001/bucket_0"},
+  {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":2}\t1\t2", 
"/delta_001_001_0001/bucket_0"},
   {"{\"writeid\":1,\"bucketid\":536870914,\"rowid\":0}\t9\t10", 
"/delta_001_001_0002/bucket_0"},
   {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t70\t80", 
"/delta_002_002_/bucket_0"}
 };
@@ -661,8 +661,8 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree  
~/dev/hiverwgit/itests/h
 }
 
 String[][] expected2 = {
-  {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":0}\t1\t2", 
"warehouse/t/delta_001_001_0001/bucket_0"},
-  {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":1}\t3\t4", 
"warehouse/t/delta_001_001_0001/bucket_0"},
+  {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":1}\t1\t2", 
"warehouse/t/delta_001_001_0001/bucket_0"},
+  {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":0}\t3\t4", 
"warehouse/t/delta_001_001_0001/bucket_0"},
   {"{\"writeid\":1,\"bucketid\":536870914,\"rowid\":0}\t5\t6", 
"warehouse/t/delta_001_001_0002/bucket_0"},
   {"{\"writeid\":1,\"bucketid\":536870914,\"rowid\":1}\t7\t8", 
"warehouse/t/delta_001_001_0002/bucket_0"},
   {"{\"writeid\":1,\"bucketid\":536870915,\"rowid\":0}\t9\t10", 
"warehouse/t/delta_001_001_0003/bucket_0"}

hive git commit: HIVE-19270 - TestAcidOnTez tests are failing (Eugene Koifman, reviewed by Sankar Hariappan)

[ekoifman]

t;/delta_001_001_0001/bucket_0"},
+  {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":2}\t1\t2", 
"/delta_001_001_0001/bucket_0"},
   {"{\"writeid\":1,\"bucketid\":536870914,\"rowid\":0}\t9\t10", 
"/delta_001_001_0002/bucket_0"},
   {"{\"writeid\":1,\"bucketid\":536870914,\"rowid\":1}\t7\t8", 
"/delta_001_001_0002/bucket_0"},
   {"{\"writeid\":1,\"bucketid\":536870914,\"rowid\":2}\t5\t6", 
"/delta_001_001_0002/bucket_0"},
@@ -476,8 +476,8 @@ public class TestAcidOnTez {
   LOG.warn(s);
 }
 String[][] expected2 = {
-  {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":0}\t1\t2", 
"/delta_001_001_0001/bucket_0"},
   {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":1}\t3\t4", 
"/delta_001_001_0001/bucket_0"},
+  {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":2}\t1\t2", 
"/delta_001_001_0001/bucket_0"},
   {"{\"writeid\":1,\"bucketid\":536870914,\"rowid\":0}\t9\t10", 
"/delta_001_001_0002/bucket_0"},
   {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t70\t80", 
"/delta_002_002_/bucket_0"}
 };
@@ -661,8 +661,8 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree  
~/dev/hiverwgit/itests/h
 }
 
 String[][] expected2 = {
-  {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":0}\t1\t2", 
"warehouse/t/delta_001_001_0001/bucket_0"},
-  {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":1}\t3\t4", 
"warehouse/t/delta_001_001_0001/bucket_0"},
+  {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":1}\t1\t2", 
"warehouse/t/delta_001_001_0001/bucket_0"},
+  {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":0}\t3\t4", 
"warehouse/t/delta_001_001_0001/bucket_0"},
   {"{\"writeid\":1,\"bucketid\":536870914,\"rowid\":0}\t5\t6", 
"warehouse/t/delta_001_001_0002/bucket_0"},
   {"{\"writeid\":1,\"bucketid\":536870914,\"rowid\":1}\t7\t8", 
"warehouse/t/delta_001_001_0002/bucket_0"},
   {"{\"writeid\":1,\"bucketid\":536870915,\"rowid\":0}\t9\t10", 
"warehouse/t/delta_001_001_0003/bucket_0"}

hive git commit: HIVE-19315 - Test failure org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2#testWriteSetTracking3 (Eugene Koifman, reviewed by Jason Dere)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/branch-3 3edde33d5 -> b0bc75b3c


HIVE-19315 - Test failure 
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2#testWriteSetTracking3 
(Eugene Koifman, reviewed by Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b0bc75b3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b0bc75b3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b0bc75b3

Branch: refs/heads/branch-3
Commit: b0bc75b3cc4a0bca37b24a3997e09c7bbf9eee23
Parents: 3edde33
Author: Eugene Koifman 
Authored: Thu Apr 26 10:38:20 2018 -0700
Committer: Eugene Koifman 
Committed: Thu Apr 26 10:40:28 2018 -0700

--
 .../test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/b0bc75b3/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
--
diff --git 
a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java 
b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index 3c2f9d4..9c32dd5 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -992,7 +992,7 @@ public class TestDbTxnManager2 {
 Assert.assertTrue("Didn't get exception", expectedException != null);
 Assert.assertEquals("Got wrong message code", ErrorMsg.TXN_ABORTED, 
expectedException.getCanonicalErrorMsg());
 Assert.assertEquals("Exception msg didn't match",
-  "Aborting [txnid:"+txnId2+","+txnId2+"] due to a write conflict on 
default/TAB_PART/p=blah committed by [txnid:"+txnId+","+txnId2+"] u/u",
+  "Aborting [txnid:"+txnId2+","+txnId2+"] due to a write conflict on 
default/tab_part/p=blah committed by [txnid:"+txnId+","+txnId2+"] u/u",
   expectedException.getCause().getMessage());
   }
   /**

hive git commit: HIVE-19315 - Test failure org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2#testWriteSetTracking3 (Eugene Koifman, reviewed by Jason Dere)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 823aadad5 -> 913911b86


HIVE-19315 - Test failure 
org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2#testWriteSetTracking3 
(Eugene Koifman, reviewed by Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/913911b8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/913911b8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/913911b8

Branch: refs/heads/master
Commit: 913911b8659ac443bff1f583318eed7e0a9a8025
Parents: 823aada
Author: Eugene Koifman 
Authored: Thu Apr 26 10:38:20 2018 -0700
Committer: Eugene Koifman 
Committed: Thu Apr 26 10:38:33 2018 -0700

--
 .../test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/913911b8/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
--
diff --git 
a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java 
b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index 3c2f9d4..9c32dd5 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -992,7 +992,7 @@ public class TestDbTxnManager2 {
 Assert.assertTrue("Didn't get exception", expectedException != null);
 Assert.assertEquals("Got wrong message code", ErrorMsg.TXN_ABORTED, 
expectedException.getCanonicalErrorMsg());
 Assert.assertEquals("Exception msg didn't match",
-  "Aborting [txnid:"+txnId2+","+txnId2+"] due to a write conflict on 
default/TAB_PART/p=blah committed by [txnid:"+txnId+","+txnId2+"] u/u",
+  "Aborting [txnid:"+txnId2+","+txnId2+"] due to a write conflict on 
default/tab_part/p=blah committed by [txnid:"+txnId+","+txnId2+"] u/u",
   expectedException.getCause().getMessage());
   }
   /**

hive git commit: HIVE-19260 - Streaming Ingest API doesn't normalize db.table names (Eugene Koifman, reviewed by Prasanth Jayachandran)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/branch-3 656af1411 -> cfdb272c7


HIVE-19260 - Streaming Ingest API doesn't normalize db.table names (Eugene 
Koifman, reviewed by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cfdb272c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cfdb272c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cfdb272c

Branch: refs/heads/branch-3
Commit: cfdb272c79bbdfa8b3c0c6481ec057cd91d53ad7
Parents: 656af14
Author: Eugene Koifman 
Authored: Wed Apr 25 12:02:19 2018 -0700
Committer: Eugene Koifman 
Committed: Wed Apr 25 12:02:19 2018 -0700

--
 .../hive/hcatalog/streaming/HiveEndPoint.java   |   4 +-
 .../hive/hcatalog/streaming/TestStreaming.java  |  11 +-
 .../hadoop/hive/ql/txn/compactor/Cleaner.java   | 119 +--
 .../hadoop/hive/ql/txn/compactor/Initiator.java |   2 +-
 .../hadoop/hive/metastore/txn/TxnHandler.java   |  22 ++--
 .../apache/hive/streaming/TestStreaming.java|   9 +-
 6 files changed, 139 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/cfdb272c/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
--
diff --git 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
index 6d248ea..8582e9a 100644
--- 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
+++ 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
@@ -88,13 +88,13 @@ public class HiveEndPoint {
 if (database==null) {
   throw new IllegalArgumentException("Database cannot be null for 
HiveEndPoint");
 }
-this.database = database;
-this.table = table;
+this.database = database.toLowerCase();
 if (table==null) {
   throw new IllegalArgumentException("Table cannot be null for 
HiveEndPoint");
 }
 this.partitionVals = partitionVals==null ? new ArrayList()
  : new ArrayList( 
partitionVals );
+this.table = table.toLowerCase();
   }
 
 

http://git-wip-us.apache.org/repos/asf/hive/blob/cfdb272c/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
--
diff --git 
a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
 
b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
index 3733e3d..fe2b1c1 100644
--- 
a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
+++ 
b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
@@ -67,6 +67,8 @@ import org.apache.hadoop.hive.metastore.api.TxnState;
 import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 import org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService;
 import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
+import org.apache.hadoop.hive.metastore.txn.TxnStore;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
 import org.apache.hadoop.hive.ql.DriverFactory;
 import org.apache.hadoop.hive.ql.IDriver;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
@@ -353,10 +355,10 @@ public class TestStreaming {
 //todo: why does it need transactional_properties?
 queryTable(driver, "create table default.streamingnobuckets (a string, b 
string) stored as orc TBLPROPERTIES('transactional'='true', 
'transactional_properties'='default')");
 queryTable(driver, "insert into default.streamingnobuckets 
values('foo','bar')");
-List rs = queryTable(driver, "select * from 
default.streamingnobuckets");
+List rs = queryTable(driver, "select * from 
default.streamingNoBuckets");
 Assert.assertEquals(1, rs.size());
 Assert.assertEquals("foo\tbar", rs.get(0));
-HiveEndPoint endPt = new HiveEndPoint(metaStoreURI, "default", 
"streamingnobuckets", null);
+HiveEndPoint endPt = new HiveEndPoint(metaStoreURI, "Default", 
"StreamingNoBuckets", null);
 String[] colNames1 = new String[] { "a", "b" };
 StreamingConnection connection = endPt.newConnection(false, "UT_" + 
Thread.currentThread().getName());
 DelimitedInputWriter wr = new DelimitedInputWriter(colNames1,",",  endPt, 
connection);
@@ -365,6 +367,11 @@ public class TestStreaming {
 txnBatch.beginNextTransaction();
 txnBatch.write("a1,b2".getBytes());
 txnBatch.write("a3,b4".getBytes());
+TxnStore txnHandler = TxnUtils.getTxnStore(conf);
+ShowLocksResponse resp = txnHandler.showLocks(new ShowLocksRequest());
+

[2/2] hive git commit: HIVE-19260 - Streaming Ingest API doesn't normalize db.table names (Eugene Koifman, reviewed by Prasanth Jayachandran)

[ekoifman]

HIVE-19260 - Streaming Ingest API doesn't normalize db.table names (Eugene 
Koifman, reviewed by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/36ef2741
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/36ef2741
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/36ef2741

Branch: refs/heads/master
Commit: 36ef27418f68d5e1f6c2f718a953cc605a5e836f
Parents: 6934bb9
Author: Eugene Koifman 
Authored: Tue Apr 24 11:31:28 2018 -0700
Committer: Eugene Koifman 
Committed: Tue Apr 24 11:31:28 2018 -0700

--
 .../hive/hcatalog/streaming/HiveEndPoint.java   |   4 +-
 .../hive/hcatalog/streaming/TestStreaming.java  |  11 +-
 .../hadoop/hive/ql/txn/compactor/Cleaner.java   | 119 +--
 .../hadoop/hive/ql/txn/compactor/Initiator.java |   2 +-
 .../hadoop/hive/metastore/txn/TxnHandler.java   |  22 ++--
 .../apache/hive/streaming/TestStreaming.java|   9 +-
 6 files changed, 139 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/36ef2741/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
--
diff --git 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
index 6d248ea..8582e9a 100644
--- 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
+++ 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
@@ -88,13 +88,13 @@ public class HiveEndPoint {
 if (database==null) {
   throw new IllegalArgumentException("Database cannot be null for 
HiveEndPoint");
 }
-this.database = database;
-this.table = table;
+this.database = database.toLowerCase();
 if (table==null) {
   throw new IllegalArgumentException("Table cannot be null for 
HiveEndPoint");
 }
 this.partitionVals = partitionVals==null ? new ArrayList()
  : new ArrayList( 
partitionVals );
+this.table = table.toLowerCase();
   }
 
 

http://git-wip-us.apache.org/repos/asf/hive/blob/36ef2741/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
--
diff --git 
a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
 
b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
index 3733e3d..fe2b1c1 100644
--- 
a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
+++ 
b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
@@ -67,6 +67,8 @@ import org.apache.hadoop.hive.metastore.api.TxnState;
 import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 import org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService;
 import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
+import org.apache.hadoop.hive.metastore.txn.TxnStore;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
 import org.apache.hadoop.hive.ql.DriverFactory;
 import org.apache.hadoop.hive.ql.IDriver;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
@@ -353,10 +355,10 @@ public class TestStreaming {
 //todo: why does it need transactional_properties?
 queryTable(driver, "create table default.streamingnobuckets (a string, b 
string) stored as orc TBLPROPERTIES('transactional'='true', 
'transactional_properties'='default')");
 queryTable(driver, "insert into default.streamingnobuckets 
values('foo','bar')");
-List rs = queryTable(driver, "select * from 
default.streamingnobuckets");
+List rs = queryTable(driver, "select * from 
default.streamingNoBuckets");
 Assert.assertEquals(1, rs.size());
 Assert.assertEquals("foo\tbar", rs.get(0));
-HiveEndPoint endPt = new HiveEndPoint(metaStoreURI, "default", 
"streamingnobuckets", null);
+HiveEndPoint endPt = new HiveEndPoint(metaStoreURI, "Default", 
"StreamingNoBuckets", null);
 String[] colNames1 = new String[] { "a", "b" };
 StreamingConnection connection = endPt.newConnection(false, "UT_" + 
Thread.currentThread().getName());
 DelimitedInputWriter wr = new DelimitedInputWriter(colNames1,",",  endPt, 
connection);
@@ -365,6 +367,11 @@ public class TestStreaming {
 txnBatch.beginNextTransaction();
 txnBatch.write("a1,b2".getBytes());
 txnBatch.write("a3,b4".getBytes());
+TxnStore txnHandler = TxnUtils.getTxnStore(conf);
+ShowLocksResponse resp = txnHandler.showLocks(new ShowLocksRequest());
+Assert.assertEquals(resp.getLocksSize(), 1);
+Assert.assertEquals("streamingnobuckets", 

[1/2] hive git commit: HIVE-19137 - orcfiledump doesn't print hive.acid.version value (Igor Kryvenko via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 56c3a9570 -> 36ef27418


HIVE-19137 - orcfiledump doesn't print hive.acid.version value (Igor Kryvenko 
via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6934bb96
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6934bb96
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6934bb96

Branch: refs/heads/master
Commit: 6934bb96704045d1f35213420cc1cfc984080797
Parents: 56c3a95
Author: Igor Kryvenko 
Authored: Tue Apr 24 11:22:46 2018 -0700
Committer: Eugene Koifman 
Committed: Tue Apr 24 11:22:46 2018 -0700

--
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java | 28 +++-
 .../hive/ql/io/orc/TestInputOutputFormat.java   |  4 +--
 2 files changed, 17 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/6934bb96/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index 212e0a6..fd84978 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -27,8 +27,6 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.hive.common.HiveStatsUtils;
-import org.apache.hadoop.hive.common.JavaUtils;
-import org.apache.hadoop.hive.common.ValidTxnList;
 import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
 import org.apache.hadoop.hive.common.ValidWriteIdList;
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -43,15 +41,12 @@ import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
 import org.apache.hadoop.hive.ql.io.orc.OrcRecordUpdater;
 import org.apache.hadoop.hive.ql.io.orc.Reader;
 import org.apache.hadoop.hive.ql.io.orc.Writer;
-import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
-import org.apache.hadoop.hive.ql.lockmgr.LockException;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
 import org.apache.hadoop.hive.ql.plan.TableScanDesc;
 import org.apache.hadoop.hive.shims.HadoopShims;
 import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId;
 import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.mapred.JobConf;
 import org.apache.hive.common.util.Ref;
 import org.apache.orc.FileFormatException;
 import org.apache.orc.impl.OrcAcidUtils;
@@ -61,7 +56,7 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.io.Serializable;
-import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -1705,6 +1700,7 @@ public class AcidUtils {
   public static final class OrcAcidVersion {
 private static final String ACID_VERSION_KEY = "hive.acid.version";
 private static final String ACID_FORMAT = "_orc_acid_version";
+private static final Charset UTF8 = Charset.forName("UTF-8");
 public static final int ORC_ACID_VERSION_DEFAULT = 0;
 /**
  * 2 is the version of Acid released in Hive 3.0.
@@ -1716,9 +1712,7 @@ public class AcidUtils {
  */
 public static void setAcidVersionInDataFile(Writer writer) {
   //so that we know which version wrote the file
-  ByteBuffer bf = ByteBuffer.allocate(4).putInt(ORC_ACID_VERSION);
-  bf.rewind(); //don't ask - some ByteBuffer weridness. w/o this, empty 
buffer is written
-  writer.addUserMetadata(ACID_VERSION_KEY, bf);
+  writer.addUserMetadata(ACID_VERSION_KEY, 
UTF8.encode(String.valueOf(ORC_ACID_VERSION)));
 }
 /**
  * This is smart enough to handle streaming ingest where there could be a
@@ -1735,8 +1729,10 @@ public class AcidUtils {
   .filesystem(fs)
   //make sure to check for side file in case streaming ingest died
   .maxLength(getLogicalLength(fs, fileStatus)));
-  if(orcReader.hasMetadataValue(ACID_VERSION_KEY)) {
-return orcReader.getMetadataValue(ACID_VERSION_KEY).getInt();
+  if (orcReader.hasMetadataValue(ACID_VERSION_KEY)) {
+char[] versionChar = 
UTF8.decode(orcReader.getMetadataValue(ACID_VERSION_KEY)).array();
+String version = new String(versionChar);
+return Integer.valueOf(version);
   }
   return ORC_ACID_VERSION_DEFAULT;
 }
@@ -1748,7 +1744,7 @@ public class AcidUtils {
   Path formatFile = getVersionFilePath(deltaOrBaseDir);
   if(!fs.exists(formatFile)) {
 try (FSDataOutputStream strm = fs.create(formatFile, false)) {
-  strm.writeInt(ORC_ACID_VERSION);
+   

hive git commit: HIVE-19137 - orcfiledump doesn't print hive.acid.version value (Igor Kryvenko via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/branch-3 e8568f282 -> e7552df73


HIVE-19137 - orcfiledump doesn't print hive.acid.version value (Igor Kryvenko 
via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e7552df7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e7552df7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e7552df7

Branch: refs/heads/branch-3
Commit: e7552df7356106c157de7767e12ceb3dccb7edc2
Parents: e8568f2
Author: Igor Kryvenko 
Authored: Tue Apr 24 11:20:25 2018 -0700
Committer: Eugene Koifman 
Committed: Tue Apr 24 11:20:25 2018 -0700

--
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java | 28 +++-
 .../hive/ql/io/orc/TestInputOutputFormat.java   |  4 +--
 2 files changed, 17 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/e7552df7/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index 212e0a6..fd84978 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -27,8 +27,6 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.hive.common.HiveStatsUtils;
-import org.apache.hadoop.hive.common.JavaUtils;
-import org.apache.hadoop.hive.common.ValidTxnList;
 import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
 import org.apache.hadoop.hive.common.ValidWriteIdList;
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -43,15 +41,12 @@ import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
 import org.apache.hadoop.hive.ql.io.orc.OrcRecordUpdater;
 import org.apache.hadoop.hive.ql.io.orc.Reader;
 import org.apache.hadoop.hive.ql.io.orc.Writer;
-import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
-import org.apache.hadoop.hive.ql.lockmgr.LockException;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
 import org.apache.hadoop.hive.ql.plan.TableScanDesc;
 import org.apache.hadoop.hive.shims.HadoopShims;
 import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId;
 import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.mapred.JobConf;
 import org.apache.hive.common.util.Ref;
 import org.apache.orc.FileFormatException;
 import org.apache.orc.impl.OrcAcidUtils;
@@ -61,7 +56,7 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.io.Serializable;
-import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -1705,6 +1700,7 @@ public class AcidUtils {
   public static final class OrcAcidVersion {
 private static final String ACID_VERSION_KEY = "hive.acid.version";
 private static final String ACID_FORMAT = "_orc_acid_version";
+private static final Charset UTF8 = Charset.forName("UTF-8");
 public static final int ORC_ACID_VERSION_DEFAULT = 0;
 /**
  * 2 is the version of Acid released in Hive 3.0.
@@ -1716,9 +1712,7 @@ public class AcidUtils {
  */
 public static void setAcidVersionInDataFile(Writer writer) {
   //so that we know which version wrote the file
-  ByteBuffer bf = ByteBuffer.allocate(4).putInt(ORC_ACID_VERSION);
-  bf.rewind(); //don't ask - some ByteBuffer weridness. w/o this, empty 
buffer is written
-  writer.addUserMetadata(ACID_VERSION_KEY, bf);
+  writer.addUserMetadata(ACID_VERSION_KEY, 
UTF8.encode(String.valueOf(ORC_ACID_VERSION)));
 }
 /**
  * This is smart enough to handle streaming ingest where there could be a
@@ -1735,8 +1729,10 @@ public class AcidUtils {
   .filesystem(fs)
   //make sure to check for side file in case streaming ingest died
   .maxLength(getLogicalLength(fs, fileStatus)));
-  if(orcReader.hasMetadataValue(ACID_VERSION_KEY)) {
-return orcReader.getMetadataValue(ACID_VERSION_KEY).getInt();
+  if (orcReader.hasMetadataValue(ACID_VERSION_KEY)) {
+char[] versionChar = 
UTF8.decode(orcReader.getMetadataValue(ACID_VERSION_KEY)).array();
+String version = new String(versionChar);
+return Integer.valueOf(version);
   }
   return ORC_ACID_VERSION_DEFAULT;
 }
@@ -1748,7 +1744,7 @@ public class AcidUtils {
   Path formatFile = getVersionFilePath(deltaOrBaseDir);
   if(!fs.exists(formatFile)) {
 try (FSDataOutputStream strm = fs.create(formatFile, false)) {
-  strm.writeInt(ORC_ACID_VERSION);
+   

[2/2] hive git commit: HIVE-18739 - Add support for Import/Export from Acid table (Eugene Koifman, reviewed by Sergey Shelukhin)

[ekoifman]

HIVE-18739 - Add support for Import/Export from Acid table (Eugene Koifman, 
reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a3e535f9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a3e535f9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a3e535f9

Branch: refs/heads/branch-3
Commit: a3e535f944d852209ca299e703860780fbd53955
Parents: 8584947
Author: Eugene Koifman 
Authored: Thu Apr 19 09:21:41 2018 -0700
Committer: Eugene Koifman 
Committed: Thu Apr 19 09:21:41 2018 -0700

--
 .../apache/hadoop/hive/common/JavaUtils.java|   3 +-
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |   2 +-
 .../apache/hadoop/hive/ql/exec/ExportTask.java  |   1 +
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java |   4 +
 .../apache/hadoop/hive/ql/metadata/Hive.java|   9 +-
 .../ql/metadata/SessionHiveMetaStoreClient.java | 217 
 .../hive/ql/parse/BaseSemanticAnalyzer.java |   7 +
 .../hive/ql/parse/ExportSemanticAnalyzer.java   |  19 +-
 .../hive/ql/parse/ImportSemanticAnalyzer.java   | 116 ++--
 .../hive/ql/parse/SemanticAnalyzerFactory.java  |   3 +
 .../ql/parse/UpdateDeleteSemanticAnalyzer.java  | 258 -
 .../apache/hadoop/hive/ql/plan/CopyWork.java|   6 +-
 .../apache/hadoop/hive/ql/plan/ExportWork.java  |  28 +-
 .../hadoop/hive/ql/plan/ImportTableDesc.java|   2 +-
 .../hadoop/hive/ql/session/SessionState.java|  10 +-
 .../hadoop/hive/ql/TestTxnAddPartition.java |   2 +-
 .../org/apache/hadoop/hive/ql/TestTxnExIm.java  | 538 +++
 .../apache/hadoop/hive/ql/TestTxnLoadData.java  |   2 +-
 .../apache/hadoop/hive/ql/TestTxnNoBuckets.java |  24 +-
 .../hive/metastore/HiveMetaStoreClient.java |   2 +-
 .../hadoop/hive/metastore/ObjectStore.java  |  18 +-
 .../hive/metastore/utils/MetaStoreUtils.java|  20 +
 22 files changed, 1201 insertions(+), 90 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/a3e535f9/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java 
b/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
index 75c07b4..7894ec1 100644
--- a/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
@@ -228,10 +228,11 @@ public final class JavaUtils {
 @Override
 public boolean accept(Path path) {
   String name = path.getName();
+  //todo: what if this is a base?
   if (!name.startsWith(DELTA_PREFIX + "_")) return false;
   String idStr = name.substring(DELTA_PREFIX.length() + 1, 
DELTA_PREFIX.length() + 1 + DELTA_DIGITS_LEN);
   try {
-Long.parseLong(idStr);
+Long.parseLong(idStr);//what for? sanity check?
   } catch (NumberFormatException ex) {
 return false;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/a3e535f9/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 5b26b84..9a487cd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -3965,7 +3965,7 @@ public class DDLTask extends Task implements 
Serializable {
* how this is desirable.
*
* As of HIVE-14993, WriteEntity with different WriteType must be considered 
different.
-   * So WriteEntity create in DDLTask cause extra output in golden files, but 
only because
+   * So WriteEntity created in DDLTask cause extra output in golden files, but 
only because
* DDLTask sets a different WriteType for the same Entity.
*
* In the spirit of bug-for-bug compatibility, this method ensures we only 
add new

http://git-wip-us.apache.org/repos/asf/hive/blob/a3e535f9/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
index 91af814..aba6591 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
@@ -52,6 +52,7 @@ public class ExportTask extends Task implements 
Serializable {
   conf, false);
   Hive db = getHive();
   LOG.debug("Exporting data to: {}", exportPaths.getExportRootDir());
+  work.acidPostProcess(db);
   TableExport tableExport = new TableExport(
   exportPaths, 

[1/2] hive git commit: HIVE-18739 - Add support for Import/Export from Acid table (Eugene Koifman, reviewed by Sergey Shelukhin)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/branch-3 8584947ef -> a3e535f94


http://git-wip-us.apache.org/repos/asf/hive/blob/a3e535f9/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
index a4df509..4b2f961 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
@@ -32,8 +32,6 @@ import org.apache.hadoop.hive.ql.io.BucketCodec;
 import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
 import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TestName;
@@ -191,8 +189,9 @@ public class TestTxnNoBuckets extends 
TxnCommandsBaseForTests {
 checkExpected(rs, expected, "Unexpected row count after ctas from non acid 
table");
 
 runStatementOnDriver("insert into " + Table.ACIDTBL + 
makeValuesClause(values));
+//todo: try this with acid default - it seem making table acid in listener 
is too late
 runStatementOnDriver("create table myctas2 stored as ORC TBLPROPERTIES 
('transactional" +
-  "'='true', 'transactional_properties'='default') as select a, b from " + 
Table.ACIDTBL);//todo: try this with acid default - it seem makeing table acid 
in listener is too late
+  "'='true', 'transactional_properties'='default') as select a, b from " + 
Table.ACIDTBL);
 rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from 
myctas2 order by ROW__ID");
 String expected2[][] = {
 {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t3\t4", 
"warehouse/myctas2/delta_001_001_/bucket_0"},
@@ -234,7 +233,7 @@ public class TestTxnNoBuckets extends 
TxnCommandsBaseForTests {
 
   /**
* Insert into unbucketed acid table from union all query
-   * Union All is flattend so nested subdirs are created and acid move drops 
them since
+   * Union All is flattened so nested subdirs are created and acid move drops 
them since
* delta dirs have unique names
*/
   @Test
@@ -529,11 +528,26 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree 
/Users/ekoifman/dev/hiver
 CommandProcessorResponse cpr = runStatementOnDriverNegative("create table 
myctas " +
   "clustered by (a) into 2 buckets stored as ORC TBLPROPERTIES 
('transactional'='true') as " +
   "select a, b from " + Table.NONACIDORCTBL);
-int j = ErrorMsg.CTAS_PARCOL_COEXISTENCE.getErrorCode();//this code 
doesn't propagate
+int j = ErrorMsg.CTAS_PARCOL_COEXISTENCE.getErrorCode(); //this code 
doesn't propagate
 //Assert.assertEquals("Wrong msg", 
ErrorMsg.CTAS_PARCOL_COEXISTENCE.getErrorCode(), cpr.getErrorCode());
 Assert.assertTrue(cpr.getErrorMessage().contains("CREATE-TABLE-AS-SELECT 
does not support"));
   }
   /**
+   * Currently CTAS doesn't support partitioned tables.  Correspondingly Acid 
only supports CTAS for
+   * un-partitioned tables.  This test is here to make sure that if CTAS is 
made to support
+   * un-partitioned tables, that it raises a red flag for Acid.
+   */
+  @Test
+  public void testCtasPartitioned() throws Exception {
+runStatementOnDriver("insert into " + Table.NONACIDNONBUCKET + "(a,b) 
values(1,2),(1,3)");
+CommandProcessorResponse cpr = runStatementOnDriverNegative("create table 
myctas partitioned " +
+"by (b int) stored as " +
+"ORC TBLPROPERTIES ('transactional'='true') as select a, b from " + 
Table.NONACIDORCTBL);
+int j = ErrorMsg.CTAS_PARCOL_COEXISTENCE.getErrorCode();//this code 
doesn't propagate
+Assert.assertTrue(cpr.getErrorMessage().contains("CREATE-TABLE-AS-SELECT 
does not support " +
+"partitioning in the target table"));
+  }
+  /**
* Tests to check that we are able to use vectorized acid reader,
* VectorizedOrcAcidRowBatchReader, when reading "original" files,
* i.e. those that were written before the table was converted to acid.

http://git-wip-us.apache.org/repos/asf/hive/blob/a3e535f9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
--
diff --git 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index b0f64db..fcf34f0 100644
--- 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient

[1/2] hive git commit: HIVE-18739 - Add support for Import/Export from Acid table (Eugene Koifman, reviewed by Sergey Shelukhin)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master bd6b58258 -> 699c5768c


http://git-wip-us.apache.org/repos/asf/hive/blob/699c5768/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
index a4df509..4b2f961 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
@@ -32,8 +32,6 @@ import org.apache.hadoop.hive.ql.io.BucketCodec;
 import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
 import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TestName;
@@ -191,8 +189,9 @@ public class TestTxnNoBuckets extends 
TxnCommandsBaseForTests {
 checkExpected(rs, expected, "Unexpected row count after ctas from non acid 
table");
 
 runStatementOnDriver("insert into " + Table.ACIDTBL + 
makeValuesClause(values));
+//todo: try this with acid default - it seem making table acid in listener 
is too late
 runStatementOnDriver("create table myctas2 stored as ORC TBLPROPERTIES 
('transactional" +
-  "'='true', 'transactional_properties'='default') as select a, b from " + 
Table.ACIDTBL);//todo: try this with acid default - it seem makeing table acid 
in listener is too late
+  "'='true', 'transactional_properties'='default') as select a, b from " + 
Table.ACIDTBL);
 rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from 
myctas2 order by ROW__ID");
 String expected2[][] = {
 {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t3\t4", 
"warehouse/myctas2/delta_001_001_/bucket_0"},
@@ -234,7 +233,7 @@ public class TestTxnNoBuckets extends 
TxnCommandsBaseForTests {
 
   /**
* Insert into unbucketed acid table from union all query
-   * Union All is flattend so nested subdirs are created and acid move drops 
them since
+   * Union All is flattened so nested subdirs are created and acid move drops 
them since
* delta dirs have unique names
*/
   @Test
@@ -529,11 +528,26 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree 
/Users/ekoifman/dev/hiver
 CommandProcessorResponse cpr = runStatementOnDriverNegative("create table 
myctas " +
   "clustered by (a) into 2 buckets stored as ORC TBLPROPERTIES 
('transactional'='true') as " +
   "select a, b from " + Table.NONACIDORCTBL);
-int j = ErrorMsg.CTAS_PARCOL_COEXISTENCE.getErrorCode();//this code 
doesn't propagate
+int j = ErrorMsg.CTAS_PARCOL_COEXISTENCE.getErrorCode(); //this code 
doesn't propagate
 //Assert.assertEquals("Wrong msg", 
ErrorMsg.CTAS_PARCOL_COEXISTENCE.getErrorCode(), cpr.getErrorCode());
 Assert.assertTrue(cpr.getErrorMessage().contains("CREATE-TABLE-AS-SELECT 
does not support"));
   }
   /**
+   * Currently CTAS doesn't support partitioned tables.  Correspondingly Acid 
only supports CTAS for
+   * un-partitioned tables.  This test is here to make sure that if CTAS is 
made to support
+   * un-partitioned tables, that it raises a red flag for Acid.
+   */
+  @Test
+  public void testCtasPartitioned() throws Exception {
+runStatementOnDriver("insert into " + Table.NONACIDNONBUCKET + "(a,b) 
values(1,2),(1,3)");
+CommandProcessorResponse cpr = runStatementOnDriverNegative("create table 
myctas partitioned " +
+"by (b int) stored as " +
+"ORC TBLPROPERTIES ('transactional'='true') as select a, b from " + 
Table.NONACIDORCTBL);
+int j = ErrorMsg.CTAS_PARCOL_COEXISTENCE.getErrorCode();//this code 
doesn't propagate
+Assert.assertTrue(cpr.getErrorMessage().contains("CREATE-TABLE-AS-SELECT 
does not support " +
+"partitioning in the target table"));
+  }
+  /**
* Tests to check that we are able to use vectorized acid reader,
* VectorizedOrcAcidRowBatchReader, when reading "original" files,
* i.e. those that were written before the table was converted to acid.

http://git-wip-us.apache.org/repos/asf/hive/blob/699c5768/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
--
diff --git 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 108add0..7ba053d 100644
--- 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient

[2/2] hive git commit: HIVE-18739 - Add support for Import/Export from Acid table (Eugene Koifman, reviewed by Sergey Shelukhin)

[ekoifman]

HIVE-18739 - Add support for Import/Export from Acid table (Eugene Koifman, 
reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/699c5768
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/699c5768
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/699c5768

Branch: refs/heads/master
Commit: 699c5768c88967abd507122d775bd5955ca45218
Parents: bd6b582
Author: Eugene Koifman 
Authored: Tue Apr 17 18:23:13 2018 -0700
Committer: Eugene Koifman 
Committed: Tue Apr 17 18:23:48 2018 -0700

--
 .../apache/hadoop/hive/common/JavaUtils.java|   3 +-
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |   2 +-
 .../apache/hadoop/hive/ql/exec/ExportTask.java  |   1 +
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java |   4 +
 .../apache/hadoop/hive/ql/metadata/Hive.java|   9 +-
 .../ql/metadata/SessionHiveMetaStoreClient.java | 217 
 .../hive/ql/parse/BaseSemanticAnalyzer.java |   7 +
 .../hive/ql/parse/ExportSemanticAnalyzer.java   |  19 +-
 .../hive/ql/parse/ImportSemanticAnalyzer.java   | 111 ++--
 .../hive/ql/parse/SemanticAnalyzerFactory.java  |   3 +
 .../ql/parse/UpdateDeleteSemanticAnalyzer.java  | 258 -
 .../apache/hadoop/hive/ql/plan/CopyWork.java|   6 +-
 .../apache/hadoop/hive/ql/plan/ExportWork.java  |  28 +-
 .../hadoop/hive/ql/plan/ImportTableDesc.java|   2 +-
 .../hadoop/hive/ql/session/SessionState.java|  10 +-
 .../hadoop/hive/ql/TestTxnAddPartition.java |   2 +-
 .../org/apache/hadoop/hive/ql/TestTxnExIm.java  | 538 +++
 .../apache/hadoop/hive/ql/TestTxnLoadData.java  |   2 +-
 .../apache/hadoop/hive/ql/TestTxnNoBuckets.java |  24 +-
 .../hive/metastore/HiveMetaStoreClient.java |   2 +-
 .../hadoop/hive/metastore/ObjectStore.java  |  18 +-
 .../hive/metastore/utils/MetaStoreUtils.java|  20 +
 22 files changed, 1197 insertions(+), 89 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/699c5768/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java 
b/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
index 75c07b4..7894ec1 100644
--- a/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
@@ -228,10 +228,11 @@ public final class JavaUtils {
 @Override
 public boolean accept(Path path) {
   String name = path.getName();
+  //todo: what if this is a base?
   if (!name.startsWith(DELTA_PREFIX + "_")) return false;
   String idStr = name.substring(DELTA_PREFIX.length() + 1, 
DELTA_PREFIX.length() + 1 + DELTA_DIGITS_LEN);
   try {
-Long.parseLong(idStr);
+Long.parseLong(idStr);//what for? sanity check?
   } catch (NumberFormatException ex) {
 return false;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/699c5768/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 2466da9..c8cb8a4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -3969,7 +3969,7 @@ public class DDLTask extends Task implements 
Serializable {
* how this is desirable.
*
* As of HIVE-14993, WriteEntity with different WriteType must be considered 
different.
-   * So WriteEntity create in DDLTask cause extra output in golden files, but 
only because
+   * So WriteEntity created in DDLTask cause extra output in golden files, but 
only because
* DDLTask sets a different WriteType for the same Entity.
*
* In the spirit of bug-for-bug compatibility, this method ensures we only 
add new

http://git-wip-us.apache.org/repos/asf/hive/blob/699c5768/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
index 91af814..aba6591 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
@@ -52,6 +52,7 @@ public class ExportTask extends Task implements 
Serializable {
   conf, false);
   Hive db = getHive();
   LOG.debug("Exporting data to: {}", exportPaths.getExportRootDir());
+  work.acidPostProcess(db);
   TableExport tableExport = new TableExport(
   exportPaths, 

hive git commit: HIVE-18845 - SHOW COMAPCTIONS should show host name (Igor Kryvenko via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 54046353e -> fedb03aba


HIVE-18845 - SHOW COMAPCTIONS should show host name (Igor Kryvenko via Eugene 
Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fedb03ab
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fedb03ab
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fedb03ab

Branch: refs/heads/master
Commit: fedb03abad6b46124af6b5187213ed03e051e4d1
Parents: 5404635
Author: Igor Kryvenko 
Authored: Tue Apr 17 09:48:15 2018 -0700
Committer: Eugene Koifman 
Committed: Tue Apr 17 09:48:15 2018 -0700

--
 ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java| 6 +-
 .../org/apache/hadoop/hive/ql/plan/ShowCompactionsDesc.java| 4 ++--
 ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out| 4 ++--
 3 files changed, 9 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/fedb03ab/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index afcfe09..2466da9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -3181,6 +3181,8 @@ public class DDLTask extends Task implements 
Serializable {
   os.write(separator);
   os.writeBytes("State");
   os.write(separator);
+  os.writeBytes("Hostname");
+  os.write(separator);
   os.writeBytes("Worker");
   os.write(separator);
   os.writeBytes("Start Time");
@@ -3206,7 +3208,9 @@ public class DDLTask extends Task implements 
Serializable {
   os.writeBytes(e.getState());
   os.write(separator);
   String wid = e.getWorkerid();
-  os.writeBytes(wid == null ? noVal : wid);
+  os.writeBytes(wid == null ? noVal : wid.split("-")[0]);
+  os.write(separator);
+  os.writeBytes(wid == null ? noVal : wid.split("-")[1]);
   os.write(separator);
   os.writeBytes(e.isSetStart() ? Long.toString(e.getStart()) : noVal);
   os.write(separator);

http://git-wip-us.apache.org/repos/asf/hive/blob/fedb03ab/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowCompactionsDesc.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowCompactionsDesc.java 
b/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowCompactionsDesc.java
index 1691665..093800f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowCompactionsDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowCompactionsDesc.java
@@ -28,8 +28,8 @@ public class ShowCompactionsDesc extends DDLDesc implements 
Serializable {
 
   private static final long serialVersionUID = 1L;
   private static final String schema =
-
"compactionid,dbname,tabname,partname,type,state,workerid,starttime,duration,hadoopjobid#"
 +
-  "string:string:string:string:string:string:string:string:string:string";
+  
"compactionid,dbname,tabname,partname,type,state,hostname,workerid,starttime,duration,hadoopjobid#"
 +
+  
"string:string:string:string:string:string:string:string:string:string:string";
 
   private String resFile;
 

http://git-wip-us.apache.org/repos/asf/hive/blob/fedb03ab/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out
--
diff --git a/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out 
b/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out
index b86d84c..aaf1c78 100644
--- a/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out
+++ b/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out
@@ -69,8 +69,8 @@ PREHOOK: query: show compactions
 PREHOOK: type: SHOW COMPACTIONS
 POSTHOOK: query: show compactions
 POSTHOOK: type: SHOW COMPACTIONS
-CompactionId   DatabaseTable   Partition   TypeState   Worker  
Start Time  Duration(ms)HadoopJobId
-1  default partitioned_acid_table  p=abc   MINOR   initiated---
 --- --- --- 
+CompactionId   DatabaseTable   Partition   TypeState   
HostnameWorker  Start Time  Duration(ms)HadoopJobId
+1  default partitioned_acid_table  p=abc   MINOR   initiated---
 --- --- --- --- 
 PREHOOK: query: drop table partitioned_acid_table
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@partitioned_acid_table

hive git commit: HIVE-16144 - CompactionInfo doesn't have equals/hashCode but used in Set (Igor Kryvenko via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master c46c4c90f -> 996db32ba


HIVE-16144 - CompactionInfo doesn't have equals/hashCode but used in Set (Igor 
Kryvenko via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/996db32b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/996db32b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/996db32b

Branch: refs/heads/master
Commit: 996db32ba31c030b26c059ef575cb3d5b68ab1b3
Parents: c46c4c9
Author: Eugene Koifman 
Authored: Mon Apr 16 09:54:41 2018 -0700
Committer: Eugene Koifman 
Committed: Mon Apr 16 09:55:11 2018 -0700

--
 .../hive/ql/txn/compactor/TestCompactor.java | 18 ++
 .../hive/metastore/txn/CompactionInfo.java   | 19 +++
 2 files changed, 37 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/996db32b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
index b19aa23..82ba775 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
@@ -1311,6 +1311,24 @@ public class TestCompactor {
 3141, orcReader.getCompressionSize());
   }
 
+  @Test
+  public void testCompactionInfoEquals() {
+CompactionInfo compactionInfo = new CompactionInfo("dbName", "tableName", 
"partName", CompactionType.MINOR);
+CompactionInfo compactionInfo1 = new CompactionInfo("dbName", "tableName", 
"partName", CompactionType.MINOR);
+Assert.assertTrue("The object must be equal", 
compactionInfo.equals(compactionInfo));
+
+Assert.assertFalse("The object must be not equal", 
compactionInfo.equals(new Object()));
+Assert.assertTrue("The object must be equal", 
compactionInfo.equals(compactionInfo1));
+  }
+
+  @Test
+  public void testCompactionInfoHashCode() {
+CompactionInfo compactionInfo = new CompactionInfo("dbName", "tableName", 
"partName", CompactionType.MINOR);
+CompactionInfo compactionInfo1 = new CompactionInfo("dbName", "tableName", 
"partName", CompactionType.MINOR);
+
+Assert.assertEquals("The hash codes must be equal", 
compactionInfo.hashCode(), compactionInfo1.hashCode());
+  }
+
   private void writeBatch(StreamingConnection connection, DelimitedInputWriter 
writer,
   boolean closeEarly)
   throws InterruptedException, StreamingException {

http://git-wip-us.apache.org/repos/asf/hive/blob/996db32b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
--
diff --git 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
index b74f69d..b34b7d7 100644
--- 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
+++ 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
@@ -110,6 +110,25 @@ public class CompactionInfo implements 
Comparable {
   "highestWriteId:" + highestWriteId;
   }
 
+  @Override
+  public int hashCode() {
+int result = 17;
+result = 31 * result + this.getFullPartitionName().hashCode();
+return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+if (obj == this) {
+  return true;
+}
+if (!(obj instanceof CompactionInfo)) {
+  return false;
+}
+CompactionInfo info = (CompactionInfo) obj;
+return this.compareTo(info) == 0;
+  }
+
   /**
* loads object from a row in Select * from COMPACTION_QUEUE
* @param rs ResultSet after call to rs.next()

hive git commit: HIVE-19100 : investigate TestStreaming failures(Eugene Koifman, reviewed by Alan Gates)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master e54acc611 -> 078b9c333


HIVE-19100 : investigate TestStreaming failures(Eugene Koifman, reviewed by 
Alan Gates)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/078b9c33
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/078b9c33
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/078b9c33

Branch: refs/heads/master
Commit: 078b9c333076b864115f36f3698ceff6af5b2344
Parents: e54acc6
Author: Eugene Koifman 
Authored: Wed Apr 4 12:50:13 2018 -0700
Committer: Eugene Koifman 
Committed: Wed Apr 4 12:50:37 2018 -0700

--
 .../hive/hcatalog/streaming/TestStreaming.java  | 10 +++
 .../hive/ql/parse/DDLSemanticAnalyzer.java  | 29 
 2 files changed, 22 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/078b9c33/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
--
diff --git 
a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
 
b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
index a9ab90b..3733e3d 100644
--- 
a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
+++ 
b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
@@ -2101,7 +2101,7 @@ public class TestStreaming {
 
   /  UTILS --- /
   // returns Path of the partition created (if any) else Path of table
-  public static Path createDbAndTable(IDriver driver, String databaseName,
+  private static Path createDbAndTable(IDriver driver, String databaseName,
   String tableName, List partVals,
   String[] colNames, String[] colTypes,
   String[] bucketCols,
@@ -2147,7 +2147,7 @@ public class TestStreaming {
   private static String getTableColumnsStr(String[] colNames, String[] 
colTypes) {
 StringBuilder sb = new StringBuilder();
 for (int i=0; i < colNames.length; ++i) {
-  sb.append(colNames[i] + " " + colTypes[i]);
+  sb.append(colNames[i]).append(" ").append(colTypes[i]);
   if (i

hive git commit: HIVE-18814 - Support Add Partition For Acid tables (Eugene Koifman, reviewed by Alan Gates)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 00d0e56e3 -> fdc1e1fbd


HIVE-18814 - Support Add Partition For Acid tables (Eugene Koifman, reviewed by 
Alan Gates)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fdc1e1fb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fdc1e1fb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fdc1e1fb

Branch: refs/heads/master
Commit: fdc1e1fbdafa668171cd3a6bf5fd2e33a10f8d64
Parents: 00d0e56
Author: Eugene Koifman 
Authored: Tue Apr 3 10:54:18 2018 -0700
Committer: Eugene Koifman 
Committed: Tue Apr 3 10:54:42 2018 -0700

--
 .../hive/ql/parse/DDLSemanticAnalyzer.java  |  64 -
 .../hadoop/hive/ql/TestTxnAddPartition.java | 260 +++
 .../apache/hadoop/hive/ql/TestTxnLoadData.java  |  10 +-
 .../hadoop/hive/ql/TxnCommandsBaseForTests.java |  22 +-
 .../hive/ql/lockmgr/TestDbTxnManager2.java  |  32 +++
 5 files changed, 374 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/fdc1e1fb/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index ed1c0ab..08d26dc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -73,6 +73,7 @@ import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.QueryState;
 import org.apache.hadoop.hive.ql.exec.ArchiveUtils;
 import org.apache.hadoop.hive.ql.exec.ColumnStatsUpdateTask;
+import org.apache.hadoop.hive.ql.exec.DDLTask;
 import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
 import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.exec.TaskFactory;
@@ -3389,7 +3390,13 @@ public class DDLSemanticAnalyzer extends 
BaseSemanticAnalyzer {
 Table tab = getTable(qualified);
 boolean isView = tab.isView();
 validateAlterTableType(tab, AlterTableTypes.ADDPARTITION, expectView);
-outputs.add(new WriteEntity(tab, WriteEntity.WriteType.DDL_SHARED));
+outputs.add(new WriteEntity(tab,
+/*use DDL_EXCLUSIVE to cause X lock to prevent races between 
concurrent add partition calls
+with IF NOT EXISTS.  w/o this 2 concurrent calls to add the same 
partition may both add
+data since for transactional tables creating partition metadata and 
moving data there are
+2 separate actions. */
+ifNotExists && AcidUtils.isTransactionalTable(tab) ? 
WriteType.DDL_EXCLUSIVE
+: WriteEntity.WriteType.DDL_SHARED));
 
 int numCh = ast.getChildCount();
 int start = ifNotExists ? 1 : 0;
@@ -3446,7 +3453,10 @@ public class DDLSemanticAnalyzer extends 
BaseSemanticAnalyzer {
   return;
 }
 
-rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), 
addPartitionDesc)));
+Task ddlTask =
+TaskFactory.get(new DDLWork(getInputs(), getOutputs(), 
addPartitionDesc));
+rootTasks.add(ddlTask);
+handleTransactionalTable(tab, addPartitionDesc, ddlTask);
 
 if (isView) {
   // Compile internal query to capture underlying table partition 
dependencies
@@ -3489,6 +3499,56 @@ public class DDLSemanticAnalyzer extends 
BaseSemanticAnalyzer {
   }
 
   /**
+   * Add partition for Transactional tables needs to add (copy/rename) the 
data so that it lands
+   * in a delta_x_x/ folder in the partition dir.
+   */
+  private void handleTransactionalTable(Table tab, AddPartitionDesc 
addPartitionDesc,
+  Task ddlTask) throws SemanticException {
+if(!AcidUtils.isTransactionalTable(tab)) {
+  return;
+}
+Long writeId;
+int stmtId;
+try {
+  writeId = SessionState.get().getTxnMgr().getTableWriteId(tab.getDbName(),
+  tab.getTableName());
+} catch (LockException ex) {
+  throw new SemanticException("Failed to allocate the write id", ex);
+}
+stmtId = SessionState.get().getTxnMgr().getStmtIdAndIncrement();
+
+for (int index = 0; index < addPartitionDesc.getPartitionCount(); index++) 
{
+  OnePartitionDesc desc = addPartitionDesc.getPartition(index);
+  if (desc.getLocation() != null) {
+if(addPartitionDesc.isIfNotExists()) {
+  //Don't add
+  Partition oldPart = getPartition(tab, desc.getPartSpec(), false);
+  if(oldPart != null) {
+continue;
+  }
+}
+LoadTableDesc loadTableWork = new LoadTableDesc(new 
Path(desc.getLocation()),
+Utilities.getTableDesc(tab), desc.getPartSpec(),
+

hive git commit: HIVE-19029 - Load Data should prevent loading acid files (Eugene Koifman, reviewed by Jason Dere)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master 6fd240f6e -> 3e3292b6c


HIVE-19029 - Load Data should prevent loading acid files (Eugene Koifman, 
reviewed by Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3e3292b6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3e3292b6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3e3292b6

Branch: refs/heads/master
Commit: 3e3292b6c6170671e9a2bf6501c12fff84af15a7
Parents: 6fd240f
Author: Eugene Koifman 
Authored: Thu Mar 29 15:21:26 2018 -0700
Committer: Eugene Koifman 
Committed: Thu Mar 29 15:21:26 2018 -0700

--
 .../org/apache/hadoop/hive/ql/ErrorMsg.java |  4 ++
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java |  3 ++
 .../hive/ql/parse/LoadSemanticAnalyzer.java | 40 +++-
 .../apache/hadoop/hive/ql/TestTxnLoadData.java  | 24 +++-
 4 files changed, 68 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/3e3292b6/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java 
b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index 1faa50a..f3e40eb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -457,6 +457,10 @@ public enum ErrorMsg {
   HIVE_GROUPING_SETS_SIZE_LIMIT(10411,
 "Grouping sets size cannot be greater than 64"),
   REBUILD_NO_MATERIALIZED_VIEW(10412, "Rebuild command only valid for 
materialized views"),
+  LOAD_DATA_ACID_FILE(10413,
+  "\"{0}\" was created created by Acid write - it cannot be loaded into 
anther Acid table",
+  true),
+
 
   //== 2 range starts here 
//
 

http://git-wip-us.apache.org/repos/asf/hive/blob/3e3292b6/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index 1828f0a..a9ebc90 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -1653,6 +1653,9 @@ public class AcidUtils {
 //directory is empty or doesn't have any that could have been produced 
by load data
 return false;
   }
+  return isRawFormatFile(dataFile, fs);
+}
+public static boolean isRawFormatFile(Path dataFile, FileSystem fs) throws 
IOException {
   try {
 Reader reader = OrcFile.createReader(dataFile, 
OrcFile.readerOptions(fs.getConf()));
 /*

http://git-wip-us.apache.org/repos/asf/hive/blob/3e3292b6/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
index d5aace0..e49089b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
@@ -150,7 +150,8 @@ public class LoadSemanticAnalyzer extends 
BaseSemanticAnalyzer {
 }
 
 try {
-  srcs = matchFilesOrDir(FileSystem.get(fromURI, conf), new Path(fromURI));
+  FileSystem fileSystem = FileSystem.get(fromURI, conf);
+  srcs = matchFilesOrDir(fileSystem, new Path(fromURI));
   if (srcs == null || srcs.length == 0) {
 throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(ast,
 "No files matching path " + fromURI));
@@ -162,6 +163,7 @@ public class LoadSemanticAnalyzer extends 
BaseSemanticAnalyzer {
   "source contains directory: " + oneSrc.getPath().toString()));
 }
   }
+  validateAcidFiles(table, srcs, fileSystem);
   // Do another loop if table is bucketed
   List bucketCols = table.getBucketCols();
   if (bucketCols != null && !bucketCols.isEmpty()) {
@@ -198,12 +200,24 @@ public class LoadSemanticAnalyzer extends 
BaseSemanticAnalyzer {
   if (bucketArray[bucketId]) {
 throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(
 "Multiple files for same bucket : " + bucketId
-+ ". Only 1 file per bucket allowed in single load 
command. To load multiple files for same bucket, use multiple statements for 
table "
++ ". Only 1 file per bucket allowed in single load 
command. To load " +
+"multiple files for same bucket, use multiple 

hive git commit: HIVE-18693 - Snapshot Isolation does not work for Micromanaged table when a insert transaction is aborted (Steve Yeom via Eugene Koifman)

[ekoifman]

Repository: hive
Updated Branches:
  refs/heads/master d5cb7f62e -> 52c6e8938


HIVE-18693 - Snapshot Isolation does not work for Micromanaged table when a 
insert transaction is aborted (Steve Yeom via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/52c6e893
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/52c6e893
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/52c6e893

Branch: refs/heads/master
Commit: 52c6e8938b7b0c6179b9c8208af2d77ca6ad0c1c
Parents: d5cb7f6
Author: Eugene Koifman 
Authored: Thu Mar 15 11:19:15 2018 -0700
Committer: Eugene Koifman 
Committed: Thu Mar 15 11:19:15 2018 -0700

--
 .../streaming/mutate/client/lock/Lock.java  |   4 +-
 .../streaming/mutate/client/lock/TestLock.java  |  10 +-
 .../hadoop/hive/ql/lockmgr/DbTxnManager.java|   4 +-
 .../hadoop/hive/ql/txn/compactor/Initiator.java |  13 +-
 .../hive/ql/TestTxnCommandsForMmTable.java  | 121 ++-
 .../gen/thrift/gen-cpp/hive_metastore_types.cpp |  24 ++--
 .../gen/thrift/gen-cpp/hive_metastore_types.h   |  14 +--
 .../hive/metastore/api/LockComponent.java   | 112 -
 .../src/gen/thrift/gen-php/metastore/Types.php  |  16 +--
 .../gen/thrift/gen-py/hive_metastore/ttypes.py  |  18 +--
 .../gen/thrift/gen-rb/hive_metastore_types.rb   |   4 +-
 .../hive/metastore/LockComponentBuilder.java|   4 +-
 .../hadoop/hive/metastore/txn/TxnHandler.java   |   5 +-
 .../src/main/thrift/hive_metastore.thrift   |   2 +-
 14 files changed, 236 insertions(+), 115 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/52c6e893/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/client/lock/Lock.java
--
diff --git 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/client/lock/Lock.java
 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/client/lock/Lock.java
index 429555f..abbb125 100644
--- 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/client/lock/Lock.java
+++ 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/client/lock/Lock.java
@@ -182,9 +182,9 @@ public class Lock {
   //todo: DataOperationType is set conservatively here, we'd really want 
to distinguish update/delete
   //and insert/select and if resource (that is written to) is ACID or not
   if (sinks.contains(table)) {
-
componentBuilder.setSemiShared().setOperationType(DataOperationType.UPDATE).setIsFullAcid(true);
+
componentBuilder.setSemiShared().setOperationType(DataOperationType.UPDATE).setIsTransactional(true);
   } else {
-
componentBuilder.setShared().setOperationType(DataOperationType.INSERT).setIsFullAcid(true);
+
componentBuilder.setShared().setOperationType(DataOperationType.INSERT).setIsTransactional(true);
   }
   LockComponent component = componentBuilder.build();
   requestBuilder.addLockComponent(component);

http://git-wip-us.apache.org/repos/asf/hive/blob/52c6e893/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/client/lock/TestLock.java
--
diff --git 
a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/client/lock/TestLock.java
 
b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/client/lock/TestLock.java
index e454942..0a46faf 100644
--- 
a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/client/lock/TestLock.java
+++ 
b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/client/lock/TestLock.java
@@ -176,13 +176,13 @@ public class TestLock {
 LockComponent expected1 = new LockComponent(LockType.SHARED_READ, 
LockLevel.TABLE, "DB");
 expected1.setTablename("SOURCE_1");
 expected1.setOperationType(DataOperationType.INSERT);
-expected1.setIsAcid(true);
+expected1.setIsTransactional(true);
 assertTrue(components.contains(expected1));
 
 LockComponent expected2 = new LockComponent(LockType.SHARED_READ, 
LockLevel.TABLE, "DB");
 expected2.setTablename("SOURCE_2");
 expected2.setOperationType(DataOperationType.INSERT);
-expected2.setIsAcid(true);
+expected2.setIsTransactional(true);
 assertTrue(components.contains(expected2));
   }
 
@@ -203,19 +203,19 @@ public class TestLock {
 LockComponent expected1 = new LockComponent(LockType.SHARED_READ, 
LockLevel.TABLE, "DB");
 expected1.setTablename("SOURCE_1");
 expected1.setOperationType(DataOperationType.INSERT);
-expected1.setIsAcid(true);
+expected1.setIsTransactional(true);
 assertTrue(components.contains(expected1));