[25/51] [abbrv] hive git commit: HIVE-10632 : Make sure TXN_COMPONENTS gets cleaned up if table is dropped before compaction (Wei Zheng, reviewed by Alan Gates)
HIVE-10632 : Make sure TXN_COMPONENTS gets cleaned up if table is dropped
before compaction (Wei Zheng, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/456a91ec
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/456a91ec
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/456a91ec
Branch: refs/heads/llap
Commit: 456a91ecde6a449177a76fb34ad9b5f13983821b
Parents: ff55d0a
Author: Wei Zheng
Authored: Thu Mar 10 14:37:35 2016 -0800
Committer: Wei Zheng
Committed: Thu Mar 10 14:37:35 2016 -0800
--
.../hive/metastore/AcidEventListener.java | 94 +
.../hadoop/hive/metastore/HiveMetaStore.java| 1 +
.../hadoop/hive/metastore/txn/TxnDbUtil.java| 20 +-
.../hadoop/hive/metastore/txn/TxnHandler.java | 167 ++-
.../hadoop/hive/metastore/txn/TxnStore.java | 37 ++--
.../hadoop/hive/metastore/txn/TxnUtils.java | 18 ++
.../org/apache/hadoop/hive/ql/io/AcidUtils.java | 27 +--
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 22 +-
.../hive/ql/lockmgr/TestDbTxnManager2.java | 209 ++-
9 files changed, 518 insertions(+), 77 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/456a91ec/metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
--
diff --git
a/metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
new file mode 100644
index 000..71ad916
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.HiveObjectType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.apache.hadoop.hive.metastore.txn.TxnStore;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+
+
+/**
+ * It handles cleanup of dropped partition/table/database in ACID related
metastore tables
+ */
+public class AcidEventListener extends MetaStoreEventListener {
+
+ private TxnStore txnHandler;
+ private HiveConf hiveConf;
+
+ public AcidEventListener(Configuration configuration) {
+super(configuration);
+hiveConf = (HiveConf) configuration;
+ }
+
+ @Override
+ public void onDropDatabase (DropDatabaseEvent dbEvent) throws MetaException {
+// We can loop thru all the tables to check if they are ACID first and
then perform cleanup,
+// but it's more efficient to unconditionally perform cleanup for the
database, especially
+// when there are a lot of tables
+txnHandler = getTxnHandler();
+txnHandler.cleanupRecords(HiveObjectType.DATABASE, dbEvent.getDatabase(),
null, null);
+ }
+
+ @Override
+ public void onDropTable(DropTableEvent tableEvent) throws MetaException {
+if (TxnUtils.isAcidTable(tableEvent.getTable())) {
+ txnHandler = getTxnHandler();
+ txnHandler.cleanupRecords(HiveObjectType.TABLE, null,
tableEvent.getTable(), null);
+}
+ }
+
+ @Override
+ public void onDropPartition(DropPartitionEvent partitionEvent) throws
MetaException {
+if (TxnUtils.isAcidTable(partitionEvent.getTable())) {
+ txnHandler = getTxnHandler();
+ txnHandler.cleanupRecords(HiveObjectType.PARTITION, null,
partitionEvent.getTable(),
+ partitionEvent.getPartitionIterator());
+}
+ }
+
+ private TxnStore getTxnHandler() {
+boolean hackOn = HiveConf.getBoolVar(hiveConf,
HiveConf.ConfVars.HIVE_IN_TEST) ||
+HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_IN_TEZ_TEST);
+String origTxnMgr = null;
+
hive git commit: HIVE-10632 : Make sure TXN_COMPONENTS gets cleaned up if table is dropped before compaction (Wei Zheng, reviewed by Alan Gates)
Repository: hive
Updated Branches:
refs/heads/branch-1 73a677be3 -> 214e4b6ff
HIVE-10632 : Make sure TXN_COMPONENTS gets cleaned up if table is dropped
before compaction (Wei Zheng, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/214e4b6f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/214e4b6f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/214e4b6f
Branch: refs/heads/branch-1
Commit: 214e4b6ffedbdc0f610babcf1156cd32f0659db3
Parents: 73a677b
Author: Wei Zheng
Authored: Thu Mar 10 16:57:26 2016 -0800
Committer: Wei Zheng
Committed: Thu Mar 10 16:57:26 2016 -0800
--
.../hive/metastore/AcidEventListener.java | 69 +++
.../hadoop/hive/metastore/HiveMetaStore.java| 1 +
.../hadoop/hive/metastore/txn/TxnDbUtil.java| 20 +-
.../hadoop/hive/metastore/txn/TxnHandler.java | 183 +-
.../org/apache/hadoop/hive/ql/io/AcidUtils.java | 25 +--
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 22 +--
.../hive/ql/lockmgr/TestDbTxnManager2.java | 186 +++
7 files changed, 460 insertions(+), 46 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/214e4b6f/metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
--
diff --git
a/metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
new file mode 100644
index 000..767bc54
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.HiveObjectType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.apache.hadoop.hive.metastore.txn.TxnHandler;
+
+
+/**
+ * It handles cleanup of dropped partition/table/database in ACID related
metastore tables
+ */
+public class AcidEventListener extends MetaStoreEventListener {
+
+ private TxnHandler txnHandler;
+ private HiveConf hiveConf;
+
+ public AcidEventListener(Configuration configuration) {
+super(configuration);
+hiveConf = (HiveConf) configuration;
+ }
+
+ @Override
+ public void onDropDatabase (DropDatabaseEvent dbEvent) throws MetaException {
+// We can loop thru all the tables to check if they are ACID first and
then perform cleanup,
+// but it's more efficient to unconditionally perform cleanup for the
database, especially
+// when there are a lot of tables
+txnHandler = new TxnHandler(hiveConf);
+txnHandler.cleanupRecords(HiveObjectType.DATABASE, dbEvent.getDatabase(),
null, null);
+ }
+
+ @Override
+ public void onDropTable(DropTableEvent tableEvent) throws MetaException {
+if (TxnHandler.isAcidTable(tableEvent.getTable())) {
+ txnHandler = new TxnHandler(hiveConf);
+ txnHandler.cleanupRecords(HiveObjectType.TABLE, null,
tableEvent.getTable(), null);
+}
+ }
+
+ @Override
+ public void onDropPartition(DropPartitionEvent partitionEvent) throws
MetaException {
+if (TxnHandler.isAcidTable(partitionEvent.getTable())) {
+ txnHandler = new TxnHandler(hiveConf);
+ txnHandler.cleanupRecords(HiveObjectType.PARTITION, null,
partitionEvent.getTable(),
+ partitionEvent.getPartitionIterator());
+}
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/214e4b6f/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
--
diff --git
a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.ja
hive git commit: HIVE-10632 : Make sure TXN_COMPONENTS gets cleaned up if table is dropped before compaction (Wei Zheng, reviewed by Alan Gates)
Repository: hive
Updated Branches:
refs/heads/master ff55d0a67 -> 456a91ecd
HIVE-10632 : Make sure TXN_COMPONENTS gets cleaned up if table is dropped
before compaction (Wei Zheng, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/456a91ec
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/456a91ec
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/456a91ec
Branch: refs/heads/master
Commit: 456a91ecde6a449177a76fb34ad9b5f13983821b
Parents: ff55d0a
Author: Wei Zheng
Authored: Thu Mar 10 14:37:35 2016 -0800
Committer: Wei Zheng
Committed: Thu Mar 10 14:37:35 2016 -0800
--
.../hive/metastore/AcidEventListener.java | 94 +
.../hadoop/hive/metastore/HiveMetaStore.java| 1 +
.../hadoop/hive/metastore/txn/TxnDbUtil.java| 20 +-
.../hadoop/hive/metastore/txn/TxnHandler.java | 167 ++-
.../hadoop/hive/metastore/txn/TxnStore.java | 37 ++--
.../hadoop/hive/metastore/txn/TxnUtils.java | 18 ++
.../org/apache/hadoop/hive/ql/io/AcidUtils.java | 27 +--
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 22 +-
.../hive/ql/lockmgr/TestDbTxnManager2.java | 209 ++-
9 files changed, 518 insertions(+), 77 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/456a91ec/metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
--
diff --git
a/metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
new file mode 100644
index 000..71ad916
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.HiveObjectType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.apache.hadoop.hive.metastore.txn.TxnStore;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+
+
+/**
+ * It handles cleanup of dropped partition/table/database in ACID related
metastore tables
+ */
+public class AcidEventListener extends MetaStoreEventListener {
+
+ private TxnStore txnHandler;
+ private HiveConf hiveConf;
+
+ public AcidEventListener(Configuration configuration) {
+super(configuration);
+hiveConf = (HiveConf) configuration;
+ }
+
+ @Override
+ public void onDropDatabase (DropDatabaseEvent dbEvent) throws MetaException {
+// We can loop thru all the tables to check if they are ACID first and
then perform cleanup,
+// but it's more efficient to unconditionally perform cleanup for the
database, especially
+// when there are a lot of tables
+txnHandler = getTxnHandler();
+txnHandler.cleanupRecords(HiveObjectType.DATABASE, dbEvent.getDatabase(),
null, null);
+ }
+
+ @Override
+ public void onDropTable(DropTableEvent tableEvent) throws MetaException {
+if (TxnUtils.isAcidTable(tableEvent.getTable())) {
+ txnHandler = getTxnHandler();
+ txnHandler.cleanupRecords(HiveObjectType.TABLE, null,
tableEvent.getTable(), null);
+}
+ }
+
+ @Override
+ public void onDropPartition(DropPartitionEvent partitionEvent) throws
MetaException {
+if (TxnUtils.isAcidTable(partitionEvent.getTable())) {
+ txnHandler = getTxnHandler();
+ txnHandler.cleanupRecords(HiveObjectType.PARTITION, null,
partitionEvent.getTable(),
+ partitionEvent.getPartitionIterator());
+}
+ }
+
+ private TxnStore getTxnHandler() {
+boolean hackOn = HiveConf.getBoolVar(hiveConf,
HiveConf.ConfVars.HIVE_IN_TEST) ||
+HiveConf.getBoolV
