[hive] branch master updated: HIVE-25716: Fix of flaky test TestCompactionMetrics#testOldestReadyForCleaningAge (Viktor Csomor, reviewed by Denys Kuzmenko and Antal Sinkovits)
This is an automated email from the ASF dual-hosted git repository.
asinkovits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 4566f07 HIVE-25716: Fix of flaky test
TestCompactionMetrics#testOldestReadyForCleaningAge (Viktor Csomor, reviewed by
Denys Kuzmenko and Antal Sinkovits)
4566f07 is described below
commit 4566f076a2b3d6d87b258570a373dd60f7152786
Author: Viktor Csomor
AuthorDate: Thu Dec 9 11:15:45 2021 +0100
HIVE-25716: Fix of flaky test
TestCompactionMetrics#testOldestReadyForCleaningAge (Viktor Csomor, reviewed by
Denys Kuzmenko and Antal Sinkovits)
Closes #2837
---
.../hive/ql/txn/compactor/CompactorTest.java | 10 ++--
.../ql/txn/compactor/TestCompactionMetrics.java| 60 ++
2 files changed, 44 insertions(+), 26 deletions(-)
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
index 5dc01f9..c2b78ad 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
@@ -292,7 +292,8 @@ public abstract class CompactorTest {
burnThroughTransactions(dbName, tblName, num, open, aborted, null);
}
- protected void burnThroughTransactions(String dbName, String tblName, int
num, Set open, Set aborted, LockRequest lockReq)
+ protected void burnThroughTransactions(String dbName, String tblName, int
num, Set open, Set aborted,
+ LockRequest lockReq)
throws MetaException, NoSuchTxnException, TxnAbortedException {
OpenTxnsResponse rsp = txnHandler.openTxns(new OpenTxnRequest(num, "me",
"localhost"));
AllocateTableWriteIdsRequest awiRqst = new
AllocateTableWriteIdsRequest(dbName, tblName);
@@ -300,14 +301,15 @@ public abstract class CompactorTest {
AllocateTableWriteIdsResponse awiResp =
txnHandler.allocateTableWriteIds(awiRqst);
int i = 0;
for (long tid : rsp.getTxn_ids()) {
- assert(awiResp.getTxnToWriteIds().get(i++).getTxnId() == tid);
- if(lockReq != null) {
+ assert (awiResp.getTxnToWriteIds().get(i).getTxnId() == tid);
+ ++i;
+ if (lockReq != null) {
lockReq.setTxnid(tid);
txnHandler.lock(lockReq);
}
if (aborted != null && aborted.contains(tid)) {
txnHandler.abortTxn(new AbortTxnRequest(tid));
- } else if (open == null || (open != null && !open.contains(tid))) {
+ } else if (open == null || !open.contains(tid)) {
txnHandler.commitTxn(new CommitTxnRequest(tid));
}
}
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCompactionMetrics.java
b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCompactionMetrics.java
index 75c722b..eda09c1 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCompactionMetrics.java
+++
b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCompactionMetrics.java
@@ -179,41 +179,49 @@ public class TestCompactionMetrics extends CompactorTest
{
}
@Test
- @org.junit.Ignore("HIVE-25716")
public void testOldestReadyForCleaningAge() throws Exception {
conf.setIntVar(HiveConf.ConfVars.COMPACTOR_MAX_NUM_DELTA, 1);
-long oldStart = System.currentTimeMillis();
-Table old = newTable("default", "old_rfc", true);
-Partition oldP = newPartition(old, "part");
+final String DB_NAME = "default";
+final String OLD_TABLE_NAME = "old_rfc";
+final String OLD_PARTITION_NAME = "part";
+final String YOUNG_TABLE_NAME = "young_rfc";
+final String YOUNG_PARTITION_NAME = "part";
+
+long oldTableStart = System.currentTimeMillis();
+Table old = newTable(DB_NAME, OLD_TABLE_NAME, true);
+Partition oldP = newPartition(old, OLD_PARTITION_NAME);
addBaseFile(old, oldP, 20L, 20);
addDeltaFile(old, oldP, 21L, 22L, 2);
addDeltaFile(old, oldP, 23L, 24L, 2);
-burnThroughTransactions("default", "old_rfc", 25);
-CompactionRequest rqst = new CompactionRequest("default", "old_rfc",
CompactionType.MINOR);
-rqst.setPartitionname("ds=part");
-txnHandler.compact(rqst);
-startWorker();
+burnThroughTransactions(DB_NAME, OLD_TABLE_NAME, 25);
+doCompaction(DB_NAME, OLD_TABLE_NAME, OLD_PARTITION_NAME,
CompactionType.MINOR);
+long oldTableEnd = System.currentTimeMillis();
-long youngStart = System.currentTimeMillis();
-Table young = newTable("default", "young_rfc", true);
-Partition youngP = newPartition(young, "part");
+Table young = newTable(DB_NAME, YOUNG_TABLE_NAME, true);
+Partition youngP = newPartition(young, YOUNG_PARTITION_NAME);
addBaseFile(young, youngP, 20L, 20);
addDeltaFile(young, youngP, 21L, 22L, 2);
addDeltaFile(young, youngP, 23L, 24L, 2);
-burnThroughTransactions("default", "young_rfc", 25);
-
[hive] branch master updated: HIVE-24975: Fix a bug in ValidWriteIdList comparison in TxnIdUtils (#2641) (Sourabh Goyal reviewed by Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 9f9844d HIVE-24975: Fix a bug in ValidWriteIdList comparison in
TxnIdUtils (#2641) (Sourabh Goyal reviewed by Zoltan Haindrich)
9f9844d is described below
commit 9f9844dbc881e2a9267c259b8c04e7787f7fadc4
Author: Sourabh Goyal
AuthorDate: Thu Dec 9 01:21:08 2021 -0800
HIVE-24975: Fix a bug in ValidWriteIdList comparison in TxnIdUtils (#2641)
(Sourabh Goyal reviewed by Zoltan Haindrich)
---
.../src/java/org/apache/hive/common/util/TxnIdUtils.java | 2 +-
.../src/test/org/apache/hive/common/util/TestTxnIdUtils.java | 11 +++
2 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java
b/storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java
index bd972d4..7de7464 100644
--- a/storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java
+++ b/storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java
@@ -67,7 +67,7 @@ public class TxnIdUtils {
}
} else {
if (b.getHighWatermark() != a.getInvalidWriteIds()[minLen] -1) {
-return Long.signum(b.getHighWatermark() -
(a.getInvalidWriteIds()[minLen] -1));
+return Long.signum((a.getInvalidWriteIds()[minLen] - 1) -
b.getHighWatermark());
}
if (allInvalidFrom(a.getInvalidWriteIds(), minLen,
a.getHighWatermark())) {
return 0;
diff --git
a/storage-api/src/test/org/apache/hive/common/util/TestTxnIdUtils.java
b/storage-api/src/test/org/apache/hive/common/util/TestTxnIdUtils.java
index ab5a472..1f646a3 100644
--- a/storage-api/src/test/org/apache/hive/common/util/TestTxnIdUtils.java
+++ b/storage-api/src/test/org/apache/hive/common/util/TestTxnIdUtils.java
@@ -19,6 +19,7 @@ package org.apache.hive.common.util;
import java.util.BitSet;
+import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.common.ValidReaderWriteIdList;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
@@ -190,5 +191,15 @@ public class TestTxnIdUtils {
new ValidReaderWriteIdList("default.table2", new long[] {8,10,11}, new
BitSet(), 11)),
-1);
+ValidWriteIdList a =
+new ValidReaderWriteIdList("default.test:1:1:1:");
+ValidWriteIdList b =
+new ValidReaderWriteIdList("default.test:1:9223372036854775807::");
+
+// should return -1 since b is more recent
+assertEquals(TxnIdUtils.compare(a, b), -1);
+
+// should return 1 since b is more recent
+assertEquals(TxnIdUtils.compare(b, a), 1);
}
}
[hive] branch master updated: HIVE-25737: Compaction Observability: Initiator/Worker/Cleaner cycle measurement improvements (Viktor Csomor, reviewed by Laszlo Pinter and Karen Coppage)
This is an automated email from the ASF dual-hosted git repository.
klcopp pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 6a53540 HIVE-25737: Compaction Observability:
Initiator/Worker/Cleaner cycle measurement improvements (Viktor Csomor,
reviewed by Laszlo Pinter and Karen Coppage)
6a53540 is described below
commit 6a535403a49b972bb0a488d65bab06c57e3c8c66
Author: Viktor Csomor
AuthorDate: Thu Dec 9 09:32:33 2021 +0100
HIVE-25737: Compaction Observability: Initiator/Worker/Cleaner cycle
measurement improvements (Viktor Csomor, reviewed by Laszlo Pinter and Karen
Coppage)
Closes #2827.
A daemon thread has been implemented for the Initiator that measures the
elapsed time since its start.
The PerformanceLogger approach is also kept but the metrics intended to use
the gauge style
The Age of Oldest Working Compaction metric has been implemented in the
AcidMetricService
The Age of Oldest active Cleaner metric has been implemented
- CQ_CLEANER_START field added to the COMPACTION_QUEUE table
- COMPACTION_OLDEST_CLEANING_AGE metric has been added
- markCleaning method added to the CompactionTxnHandler
- ShowCompactResponseElement extended with the cleanerStart field
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 16 ++
.../upgrade/hive/hive-schema-4.0.0.hive.sql| 21 ++-
.../upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql | 21 ++-
.../hadoop/hive/ql/txn/compactor/Cleaner.java | 31
.../hadoop/hive/ql/txn/compactor/Initiator.java| 55 +-
.../ql/txn/compactor/MetaStoreCompactorThread.java | 39 +
.../ql/txn/compactor/TestCompactionMetrics.java| 184 +
.../test/results/clientpositive/llap/sysdb.q.out | 11 +-
.../gen/thrift/gen-cpp/hive_metastore_types.cpp| 22 +++
.../src/gen/thrift/gen-cpp/hive_metastore_types.h | 12 +-
.../metastore/api/ShowCompactResponseElement.java | 106 +++-
.../metastore/ShowCompactResponseElement.php | 24 +++
.../src/gen/thrift/gen-py/hive_metastore/ttypes.py | 14 +-
.../src/gen/thrift/gen-rb/hive_metastore_types.rb | 4 +-
.../hadoop/hive/metastore/conf/MetastoreConf.java | 12 ++
.../src/main/thrift/hive_metastore.thrift | 3 +-
.../hive/metastore/metrics/AcidMetricService.java | 71 +---
.../hive/metastore/metrics/MetricsConstants.java | 4 +
.../hive/metastore/txn/CompactionTxnHandler.java | 133 +--
.../hadoop/hive/metastore/txn/TxnHandler.java | 39 +++--
.../apache/hadoop/hive/metastore/txn/TxnStore.java | 66 +++-
.../src/main/sql/derby/hive-schema-4.0.0.derby.sql | 3 +-
.../sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql | 3 +
.../src/main/sql/mssql/hive-schema-4.0.0.mssql.sql | 1 +
.../sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql | 3 +
.../src/main/sql/mysql/hive-schema-4.0.0.mysql.sql | 3 +-
.../sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql | 3 +
.../main/sql/oracle/hive-schema-4.0.0.oracle.sql | 3 +-
.../sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql | 4 +-
.../sql/postgres/hive-schema-4.0.0.postgres.sql| 3 +-
.../postgres/upgrade-3.2.0-to-4.0.0.postgres.sql | 4 +-
.../upgrade-3.1.3000-to-4.0.0.postgres.sql | 3 +
32 files changed, 800 insertions(+), 121 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 988cec8..d9eec2e 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3190,6 +3190,22 @@ public class HiveConf extends Configuration {
"has had a transaction done on it since the last major compaction. So
decreasing this\n" +
"value will increase the load on the NameNode."),
+
HIVE_COMPACTOR_INITIATOR_DURATION_UPDATE_INTERVAL("hive.compactor.initiator.duration.update.interval",
"60s",
+new TimeValidator(TimeUnit.SECONDS),
+"Time in seconds that drives the update interval of
compaction_initiator_duration metric.\n" +
+"Smaller value results in a fine grained metric update.\n" +
+"This updater can be turned off if its value less than or equals
to zero.\n"+
+"In this case the above metric will be update only after the
initiator completed one cycle.\n" +
+"The hive.compactor.initiator.on must be turned on (true) in-order
to enable the Initiator,\n" +
+"otherwise this setting has no effect."),
+
+
HIVE_COMPACTOR_CLEANER_DURATION_UPDATE_INTERVAL("hive.compactor.cleaner.duration.update.interval",
"60s",
+new TimeValidator(TimeUnit.SECONDS),
+"Time in seconds that drives the update interval of
compaction_cleaner_duration metric.\n" +
+"Smaller value results in a fine
