HIVE-16360 Improve 'No delta files or original files found to compact in' message (Eugene Koifman, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a3b4bf89 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a3b4bf89 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a3b4bf89 Branch: refs/heads/hive-14535 Commit: a3b4bf89c4200e0a6eb6de47ecb12940a0f8b885 Parents: 1a7f853 Author: Eugene Koifman <[email protected]> Authored: Tue Sep 26 08:48:05 2017 -0700 Committer: Eugene Koifman <[email protected]> Committed: Tue Sep 26 09:16:22 2017 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/common/ValidCompactorTxnList.java | 8 ++++++-- .../org/apache/hadoop/hive/common/ValidReadTxnList.java | 7 ++++--- .../java/org/apache/hadoop/hive/common/ValidTxnList.java | 4 ++++ .../apache/hadoop/hive/ql/txn/compactor/TestCompactor.java | 4 +++- .../java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java | 8 +++++++- .../hive/metastore/txn/TestValidCompactorTxnList.java | 6 +++--- .../apache/hadoop/hive/ql/txn/compactor/CompactorMR.java | 9 +++++++-- 7 files changed, 34 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java b/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java index c022577..eaa0b34 100644 --- a/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java +++ b/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java @@ -39,14 +39,18 @@ public class ValidCompactorTxnList extends ValidReadTxnList { public ValidCompactorTxnList() { super(); } + public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark) { + this(abortedTxnList, abortedBits, highWatermark, Long.MAX_VALUE); + } /** * @param abortedTxnList list of all aborted transactions * @param abortedBits bitset marking whether the corresponding transaction is aborted * @param highWatermark highest committed transaction to be considered for compaction, * equivalently (lowest_open_txn - 1). */ - public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark) { - super(abortedTxnList, abortedBits, highWatermark); // abortedBits should be all true as everything in exceptions are aborted txns + public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark, long minOpenTxnId) { + // abortedBits should be all true as everything in exceptions are aborted txns + super(abortedTxnList, abortedBits, highWatermark, minOpenTxnId); if(this.exceptions.length <= 0) { return; } http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java b/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java index 4e57772..002afd6 100644 --- a/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java +++ b/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java @@ -193,9 +193,10 @@ public class ValidReadTxnList implements ValidTxnList { public long[] getInvalidTransactions() { return exceptions; } - @VisibleForTesting - public long getMinOpenTxn() { - return minOpenTxn; + + @Override + public Long getMinOpenTxn() { + return minOpenTxn == Long.MAX_VALUE ? null : minOpenTxn; } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java b/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java index d4ac02c..108e5ca 100644 --- a/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java +++ b/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java @@ -105,4 +105,8 @@ public interface ValidTxnList { */ public RangeResponse isTxnRangeAborted(long minTxnId, long maxTxnId); + /** + * Returns smallest Open transaction in this set, {@code null} if there is none. + */ + Long getMinOpenTxn(); } http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java index 4bfc5be..aea1dfc 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java @@ -1150,7 +1150,6 @@ public class TestCompactor { /** * Users have the choice of specifying compaction related tblproperties either in CREATE TABLE * statement or in ALTER TABLE .. COMPACT statement. This tests both cases. - * @throws Exception */ @Test public void testTableProperties() throws Exception { @@ -1320,6 +1319,9 @@ public class TestCompactor { } @Override + public Long getMinOpenTxn() { return null; } + + @Override public long getHighWatermark() { return Long.MAX_VALUE; } http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java index e6c62d3..30b155f 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java @@ -81,6 +81,7 @@ public class TxnUtils { * @return a valid txn list. */ public static ValidTxnList createValidCompactTxnList(GetOpenTxnsInfoResponse txns) { + //highWater is the last txn id that has been allocated long highWater = txns.getTxn_high_water_mark(); long minOpenTxn = Long.MAX_VALUE; long[] exceptions = new long[txns.getOpen_txnsSize()]; @@ -100,7 +101,12 @@ public class TxnUtils { highWater = minOpenTxn == Long.MAX_VALUE ? highWater : minOpenTxn - 1; BitSet bitSet = new BitSet(exceptions.length); bitSet.set(0, exceptions.length); // for ValidCompactorTxnList, everything in exceptions are aborted - return new ValidCompactorTxnList(exceptions, bitSet, highWater); + if(minOpenTxn == Long.MAX_VALUE) { + return new ValidCompactorTxnList(exceptions, bitSet, highWater); + } + else { + return new ValidCompactorTxnList(exceptions, bitSet, highWater, minOpenTxn); + } } /** http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java index 91d6215..eb88e32 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java @@ -103,11 +103,11 @@ public class TestValidCompactorTxnList { public void readFromString() { ValidCompactorTxnList txns = new ValidCompactorTxnList("37:" + Long.MAX_VALUE + "::7,9,10"); Assert.assertEquals(37L, txns.getHighWatermark()); - Assert.assertEquals(Long.MAX_VALUE, txns.getMinOpenTxn()); + Assert.assertNull(txns.getMinOpenTxn()); Assert.assertArrayEquals(new long[]{7L, 9L, 10L}, txns.getInvalidTransactions()); txns = new ValidCompactorTxnList("21:" + Long.MAX_VALUE + ":"); Assert.assertEquals(21L, txns.getHighWatermark()); - Assert.assertEquals(Long.MAX_VALUE, txns.getMinOpenTxn()); + Assert.assertNull(txns.getMinOpenTxn()); Assert.assertEquals(0, txns.getInvalidTransactions().length); } @@ -115,7 +115,7 @@ public class TestValidCompactorTxnList { public void testAbortedTxn() throws Exception { ValidCompactorTxnList txnList = new ValidCompactorTxnList("5:4::1,2,3"); Assert.assertEquals(5L, txnList.getHighWatermark()); - Assert.assertEquals(4, txnList.getMinOpenTxn()); + Assert.assertEquals(4, txnList.getMinOpenTxn().longValue()); Assert.assertArrayEquals(new long[]{1L, 2L, 3L}, txnList.getInvalidTransactions()); } http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java index 04ef7fc..7f5c6d3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java @@ -265,10 +265,15 @@ public class CompactorMR { dirsToSearch.add(baseDir); } } - if (parsedDeltas.size() == 0 && dir.getOriginalFiles().size() == 0) { // Skip compaction if there's no delta files AND there's no original files - LOG.error("No delta files or original files found to compact in " + sd.getLocation() + " for compactionId=" + ci.id); + String minOpenInfo = "."; + if(txns.getMinOpenTxn() != null) { + minOpenInfo = " with min Open " + JavaUtils.txnIdToString(txns.getMinOpenTxn()) + + ". Compaction cannot compact above this txnid"; + } + LOG.error("No delta files or original files found to compact in " + sd.getLocation() + + " for compactionId=" + ci.id + minOpenInfo); return; }
