Repository: hbase Updated Branches: refs/heads/branch-1 93d82112f -> 198165ef5
HBASE-16085 Add a metric for failed compactions Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/198165ef Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/198165ef Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/198165ef Branch: refs/heads/branch-1 Commit: 198165ef5bdef938e1f9b9f2c3c4752e30aba369 Parents: 93d8211 Author: Gary Helmling <ga...@apache.org> Authored: Wed Jun 22 17:08:28 2016 -0700 Committer: Gary Helmling <ga...@apache.org> Committed: Thu Jun 23 16:04:27 2016 -0700 ---------------------------------------------------------------------- .../hbase/regionserver/MetricsRegionSource.java | 2 + .../regionserver/MetricsRegionWrapper.java | 7 ++++ .../regionserver/MetricsRegionSourceImpl.java | 4 ++ .../TestMetricsRegionSourceImpl.java | 5 +++ .../hbase/regionserver/CompactSplitThread.java | 2 + .../hadoop/hbase/regionserver/HRegion.java | 10 ++++- .../regionserver/MetricsRegionWrapperImpl.java | 5 +++ .../regionserver/MetricsRegionWrapperStub.java | 5 +++ .../hbase/regionserver/TestCompaction.java | 43 ++++++++++++++++++++ 9 files changed, 81 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/198165ef/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java index 8dc7e11..911c757 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java @@ -28,9 +28,11 @@ public interface MetricsRegionSource extends Comparable<MetricsRegionSource> { String OPS_SAMPLE_NAME = "ops"; String SIZE_VALUE_NAME = "size"; String COMPACTIONS_COMPLETED_COUNT = "compactionsCompletedCount"; + String COMPACTIONS_FAILED_COUNT = "compactionsFailedCount"; String NUM_BYTES_COMPACTED_COUNT = "numBytesCompactedCount"; String NUM_FILES_COMPACTED_COUNT = "numFilesCompactedCount"; String COMPACTIONS_COMPLETED_DESC = "Number of compactions that have completed."; + String COMPACTIONS_FAILED_DESC = "Number of compactions that have failed."; String NUM_BYTES_COMPACTED_DESC = "Sum of filesize on all files entering a finished, successful or aborted, compaction"; String NUM_FILES_COMPACTED_DESC = http://git-wip-us.apache.org/repos/asf/hbase/blob/198165ef/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java index 5912abd..0482d2a 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java @@ -100,6 +100,13 @@ public interface MetricsRegionWrapper { long getNumCompactionsCompleted(); + /** + * Returns the total number of compactions that have been reported as failed on this region. + * Note that a given compaction can be reported as both completed and failed if an exception + * is thrown in the processing after {@code HRegion.compact()}. + */ + long getNumCompactionsFailed(); + int getRegionHashCode(); /** http://git-wip-us.apache.org/repos/asf/hbase/blob/198165ef/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java index c10081c..ae579cf 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java @@ -265,6 +265,10 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource { MetricsRegionSource.COMPACTIONS_COMPLETED_DESC), this.regionWrapper.getNumCompactionsCompleted()); mrb.addCounter(Interns.info( + regionNamePrefix + MetricsRegionSource.COMPACTIONS_FAILED_COUNT, + MetricsRegionSource.COMPACTIONS_FAILED_DESC), + this.regionWrapper.getNumCompactionsFailed()); + mrb.addCounter(Interns.info( regionNamePrefix + MetricsRegionSource.NUM_BYTES_COMPACTED_COUNT, MetricsRegionSource.NUM_BYTES_COMPACTED_DESC), this.regionWrapper.getNumBytesCompacted()); http://git-wip-us.apache.org/repos/asf/hbase/blob/198165ef/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java index 6a2173d..7498a74 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java @@ -149,6 +149,11 @@ public class TestMetricsRegionSourceImpl { } @Override + public long getNumCompactionsFailed() { + return 0; + } + + @Override public int getRegionHashCode() { return regionName.hashCode(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/198165ef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java index 4a40025..4ba07bf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java @@ -546,9 +546,11 @@ public class CompactSplitThread implements CompactionRequestor, PropagatingConfi if (remoteEx != ex) { LOG.info("Compaction failed at original callstack: " + formatStackTrace(ex)); } + region.reportCompactionRequestFailure(); server.checkFileSystem(); } catch (Exception ex) { LOG.error("Compaction failed " + this, ex); + region.reportCompactionRequestFailure(); server.checkFileSystem(); } finally { LOG.debug("CompactSplitThread Status: " + CompactSplitThread.this); http://git-wip-us.apache.org/repos/asf/hbase/blob/198165ef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 25535ed..f856fb8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -291,6 +291,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi // Compaction counters final AtomicLong compactionsFinished = new AtomicLong(0L); + final AtomicLong compactionsFailed = new AtomicLong(0L); final AtomicLong compactionNumFilesCompacted = new AtomicLong(0L); final AtomicLong compactionNumBytesCompacted = new AtomicLong(0L); @@ -7884,7 +7885,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi public static final long FIXED_OVERHEAD = ClassSize.align( ClassSize.OBJECT + ClassSize.ARRAY + - 44 * ClassSize.REFERENCE + 3 * Bytes.SIZEOF_INT + + 45 * ClassSize.REFERENCE + 3 * Bytes.SIZEOF_INT + (14 * Bytes.SIZEOF_LONG) + 5 * Bytes.SIZEOF_BOOLEAN); @@ -7901,7 +7902,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi public static final long DEEP_OVERHEAD = FIXED_OVERHEAD + ClassSize.OBJECT + // closeLock (2 * ClassSize.ATOMIC_BOOLEAN) + // closed, closing - (3 * ClassSize.ATOMIC_LONG) + // memStoreSize, numPutsWithoutWAL, dataInMemoryWithoutWAL + (4 * ClassSize.ATOMIC_LONG) + // memStoreSize, numPutsWithoutWAL, dataInMemoryWithoutWAL, + // compactionsFailed (2 * ClassSize.CONCURRENT_HASHMAP) + // lockedRows, scannerReadPoints WriteState.HEAP_SIZE + // writestate ClassSize.CONCURRENT_SKIPLISTMAP + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + // stores @@ -8449,6 +8451,10 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi assert newValue >= 0; } + public void reportCompactionRequestFailure() { + compactionsFailed.incrementAndGet(); + } + /** * Do not change this sequence id. * @return sequenceId http://git-wip-us.apache.org/repos/asf/hbase/blob/198165ef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java index 1b6aa01..46e13f7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java @@ -136,6 +136,11 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable } @Override + public long getNumCompactionsFailed() { + return this.region.compactionsFailed.get(); + } + + @Override public long getMaxStoreFileAge() { return maxStoreFileAge; } http://git-wip-us.apache.org/repos/asf/hbase/blob/198165ef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java index f90409a..ab340d9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java @@ -116,6 +116,11 @@ public class MetricsRegionWrapperStub implements MetricsRegionWrapper { } @Override + public long getNumCompactionsFailed() { + return 0; + } + + @Override public int getRegionHashCode() { return 42; } http://git-wip-us.apache.org/repos/asf/hbase/blob/198165ef/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java index 72436e9..da0bf42 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java @@ -36,6 +36,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -299,6 +300,48 @@ public class TestCompaction { thread.interruptIfNecessary(); } + @Test + public void testCompactionFailure() throws Exception { + // setup a compact/split thread on a mock server + HRegionServer mockServer = Mockito.mock(HRegionServer.class); + Mockito.when(mockServer.getConfiguration()).thenReturn(r.getBaseConf()); + CompactSplitThread thread = new CompactSplitThread(mockServer); + Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread); + + // setup a region/store with some files + Store store = r.getStore(COLUMN_FAMILY); + createStoreFile(r); + for (int i = 0; i < HStore.DEFAULT_BLOCKING_STOREFILE_COUNT - 1; i++) { + createStoreFile(r); + } + + HRegion mockRegion = Mockito.spy(r); + Mockito.when(mockRegion.checkSplit()).thenThrow(new IndexOutOfBoundsException()); + + MetricsRegionWrapper metricsWrapper = new MetricsRegionWrapperImpl(r); + + long preCompletedCount = metricsWrapper.getNumCompactionsCompleted(); + long preFailedCount = metricsWrapper.getNumCompactionsFailed(); + + CountDownLatch latch = new CountDownLatch(1); + TrackableCompactionRequest request = new TrackableCompactionRequest(latch); + thread.requestCompaction(mockRegion, store, "test custom comapction", Store.PRIORITY_USER, + request, null); + // wait for the latch to complete. + latch.await(120, TimeUnit.SECONDS); + + // compaction should have completed and been marked as failed due to error in split request + long postCompletedCount = metricsWrapper.getNumCompactionsCompleted(); + long postFailedCount = metricsWrapper.getNumCompactionsFailed(); + + assertTrue("Completed count should have increased (pre=" + preCompletedCount + + ", post="+postCompletedCount+")", + postCompletedCount > preCompletedCount); + assertTrue("Failed count should have increased (pre=" + preFailedCount + + ", post=" + postFailedCount + ")", + postFailedCount > preFailedCount); + } + /** * HBASE-7947: Regression test to ensure adding to the correct list in the * {@link CompactSplitThread}
