hbase git commit: HBASE-16561 Add metrics about read/write/scan queue length and active read/write/scan handler count
Repository: hbase Updated Branches: refs/heads/branch-1 cdf539a8e -> 5ec218dbc HBASE-16561 Add metrics about read/write/scan queue length and active read/write/scan handler count Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5ec218db Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5ec218db Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5ec218db Branch: refs/heads/branch-1 Commit: 5ec218dbc22624406e0777d7987533e2429ca334 Parents: cdf539a Author: Guanghao Zhang Authored: Tue Nov 29 09:50:12 2016 +0800 Committer: zhangduo Committed: Tue Nov 29 16:00:37 2016 +0800 -- .../hbase/ipc/MetricsHBaseServerSource.java | 15 + .../hbase/ipc/MetricsHBaseServerWrapper.java| 19 ++ .../hbase/ipc/MetricsHBaseServerSourceImpl.java | 14 +++- .../ipc/FastPathBalancedQueueRpcExecutor.java | 9 ++- .../hadoop/hbase/ipc/FifoRpcScheduler.java | 30 + .../ipc/MetricsHBaseServerWrapperImpl.java | 48 + .../hadoop/hbase/ipc/RWQueueRpcExecutor.java| 62 - .../apache/hadoop/hbase/ipc/RpcExecutor.java| 71 ++-- .../apache/hadoop/hbase/ipc/RpcScheduler.java | 18 + .../hadoop/hbase/ipc/SimpleRpcScheduler.java| 30 + .../hbase/ipc/DelegatingRpcScheduler.java | 32 - .../ipc/MetricsHBaseServerWrapperStub.java | 30 + .../apache/hadoop/hbase/ipc/TestRpcMetrics.java | 7 +- 13 files changed, 355 insertions(+), 30 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/5ec218db/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java -- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java index ac14bd8..8bee67e 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java @@ -60,10 +60,25 @@ public interface MetricsHBaseServerSource extends BaseSource { String REPLICATION_QUEUE_DESC = "Number of calls in the replication call queue."; String PRIORITY_QUEUE_DESC = "Number of calls in the priority call queue."; + String WRITE_QUEUE_NAME = "numCallsInWriteQueue"; + String WRITE_QUEUE_DESC = "Number of calls in the write call queue; " + +"parsed requests waiting in scheduler to be executed"; + String READ_QUEUE_NAME = "numCallsInReadQueue"; + String READ_QUEUE_DESC = "Number of calls in the read call queue; " + +"parsed requests waiting in scheduler to be executed"; + String SCAN_QUEUE_NAME = "numCallsInScanQueue"; + String SCAN_QUEUE_DESC = "Number of calls in the scan call queue; " + +"parsed requests waiting in scheduler to be executed"; String NUM_OPEN_CONNECTIONS_NAME = "numOpenConnections"; String NUM_OPEN_CONNECTIONS_DESC = "Number of open connections."; String NUM_ACTIVE_HANDLER_NAME = "numActiveHandler"; String NUM_ACTIVE_HANDLER_DESC = "Number of active rpc handlers."; + String NUM_ACTIVE_WRITE_HANDLER_NAME = "numActiveWriteHandler"; + String NUM_ACTIVE_WRITE_HANDLER_DESC = "Number of active write rpc handlers."; + String NUM_ACTIVE_READ_HANDLER_NAME = "numActiveReadHandler"; + String NUM_ACTIVE_READ_HANDLER_DESC = "Number of active read rpc handlers."; + String NUM_ACTIVE_SCAN_HANDLER_NAME = "numActiveScanHandler"; + String NUM_ACTIVE_SCAN_HANDLER_DESC = "Number of active scan rpc handlers."; String NUM_GENERAL_CALLS_DROPPED_NAME = "numGeneralCallsDropped"; String NUM_GENERAL_CALLS_DROPPED_DESC = "Total number of calls in general queue which " + "were dropped by CoDel RPC executor"; http://git-wip-us.apache.org/repos/asf/hbase/blob/5ec218db/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapper.java -- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapper.java index 8f30205..b272cd0 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapper.java @@ -21,11 +21,30 @@ package org.apache.hadoop.hbase.ipc; public interface MetricsHBaseServerWrapper { long getTotalQueueSize(); + int getGeneralQueueLength(); + int getReplicationQueueLength(); + int getPriorityQueueLength(); + int getNumOpenConnections(); + int g
[1/2] hbase git commit: Revert "HBASE-16561 Add metrics about read/write/scan queue length and active read/write/scan handler count"
Repository: hbase Updated Branches: refs/heads/branch-1 5ec218dbc -> 7b2673db1 Revert "HBASE-16561 Add metrics about read/write/scan queue length and active read/write/scan handler count" Forget to add signoff This reverts commit 5ec218dbc22624406e0777d7987533e2429ca334. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/be042652 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/be042652 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/be042652 Branch: refs/heads/branch-1 Commit: be042652aa048c85a822463d5098f2d06cb993f1 Parents: 5ec218d Author: zhangduo Authored: Tue Nov 29 16:09:22 2016 +0800 Committer: zhangduo Committed: Tue Nov 29 16:09:22 2016 +0800 -- .../hbase/ipc/MetricsHBaseServerSource.java | 15 - .../hbase/ipc/MetricsHBaseServerWrapper.java| 19 -- .../hbase/ipc/MetricsHBaseServerSourceImpl.java | 14 +--- .../ipc/FastPathBalancedQueueRpcExecutor.java | 9 +-- .../hadoop/hbase/ipc/FifoRpcScheduler.java | 30 - .../ipc/MetricsHBaseServerWrapperImpl.java | 48 - .../hadoop/hbase/ipc/RWQueueRpcExecutor.java| 62 + .../apache/hadoop/hbase/ipc/RpcExecutor.java| 71 ++-- .../apache/hadoop/hbase/ipc/RpcScheduler.java | 18 - .../hadoop/hbase/ipc/SimpleRpcScheduler.java| 30 - .../hbase/ipc/DelegatingRpcScheduler.java | 32 + .../ipc/MetricsHBaseServerWrapperStub.java | 30 - .../apache/hadoop/hbase/ipc/TestRpcMetrics.java | 7 +- 13 files changed, 30 insertions(+), 355 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/be042652/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java -- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java index 8bee67e..ac14bd8 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java @@ -60,25 +60,10 @@ public interface MetricsHBaseServerSource extends BaseSource { String REPLICATION_QUEUE_DESC = "Number of calls in the replication call queue."; String PRIORITY_QUEUE_DESC = "Number of calls in the priority call queue."; - String WRITE_QUEUE_NAME = "numCallsInWriteQueue"; - String WRITE_QUEUE_DESC = "Number of calls in the write call queue; " + -"parsed requests waiting in scheduler to be executed"; - String READ_QUEUE_NAME = "numCallsInReadQueue"; - String READ_QUEUE_DESC = "Number of calls in the read call queue; " + -"parsed requests waiting in scheduler to be executed"; - String SCAN_QUEUE_NAME = "numCallsInScanQueue"; - String SCAN_QUEUE_DESC = "Number of calls in the scan call queue; " + -"parsed requests waiting in scheduler to be executed"; String NUM_OPEN_CONNECTIONS_NAME = "numOpenConnections"; String NUM_OPEN_CONNECTIONS_DESC = "Number of open connections."; String NUM_ACTIVE_HANDLER_NAME = "numActiveHandler"; String NUM_ACTIVE_HANDLER_DESC = "Number of active rpc handlers."; - String NUM_ACTIVE_WRITE_HANDLER_NAME = "numActiveWriteHandler"; - String NUM_ACTIVE_WRITE_HANDLER_DESC = "Number of active write rpc handlers."; - String NUM_ACTIVE_READ_HANDLER_NAME = "numActiveReadHandler"; - String NUM_ACTIVE_READ_HANDLER_DESC = "Number of active read rpc handlers."; - String NUM_ACTIVE_SCAN_HANDLER_NAME = "numActiveScanHandler"; - String NUM_ACTIVE_SCAN_HANDLER_DESC = "Number of active scan rpc handlers."; String NUM_GENERAL_CALLS_DROPPED_NAME = "numGeneralCallsDropped"; String NUM_GENERAL_CALLS_DROPPED_DESC = "Total number of calls in general queue which " + "were dropped by CoDel RPC executor"; http://git-wip-us.apache.org/repos/asf/hbase/blob/be042652/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapper.java -- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapper.java index b272cd0..8f30205 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapper.java @@ -21,30 +21,11 @@ package org.apache.hadoop.hbase.ipc; public interface MetricsHBaseServerWrapper { long getTotalQueueSize(); - int getGeneralQueueLength(); - int getReplicationQueue
[2/2] hbase git commit: HBASE-16561 Add metrics about read/write/scan queue length and active read/write/scan handler count
HBASE-16561 Add metrics about read/write/scan queue length and active read/write/scan handler count Signed-off-by: zhangduo Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7b2673db Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7b2673db Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7b2673db Branch: refs/heads/branch-1 Commit: 7b2673db1281279fef19007418759c65837f2021 Parents: be04265 Author: Guanghao Zhang Authored: Tue Nov 29 09:50:12 2016 +0800 Committer: zhangduo Committed: Tue Nov 29 16:09:56 2016 +0800 -- .../hbase/ipc/MetricsHBaseServerSource.java | 15 + .../hbase/ipc/MetricsHBaseServerWrapper.java| 19 ++ .../hbase/ipc/MetricsHBaseServerSourceImpl.java | 14 +++- .../ipc/FastPathBalancedQueueRpcExecutor.java | 9 ++- .../hadoop/hbase/ipc/FifoRpcScheduler.java | 30 + .../ipc/MetricsHBaseServerWrapperImpl.java | 48 + .../hadoop/hbase/ipc/RWQueueRpcExecutor.java| 62 - .../apache/hadoop/hbase/ipc/RpcExecutor.java| 71 ++-- .../apache/hadoop/hbase/ipc/RpcScheduler.java | 18 + .../hadoop/hbase/ipc/SimpleRpcScheduler.java| 30 + .../hbase/ipc/DelegatingRpcScheduler.java | 32 - .../ipc/MetricsHBaseServerWrapperStub.java | 30 + .../apache/hadoop/hbase/ipc/TestRpcMetrics.java | 7 +- 13 files changed, 355 insertions(+), 30 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/7b2673db/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java -- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java index ac14bd8..8bee67e 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java @@ -60,10 +60,25 @@ public interface MetricsHBaseServerSource extends BaseSource { String REPLICATION_QUEUE_DESC = "Number of calls in the replication call queue."; String PRIORITY_QUEUE_DESC = "Number of calls in the priority call queue."; + String WRITE_QUEUE_NAME = "numCallsInWriteQueue"; + String WRITE_QUEUE_DESC = "Number of calls in the write call queue; " + +"parsed requests waiting in scheduler to be executed"; + String READ_QUEUE_NAME = "numCallsInReadQueue"; + String READ_QUEUE_DESC = "Number of calls in the read call queue; " + +"parsed requests waiting in scheduler to be executed"; + String SCAN_QUEUE_NAME = "numCallsInScanQueue"; + String SCAN_QUEUE_DESC = "Number of calls in the scan call queue; " + +"parsed requests waiting in scheduler to be executed"; String NUM_OPEN_CONNECTIONS_NAME = "numOpenConnections"; String NUM_OPEN_CONNECTIONS_DESC = "Number of open connections."; String NUM_ACTIVE_HANDLER_NAME = "numActiveHandler"; String NUM_ACTIVE_HANDLER_DESC = "Number of active rpc handlers."; + String NUM_ACTIVE_WRITE_HANDLER_NAME = "numActiveWriteHandler"; + String NUM_ACTIVE_WRITE_HANDLER_DESC = "Number of active write rpc handlers."; + String NUM_ACTIVE_READ_HANDLER_NAME = "numActiveReadHandler"; + String NUM_ACTIVE_READ_HANDLER_DESC = "Number of active read rpc handlers."; + String NUM_ACTIVE_SCAN_HANDLER_NAME = "numActiveScanHandler"; + String NUM_ACTIVE_SCAN_HANDLER_DESC = "Number of active scan rpc handlers."; String NUM_GENERAL_CALLS_DROPPED_NAME = "numGeneralCallsDropped"; String NUM_GENERAL_CALLS_DROPPED_DESC = "Total number of calls in general queue which " + "were dropped by CoDel RPC executor"; http://git-wip-us.apache.org/repos/asf/hbase/blob/7b2673db/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapper.java -- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapper.java index 8f30205..b272cd0 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapper.java @@ -21,11 +21,30 @@ package org.apache.hadoop.hbase.ipc; public interface MetricsHBaseServerWrapper { long getTotalQueueSize(); + int getGeneralQueueLength(); + int getReplicationQueueLength(); + int getPriorityQueueLength(); + int getNumOpenConnections(); + int getActiveRpcHandlerCount(); + long getNumGeneralCallsDr
hbase git commit: HBASE-16302 age of last shipped op and age of last applied op should be histograms
Repository: hbase Updated Branches: refs/heads/master 346e904a2 -> 7bcbac91a HBASE-16302 age of last shipped op and age of last applied op should be histograms Signed-off-by: Ashish Singhi Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7bcbac91 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7bcbac91 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7bcbac91 Branch: refs/heads/master Commit: 7bcbac91a2385cd3009bcc277bb0f4d94084c926 Parents: 346e904 Author: Ashu Pachauri Authored: Tue Nov 29 13:51:32 2016 +0530 Committer: Ashish Singhi Committed: Tue Nov 29 13:51:32 2016 +0530 -- .../regionserver/MetricsReplicationGlobalSourceSource.java | 9 + .../regionserver/MetricsReplicationSinkSourceImpl.java | 9 + .../regionserver/MetricsReplicationSourceSourceImpl.java| 9 + .../org/apache/hadoop/metrics2/lib/MutableHistogram.java| 4 .../hbase/replication/regionserver/MetricsSource.java | 2 +- 5 files changed, 20 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/7bcbac91/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java -- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java index 0a67663..7a34e45 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java @@ -20,11 +20,12 @@ package org.apache.hadoop.hbase.replication.regionserver; import org.apache.hadoop.metrics2.lib.MutableFastCounter; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; +import org.apache.hadoop.metrics2.lib.MutableHistogram; public class MetricsReplicationGlobalSourceSource implements MetricsReplicationSourceSource{ private final MetricsReplicationSourceImpl rms; - private final MutableGaugeLong ageOfLastShippedOpGauge; + private final MutableHistogram ageOfLastShippedOpHist; private final MutableGaugeLong sizeOfLogQueueGauge; private final MutableFastCounter logReadInEditsCounter; private final MutableFastCounter logEditsFilteredCounter; @@ -47,7 +48,7 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS public MetricsReplicationGlobalSourceSource(MetricsReplicationSourceImpl rms) { this.rms = rms; -ageOfLastShippedOpGauge = rms.getMetricsRegistry().getGauge(SOURCE_AGE_OF_LAST_SHIPPED_OP, 0L); +ageOfLastShippedOpHist = rms.getMetricsRegistry().getHistogram(SOURCE_AGE_OF_LAST_SHIPPED_OP); sizeOfLogQueueGauge = rms.getMetricsRegistry().getGauge(SOURCE_SIZE_OF_LOG_QUEUE, 0L); @@ -80,7 +81,7 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS } @Override public void setLastShippedAge(long age) { -ageOfLastShippedOpGauge.set(age); +ageOfLastShippedOpHist.add(age); } @Override public void incrSizeOfLogQueue(int size) { @@ -137,7 +138,7 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS @Override public long getLastShippedAge() { -return ageOfLastShippedOpGauge.value(); +return ageOfLastShippedOpHist.getMax(); } @Override public void incrHFilesShipped(long hfiles) { http://git-wip-us.apache.org/repos/asf/hbase/blob/7bcbac91/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java -- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java index 540212a..74592d9 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java @@ -20,23 +20,24 @@ package org.apache.hadoop.hbase.replication.regionserver; import org.apache.hadoop.metrics2.lib.MutableFastCounter; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; +import org.apache.hadoop.metrics2.lib.MutableHistogram; public class MetricsReplicationSinkSourceImpl implements Metric
hbase git commit: HBASE-16302 age of last shipped op and age of last applied op should be histograms
Repository: hbase Updated Branches: refs/heads/branch-1 7b2673db1 -> b8da9f83c HBASE-16302 age of last shipped op and age of last applied op should be histograms Signed-off-by: Ashish Singhi Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b8da9f83 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b8da9f83 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b8da9f83 Branch: refs/heads/branch-1 Commit: b8da9f83cbbaf8a1257e5abb1ac438b21ba5507e Parents: 7b2673d Author: Ashu Pachauri Authored: Tue Nov 29 13:54:28 2016 +0530 Committer: Ashish Singhi Committed: Tue Nov 29 13:54:28 2016 +0530 -- .../regionserver/MetricsReplicationGlobalSourceSource.java | 9 + .../regionserver/MetricsReplicationSinkSourceImpl.java | 9 + .../regionserver/MetricsReplicationSourceSourceImpl.java| 9 + .../org/apache/hadoop/metrics2/lib/MutableHistogram.java| 4 .../hbase/replication/regionserver/MetricsSource.java | 2 +- 5 files changed, 20 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/b8da9f83/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java -- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java index 0a67663..7a34e45 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java @@ -20,11 +20,12 @@ package org.apache.hadoop.hbase.replication.regionserver; import org.apache.hadoop.metrics2.lib.MutableFastCounter; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; +import org.apache.hadoop.metrics2.lib.MutableHistogram; public class MetricsReplicationGlobalSourceSource implements MetricsReplicationSourceSource{ private final MetricsReplicationSourceImpl rms; - private final MutableGaugeLong ageOfLastShippedOpGauge; + private final MutableHistogram ageOfLastShippedOpHist; private final MutableGaugeLong sizeOfLogQueueGauge; private final MutableFastCounter logReadInEditsCounter; private final MutableFastCounter logEditsFilteredCounter; @@ -47,7 +48,7 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS public MetricsReplicationGlobalSourceSource(MetricsReplicationSourceImpl rms) { this.rms = rms; -ageOfLastShippedOpGauge = rms.getMetricsRegistry().getGauge(SOURCE_AGE_OF_LAST_SHIPPED_OP, 0L); +ageOfLastShippedOpHist = rms.getMetricsRegistry().getHistogram(SOURCE_AGE_OF_LAST_SHIPPED_OP); sizeOfLogQueueGauge = rms.getMetricsRegistry().getGauge(SOURCE_SIZE_OF_LOG_QUEUE, 0L); @@ -80,7 +81,7 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS } @Override public void setLastShippedAge(long age) { -ageOfLastShippedOpGauge.set(age); +ageOfLastShippedOpHist.add(age); } @Override public void incrSizeOfLogQueue(int size) { @@ -137,7 +138,7 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS @Override public long getLastShippedAge() { -return ageOfLastShippedOpGauge.value(); +return ageOfLastShippedOpHist.getMax(); } @Override public void incrHFilesShipped(long hfiles) { http://git-wip-us.apache.org/repos/asf/hbase/blob/b8da9f83/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java -- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java index 540212a..74592d9 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java @@ -20,23 +20,24 @@ package org.apache.hadoop.hbase.replication.regionserver; import org.apache.hadoop.metrics2.lib.MutableFastCounter; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; +import org.apache.hadoop.metrics2.lib.MutableHistogram; public class MetricsReplicationSinkSourceImpl implements Me
hbase git commit: HBASE-17192 remove use of scala-tools.org as repo.
Repository: hbase Updated Branches: refs/heads/master 7bcbac91a -> e5dad24a9 HBASE-17192 remove use of scala-tools.org as repo. Signed-off-by: zhangduo Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e5dad24a Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e5dad24a Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e5dad24a Branch: refs/heads/master Commit: e5dad24a9cb35d831a0ee1cf0eeb14b3719ab7ef Parents: 7bcbac9 Author: Sean Busbey Authored: Tue Nov 29 04:03:30 2016 -0600 Committer: Sean Busbey Committed: Tue Nov 29 08:50:12 2016 -0600 -- pom.xml | 5 - 1 file changed, 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/e5dad24a/pom.xml -- diff --git a/pom.xml b/pom.xml index e5a23e0..efe7b72 100644 --- a/pom.xml +++ b/pom.xml @@ -89,11 +89,6 @@ e.g. surefire 2.18-SNAPSHOT--> - scala-tools.org - Scala-tools Maven2 Repository - http://scala-tools.org/repo-releases - - apache.snapshots http://repository.apache.org/snapshots/
hbase git commit: HBASE-17012 Handle Offheap cells in CompressedKvEncoder (Ram)
Repository: hbase Updated Branches: refs/heads/master e5dad24a9 -> 7c43a23c0 HBASE-17012 Handle Offheap cells in CompressedKvEncoder (Ram) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7c43a23c Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7c43a23c Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7c43a23c Branch: refs/heads/master Commit: 7c43a23c07d2af1c236b3153ba932234c3a80d13 Parents: e5dad24 Author: Ramkrishna Authored: Tue Nov 29 22:27:45 2016 +0530 Committer: Ramkrishna Committed: Tue Nov 29 22:27:45 2016 +0530 -- .../java/org/apache/hadoop/hbase/CellUtil.java | 58 .../hadoop/hbase/io/TagCompressionContext.java | 32 +-- .../apache/hadoop/hbase/io/util/Dictionary.java | 51 + .../regionserver/wal/SecureWALCellCodec.java| 35 +++- .../hbase/regionserver/wal/WALCellCodec.java| 38 +++-- .../apache/hadoop/hbase/ipc/TestRpcServer.java | 1 - .../wal/TestWALCellCodecWithCompression.java| 40 -- .../hbase/wal/TestWALReaderOnSecureWAL.java | 31 +-- 8 files changed, 189 insertions(+), 97 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/7c43a23c/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java -- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java index d47cdab..86c7720 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience.Private; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.TagCompressionContext; +import org.apache.hadoop.hbase.io.util.Dictionary; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.ByteRange; import org.apache.hadoop.hbase.util.Bytes; @@ -1578,12 +1579,12 @@ public final class CellUtil { /** * Writes the row from the given cell to the output stream - * @param out The dataoutputstream to which the data has to be written + * @param out The outputstream to which the data has to be written * @param cell The cell whose contents has to be written * @param rlength the row length * @throws IOException */ - public static void writeRow(DataOutputStream out, Cell cell, short rlength) throws IOException { + public static void writeRow(OutputStream out, Cell cell, short rlength) throws IOException { if (cell instanceof ByteBufferCell) { ByteBufferUtils.copyBufferToStream(out, ((ByteBufferCell) cell).getRowByteBuffer(), ((ByteBufferCell) cell).getRowPosition(), rlength); @@ -1611,12 +1612,12 @@ public final class CellUtil { /** * Writes the family from the given cell to the output stream - * @param out The dataoutputstream to which the data has to be written + * @param out The outputstream to which the data has to be written * @param cell The cell whose contents has to be written * @param flength the family length * @throws IOException */ - public static void writeFamily(DataOutputStream out, Cell cell, byte flength) throws IOException { + public static void writeFamily(OutputStream out, Cell cell, byte flength) throws IOException { if (cell instanceof ByteBufferCell) { ByteBufferUtils.copyBufferToStream(out, ((ByteBufferCell) cell).getFamilyByteBuffer(), ((ByteBufferCell) cell).getFamilyPosition(), flength); @@ -1627,12 +1628,12 @@ public final class CellUtil { /** * Writes the qualifier from the given cell to the output stream - * @param out The dataoutputstream to which the data has to be written + * @param out The outputstream to which the data has to be written * @param cell The cell whose contents has to be written * @param qlength the qualifier length * @throws IOException */ - public static void writeQualifier(DataOutputStream out, Cell cell, int qlength) + public static void writeQualifier(OutputStream out, Cell cell, int qlength) throws IOException { if (cell instanceof ByteBufferCell) { ByteBufferUtils.copyBufferToStream(out, ((ByteBufferCell) cell).getQualifierByteBuffer(), @@ -1662,12 +1663,12 @@ public final class CellUtil { /** * Writes the value from the given cell to the output stream - * @param out The dataoutputstream to which the data has to be written + * @param out The outputstream to which the data has to be written * @param cell The cell whose contents has to be wr
[1/2] hbase git commit: HBASE-17167 Pass mvcc to client when scan
Repository: hbase Updated Branches: refs/heads/master 7c43a23c0 -> 890fcbd0e http://git-wip-us.apache.org/repos/asf/hbase/blob/890fcbd0/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java -- diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java index dc050e8..c35617b 100644 --- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java +++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java @@ -14219,6 +14219,16 @@ public final class ClientProtos { */ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ColumnFamilyTimeRangeOrBuilder getCfTimeRangeOrBuilder( int index); + +// optional uint64 mvcc_read_point = 20 [default = 0]; +/** + * optional uint64 mvcc_read_point = 20 [default = 0]; + */ +boolean hasMvccReadPoint(); +/** + * optional uint64 mvcc_read_point = 20 [default = 0]; + */ +long getMvccReadPoint(); } /** * Protobuf type {@code hbase.pb.Scan} @@ -14408,6 +14418,11 @@ public final class ClientProtos { cfTimeRange_.add(input.readMessage(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ColumnFamilyTimeRange.PARSER, extensionRegistry)); break; } +case 160: { + bitField0_ |= 0x0001; + mvccReadPoint_ = input.readUInt64(); + break; +} } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -14841,6 +14856,22 @@ public final class ClientProtos { return cfTimeRange_.get(index); } +// optional uint64 mvcc_read_point = 20 [default = 0]; +public static final int MVCC_READ_POINT_FIELD_NUMBER = 20; +private long mvccReadPoint_; +/** + * optional uint64 mvcc_read_point = 20 [default = 0]; + */ +public boolean hasMvccReadPoint() { + return ((bitField0_ & 0x0001) == 0x0001); +} +/** + * optional uint64 mvcc_read_point = 20 [default = 0]; + */ +public long getMvccReadPoint() { + return mvccReadPoint_; +} + private void initFields() { column_ = java.util.Collections.emptyList(); attribute_ = java.util.Collections.emptyList(); @@ -14861,6 +14892,7 @@ public final class ClientProtos { caching_ = 0; allowPartialResults_ = false; cfTimeRange_ = java.util.Collections.emptyList(); + mvccReadPoint_ = 0L; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -14955,6 +14987,9 @@ public final class ClientProtos { for (int i = 0; i < cfTimeRange_.size(); i++) { output.writeMessage(19, cfTimeRange_.get(i)); } + if (((bitField0_ & 0x0001) == 0x0001)) { +output.writeUInt64(20, mvccReadPoint_); + } getUnknownFields().writeTo(output); } @@ -15040,6 +15075,10 @@ public final class ClientProtos { size += com.google.protobuf.CodedOutputStream .computeMessageSize(19, cfTimeRange_.get(i)); } + if (((bitField0_ & 0x0001) == 0x0001)) { +size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(20, mvccReadPoint_); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -15149,6 +15188,11 @@ public final class ClientProtos { } result = result && getCfTimeRangeList() .equals(other.getCfTimeRangeList()); + result = result && (hasMvccReadPoint() == other.hasMvccReadPoint()); + if (hasMvccReadPoint()) { +result = result && (getMvccReadPoint() +== other.getMvccReadPoint()); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -15238,6 +15282,10 @@ public final class ClientProtos { hash = (37 * hash) + CF_TIME_RANGE_FIELD_NUMBER; hash = (53 * hash) + getCfTimeRangeList().hashCode(); } + if (hasMvccReadPoint()) { +hash = (37 * hash) + MVCC_READ_POINT_FIELD_NUMBER; +hash = (53 * hash) + hashLong(getMvccReadPoint()); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -15421,6 +15469,8 @@ public final class ClientProtos { } else { cfTimeRangeBuilder_.clear(); } +mvccReadPoint_ = 0L; +bitField0_ = (bitField0_ & ~0x0008); return this; } @@ -15548,6 +15598,10 @@ public final class ClientProtos { } else { result.cfTimeRange_ = cfTimeRangeBuilder_.build(); } +if (((from_bitField0_ & 0x0008)
[2/2] hbase git commit: HBASE-17167 Pass mvcc to client when scan
HBASE-17167 Pass mvcc to client when scan Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/890fcbd0 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/890fcbd0 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/890fcbd0 Branch: refs/heads/master Commit: 890fcbd0e6f916cc94b45b881b0cc060cc1e835c Parents: 7c43a23 Author: zhangduo Authored: Tue Nov 29 17:13:49 2016 +0800 Committer: zhangduo Committed: Wed Nov 30 10:11:04 2016 +0800 -- .../hadoop/hbase/client/ClientScanner.java | 406 +- .../org/apache/hadoop/hbase/client/HTable.java | 7 +- .../client/PackagePrivateFieldAccessor.java | 41 ++ .../org/apache/hadoop/hbase/client/Scan.java| 55 ++- .../hadoop/hbase/client/ScannerCallable.java| 3 + .../hadoop/hbase/protobuf/ProtobufUtil.java | 8 + .../hbase/shaded/protobuf/ProtobufUtil.java | 8 + .../shaded/protobuf/generated/ClientProtos.java | 412 +- .../src/main/protobuf/Client.proto | 12 +- .../hbase/protobuf/generated/ClientProtos.java | 416 ++- hbase-protocol/src/main/protobuf/Client.proto | 12 +- .../hadoop/hbase/regionserver/HRegion.java | 9 +- .../hbase/regionserver/RSRpcServices.java | 4 +- .../hbase/TestPartialResultsFromClientSide.java | 13 +- .../hbase/client/TestMvccConsistentScanner.java | 134 ++ .../hadoop/hbase/regionserver/TestTags.java | 14 +- .../regionserver/TestReplicationSink.java | 22 +- 17 files changed, 1120 insertions(+), 456 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/890fcbd0/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java index 20ed183..c4c86a6 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java @@ -120,198 +120,192 @@ public abstract class ClientScanner extends AbstractClientScanner { ClusterConnection connection, RpcRetryingCallerFactory rpcFactory, RpcControllerFactory controllerFactory, ExecutorService pool, int primaryOperationTimeout) throws IOException { - if (LOG.isTraceEnabled()) { -LOG.trace("Scan table=" + tableName -+ ", startRow=" + Bytes.toStringBinary(scan.getStartRow())); - } - this.scan = scan; - this.tableName = tableName; - this.lastNext = System.currentTimeMillis(); - this.connection = connection; - this.pool = pool; - this.primaryOperationTimeout = primaryOperationTimeout; - this.retries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, - HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); - if (scan.getMaxResultSize() > 0) { -this.maxScannerResultSize = scan.getMaxResultSize(); - } else { -this.maxScannerResultSize = conf.getLong( - HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, - HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE); - } - this.scannerTimeout = HBaseConfiguration.getInt(conf, -HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, -HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, -HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD); - - // check if application wants to collect scan metrics - initScanMetrics(scan); - - // Use the caching from the Scan. If not set, use the default cache setting for this table. - if (this.scan.getCaching() > 0) { -this.caching = this.scan.getCaching(); - } else { -this.caching = conf.getInt( -HConstants.HBASE_CLIENT_SCANNER_CACHING, -HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING); - } - - this.caller = rpcFactory. newCaller(); - this.rpcControllerFactory = controllerFactory; - - this.conf = conf; - initCache(); - initializeScannerInConstruction(); -} +if (LOG.isTraceEnabled()) { + LOG.trace( +"Scan table=" + tableName + ", startRow=" + Bytes.toStringBinary(scan.getStartRow())); +} +this.scan = scan; +this.tableName = tableName; +this.lastNext = System.currentTimeMillis(); +this.connection = connection; +this.pool = pool; +this.primaryOperationTimeout = primaryOperationTimeout; +this.retries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); +if (scan.getMaxResultSize() > 0) { + this.maxScannerResultSize = scan.getMaxResultSize(); +} else { +
hbase git commit: HBASE-17167 Pass mvcc to client when scan
Repository: hbase Updated Branches: refs/heads/branch-1 b8da9f83c -> af6978312 HBASE-17167 Pass mvcc to client when scan Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/af697831 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/af697831 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/af697831 Branch: refs/heads/branch-1 Commit: af69783128ed8bd8bf321f378adcb50d46a4f2fc Parents: b8da9f8 Author: zhangduo Authored: Tue Nov 29 20:35:34 2016 +0800 Committer: zhangduo Committed: Wed Nov 30 10:02:17 2016 +0800 -- .../hadoop/hbase/client/ClientScanner.java | 507 +-- .../org/apache/hadoop/hbase/client/HTable.java | 24 +- .../client/PackagePrivateFieldAccessor.java | 41 ++ .../org/apache/hadoop/hbase/client/Scan.java| 30 ++ .../hadoop/hbase/client/ScannerCallable.java| 3 + .../hadoop/hbase/protobuf/ProtobufUtil.java | 8 + .../hbase/protobuf/generated/ClientProtos.java | 414 +++ hbase-protocol/src/main/protobuf/Client.proto | 12 +- .../hadoop/hbase/regionserver/HRegion.java | 9 +- .../hbase/regionserver/RSRpcServices.java | 1 + .../hbase/TestPartialResultsFromClientSide.java | 13 +- .../hbase/client/TestMvccConsistentScanner.java | 139 + .../hadoop/hbase/regionserver/TestTags.java | 14 +- .../regionserver/TestReplicationSink.java | 22 +- 14 files changed, 839 insertions(+), 398 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/af697831/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java index 944f44e..0898385 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java @@ -57,54 +57,54 @@ import com.google.common.annotations.VisibleForTesting; */ @InterfaceAudience.Private public class ClientScanner extends AbstractClientScanner { -private static final Log LOG = LogFactory.getLog(ClientScanner.class); -// A byte array in which all elements are the max byte, and it is used to -// construct closest front row -static byte[] MAX_BYTE_ARRAY = Bytes.createMaxByteArray(9); -protected Scan scan; -protected boolean closed = false; -// Current region scanner is against. Gets cleared if current region goes -// wonky: e.g. if it splits on us. -protected HRegionInfo currentRegion = null; -protected ScannerCallableWithReplicas callable = null; -protected final LinkedList cache = new LinkedList(); -/** - * A list of partial results that have been returned from the server. This list should only - * contain results if this scanner does not have enough partial results to form the complete - * result. - */ -protected final LinkedList partialResults = new LinkedList(); -/** - * The row for which we are accumulating partial Results (i.e. the row of the Results stored - * inside partialResults). Changes to partialResultsRow and partialResults are kept in sync - * via the methods {@link #addToPartialResults(Result)} and {@link #clearPartialResults()} - */ -protected byte[] partialResultsRow = null; -/** - * The last cell from a not full Row which is added to cache - */ -protected Cell lastCellLoadedToCache = null; -protected final int caching; -protected long lastNext; -// Keep lastResult returned successfully in case we have to reset scanner. -protected Result lastResult = null; -protected final long maxScannerResultSize; -private final ClusterConnection connection; -private final TableName tableName; -protected final int scannerTimeout; -protected boolean scanMetricsPublished = false; -protected RpcRetryingCaller caller; -protected RpcControllerFactory rpcControllerFactory; -protected Configuration conf; -//The timeout on the primary. Applicable if there are multiple replicas for a region -//In that case, we will only wait for this much timeout on the primary before going -//to the replicas and trying the same scan. Note that the retries will still happen -//on each replica and the first successful results will be taken. A timeout of 0 is -//disallowed. -protected final int primaryOperationTimeout; -private int retries; -protected final ExecutorService pool; -private static MetaComparator metaComparator = new MetaComparator(); + private static final Log LOG = LogFactory.getLog(ClientScanner.class); + // A byte
[7/8] hbase git commit: HBASE-16904 Snapshot related changes for FS redo work
HBASE-16904 Snapshot related changes for FS redo work Signed-off-by: Sean Busbey Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/159a67c6 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/159a67c6 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/159a67c6 Branch: refs/heads/hbase-14439 Commit: 159a67c6767585ed9fb7ee357fb579ce25c30a47 Parents: 6d1813a Author: Umesh Agashe Authored: Tue Nov 1 10:32:01 2016 -0700 Committer: Sean Busbey Committed: Tue Nov 29 01:27:05 2016 -0600 -- .../hbase/client/ClientSideRegionScanner.java |8 +- .../hbase/client/TableSnapshotScanner.java | 51 +- .../apache/hadoop/hbase/fs/MasterStorage.java | 286 - .../apache/hadoop/hbase/fs/StorageContext.java |1 - .../hadoop/hbase/fs/legacy/LegacyLayout.java| 131 ++- .../hbase/fs/legacy/LegacyMasterStorage.java| 431 ++- .../fs/legacy/snapshot/ExportSnapshot.java | 1102 ++ .../legacy/snapshot/RestoreSnapshotHelper.java | 689 +++ .../fs/legacy/snapshot/SnapshotFileCache.java | 11 +- .../legacy/snapshot/SnapshotHFileCleaner.java |3 +- .../fs/legacy/snapshot/SnapshotManifest.java| 570 + .../fs/legacy/snapshot/SnapshotManifestV1.java | 209 .../fs/legacy/snapshot/SnapshotManifestV2.java | 187 +++ .../apache/hadoop/hbase/mapreduce/Driver.java |2 +- .../MultiTableSnapshotInputFormatImpl.java | 43 +- .../mapreduce/TableSnapshotInputFormat.java |2 +- .../mapreduce/TableSnapshotInputFormatImpl.java | 47 +- .../procedure/CloneSnapshotProcedure.java | 32 +- .../procedure/RestoreSnapshotProcedure.java | 24 +- .../snapshot/DisabledTableSnapshotHandler.java | 16 +- .../master/snapshot/MasterSnapshotVerifier.java | 137 +-- .../hbase/master/snapshot/SnapshotManager.java | 255 +--- .../master/snapshot/TakeSnapshotHandler.java| 80 +- .../hadoop/hbase/regionserver/HRegion.java | 18 +- .../regionserver/DumpReplicationQueues.java |3 - .../hadoop/hbase/snapshot/ExportSnapshot.java | 1084 - .../hbase/snapshot/RestoreSnapshotHelper.java | 823 - .../snapshot/SnapshotDescriptionUtils.java | 239 +--- .../hadoop/hbase/snapshot/SnapshotInfo.java | 187 ++- .../hadoop/hbase/snapshot/SnapshotManifest.java | 570 - .../hbase/snapshot/SnapshotManifestV1.java | 209 .../hbase/snapshot/SnapshotManifestV2.java | 187 --- .../hbase/snapshot/SnapshotReferenceUtil.java | 327 ++ .../snapshot/SnapshotRestoreMetaChanges.java| 157 +++ .../hbase/client/TestSnapshotFromClient.java|7 - .../fs/legacy/snapshot/TestExportSnapshot.java | 384 ++ .../snapshot/TestExportSnapshotHelpers.java | 96 ++ .../snapshot/TestRestoreSnapshotHelper.java | 181 +++ .../legacy/snapshot/TestSnapshotFileCache.java | 30 +- .../legacy/snapshot/TestSnapshotManifest.java | 146 +++ .../hbase/snapshot/SnapshotTestingUtils.java|3 + .../hbase/snapshot/TestExportSnapshot.java | 376 -- .../snapshot/TestExportSnapshotHelpers.java | 96 -- .../snapshot/TestRestoreSnapshotHelper.java | 180 --- .../hbase/snapshot/TestSnapshotManifest.java| 145 --- src/main/asciidoc/_chapters/ops_mgt.adoc| 10 +- 46 files changed, 5006 insertions(+), 4769 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java index a643428..a7ea192 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java @@ -24,8 +24,6 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; @@ -33,6 +31,8 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.metrics.ScanMetrics; +import org.apache.hadoop.hbase.fs.MasterStorage; +import org.apache.hadoop.hbase.fs.StorageIdentifier; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.RegionScanner;
[5/8] hbase git commit: HBASE-16904 Snapshot related changes for FS redo work
http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/snapshot/SnapshotManifest.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/snapshot/SnapshotManifest.java new file mode 100644 index 000..58f7bf1 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/snapshot/SnapshotManifest.java @@ -0,0 +1,570 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.fs.legacy.snapshot; + +import com.google.protobuf.CodedInputStream; +import com.google.protobuf.InvalidProtocolBufferException; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.fs.RegionStorage; +import org.apache.hadoop.hbase.fs.legacy.LegacyLayout; +import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare; +import org.apache.hadoop.hbase.mob.MobUtils; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotDataManifest; +import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.Threads; + +/** + * Utility class to help read/write the Snapshot Manifest. + * + * The snapshot format is transparent for the users of this class, + * once the snapshot is written, it will never be modified. + * On open() the snapshot will be loaded to the current in-memory format. + */ +@InterfaceAudience.Private +public final class SnapshotManifest { + private static final Log LOG = LogFactory.getLog(SnapshotManifest.class); + + public static final String SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY = "snapshot.manifest.size.limit"; + + public static final String DATA_MANIFEST_NAME = "data.manifest"; + + private List regionManifests; + private SnapshotDescription desc; + private HTableDescriptor htd; + + private final ForeignExceptionSnare monitor; + private final Configuration conf; + private final Path workingDir; + private final FileSystem fs; + private int manifestSizeLimit; + + private SnapshotManifest(final Configuration conf, final FileSystem fs, + final Path workingDir, final SnapshotDescription desc, + final ForeignExceptionSnare monitor) { +this.monitor = monitor; +this.desc = desc; +this.workingDir = workingDir; +this.conf = conf; +this.fs = fs; + +this.manifestSizeLimit = conf.getInt(SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 64 * 1024 * 1024); + } + + /** + * Return a SnapshotManifest instance, used for writing a snapshot. + * + * There are two usage pattern: + * - The Master will cr
[1/8] hbase git commit: HBASE-16904 Snapshot related changes for FS redo work
Repository: hbase Updated Branches: refs/heads/hbase-14439 6d1813a2f -> 815223453 http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestExportSnapshotHelpers.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestExportSnapshotHelpers.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestExportSnapshotHelpers.java new file mode 100644 index 000..d70e8c0 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestExportSnapshotHelpers.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.fs.legacy.snapshot; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test Export Snapshot Tool helpers + */ +@Category({RegionServerTests.class, SmallTests.class}) +public class TestExportSnapshotHelpers { + private static final Log LOG = LogFactory.getLog(TestExportSnapshotHelpers.class); + + /** + * Verfy the result of getBalanceSplits() method. + * The result are groups of files, used as input list for the "export" mappers. + * All the groups should have similar amount of data. + * + * The input list is a pair of file path and length. + * The getBalanceSplits() function sort it by length, + * and assign to each group a file, going back and forth through the groups. + */ + @Test + public void testBalanceSplit() throws Exception { +// Create a list of files +List> files = new ArrayList>(); +for (long i = 0; i <= 20; i++) { + SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder() +.setType(SnapshotFileInfo.Type.HFILE) +.setHfile("file-" + i) +.build(); + files.add(new Pair(fileInfo, i)); +} + +// Create 5 groups (total size 210) +//group 0: 20, 11, 10, 1 (total size: 42) +//group 1: 19, 12, 9, 2 (total size: 42) +//group 2: 18, 13, 8, 3 (total size: 42) +//group 3: 17, 12, 7, 4 (total size: 42) +//group 4: 16, 11, 6, 5 (total size: 42) +List>> splits = ExportSnapshot.getBalancedSplits(files, 5); +assertEquals(5, splits.size()); + +String[] split0 = new String[] {"file-20", "file-11", "file-10", "file-1", "file-0"}; +verifyBalanceSplit(splits.get(0), split0, 42); +String[] split1 = new String[] {"file-19", "file-12", "file-9", "file-2"}; +verifyBalanceSplit(splits.get(1), split1, 42); +String[] split2 = new String[] {"file-18", "file-13", "file-8", "file-3"}; +verifyBalanceSplit(splits.get(2), split2, 42); +String[] split3 = new String[] {"file-17", "file-14", "file-7", "file-4"}; +verifyBalanceSplit(splits.get(3), split3, 42); +String[] split4 = new String[] {"file-16", "file-15", "file-6", "file-5"}; +verifyBalanceSplit(splits.get(4), split4, 42); + } + + private void verifyBalanceSplit(final List> split, + final String[] expected, final long expectedSize) { +assertEquals(expected.length, split.size()); +long totalSize = 0; +for (int i = 0; i < expected.length; ++i) { + Pair fileInfo = split.get(i); + assertEquals(expected[i], fileInfo.getFirst().getHfile()); + totalSize += fileInfo.getSecond(); +} +assertEquals(expectedSize, totalSize); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestRestoreSnapshotHelper.java -- diff --git a/hbase-server/src/test/ja
[6/8] hbase git commit: HBASE-16904 Snapshot related changes for FS redo work
http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/snapshot/ExportSnapshot.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/snapshot/ExportSnapshot.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/snapshot/ExportSnapshot.java new file mode 100644 index 000..ab90aa7 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/snapshot/ExportSnapshot.java @@ -0,0 +1,1102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.fs.legacy.snapshot; + +import java.io.BufferedInputStream; +import java.io.FileNotFoundException; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedList; +import java.util.List; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileChecksum; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.fs.MasterStorage; +import org.apache.hadoop.hbase.fs.StorageContext; +import org.apache.hadoop.hbase.fs.legacy.LegacyLayout; +import org.apache.hadoop.hbase.fs.legacy.LegacyMasterStorage; +import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier; +import org.apache.hadoop.hbase.fs.legacy.io.FileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; +import org.apache.hadoop.hbase.io.WALLink; +import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; +import org.apache.hadoop.hbase.mob.MobUtils; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo; +import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; +import org.apache.hadoop.hbase.snapshot.ExportSnapshotException; +import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.HFileArchiveUtil; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; +import org.apache.hadoop.mapreduce.security.TokenCache; +import org.apache.hadoop.hbase.io.hadoopbackport.ThrottledInputStream; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +/** + * Export the specified snapshot to a given FileSystem. + * + * The .snapshot/name folder is copied to the destination cluster + * and then all the hfiles/wals are copied using a Map-Reduce Job in the .archive/ location. + * When everything is done, the second cluster can restore the snapshot. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class ExportSnapshot extends Configured implemen
[3/8] hbase git commit: HBASE-16904 Snapshot related changes for FS redo work
http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java deleted file mode 100644 index 8c24b1e..000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java +++ /dev/null @@ -1,823 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.snapshot; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.ThreadPoolExecutor; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.HFileArchiver; -import org.apache.hadoop.hbase.MetaTableAccessor; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; -import org.apache.hadoop.hbase.fs.RegionStorage; -import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier; -import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; -import org.apache.hadoop.hbase.io.Reference; -import org.apache.hadoop.hbase.mob.MobUtils; -import org.apache.hadoop.hbase.monitoring.MonitoredTask; -import org.apache.hadoop.hbase.monitoring.TaskMonitor; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; -import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.StoreFileInfo; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.ModifyRegionUtils; -import org.apache.hadoop.hbase.util.Pair; -import org.apache.hadoop.io.IOUtils; - -/** - * Helper to Restore/Clone a Snapshot - * - * The helper assumes that a table is already created, and by calling restore() - * the content present in the snapshot will be restored as the new content of the table. - * - * Clone from Snapshot: If the target table is empty, the restore operation - * is just a "clone operation", where the only operations are: - * - * for each region in the snapshot create a new region - *(note that the region will have a different name, since the encoding contains the table name) - * for each file in the region create a new HFileLink to point to the original file. - * restore the logs, if any - * - * - * Restore from Snapshot: - * - * for each region in the table verify which are available in the snapshot and which are not - * - *if the region is not present in the snapshot, remove it. - *if the region is present in the snapshot - * - * for each file in the table region verify which are available in the snapshot - * - * if the hfile is not present in the snapshot, remove it - * if the hfile is present, keep it (nothing to do) - * - * for each file in the snapshot region but not in the table - * - * create a new HFileLink that point to the original file - * - * - * - * for each region in the snapshot not present in the current table state - * - *create a new region and for each file in the region create a new HFileLink - * (This is the same as the clone operation) - * -
[8/8] hbase git commit: HBASE-16904 ADDENDUM cleanup of snapshot related changes for fs redo.
HBASE-16904 ADDENDUM cleanup of snapshot related changes for fs redo. * missing header on SnapshotRestoreMetaChanges * neuter parts of ScanPerformanceEvaluation that need to be updated * fixup test class packages on Test*RestoreSnapshotHelper * fixup test class packages on TestMob*ExportSnapshot * fixup new javadoc warnings * fixup new findbugs warning * favor commenting out and placeholders that result in test failures over failures for test-compile Signed-off-by: Umesh Agashe Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/81522345 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/81522345 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/81522345 Branch: refs/heads/hbase-14439 Commit: 815223453954d8a36f522f42b89ebf800c1964e5 Parents: 159a67c Author: Sean Busbey Authored: Mon Nov 28 15:41:59 2016 -0600 Committer: Sean Busbey Committed: Wed Nov 30 00:54:57 2016 -0600 -- .../apache/hadoop/hbase/fs/MasterStorage.java | 3 - .../procedure/CloneSnapshotProcedure.java | 1 - .../snapshot/SnapshotRestoreMetaChanges.java| 18 +++ .../hadoop/hbase/ScanPerformanceEvaluation.java | 14 ++- .../hbase/client/TestTableSnapshotScanner.java | 11 +- .../legacy/cleaner/TestSnapshotFromMaster.java | 9 +- .../snapshot/TestExportSnapshotNoCluster.java | 115 +++ .../legacy/snapshot/TestMobExportSnapshot.java | 66 +++ .../snapshot/TestMobRestoreSnapshotHelper.java | 46 .../snapshot/TestMobSecureExportSnapshot.java | 53 + .../snapshot/TestRestoreSnapshotHelper.java | 8 +- .../snapshot/TestSecureExportSnapshot.java | 58 ++ .../snapshot/TestSnapshotHFileCleaner.java | 9 +- .../TestMultiTableSnapshotInputFormatImpl.java | 12 +- .../master/snapshot/TestSnapshotManager.java| 5 +- .../hbase/snapshot/SnapshotTestingUtils.java| 74 ++-- .../snapshot/TestExportSnapshotNoCluster.java | 115 --- .../hbase/snapshot/TestMobExportSnapshot.java | 65 --- .../snapshot/TestMobRestoreSnapshotHelper.java | 46 .../snapshot/TestMobSecureExportSnapshot.java | 53 - .../snapshot/TestSecureExportSnapshot.java | 58 -- .../snapshot/TestSnapshotDescriptionUtils.java | 3 +- 22 files changed, 438 insertions(+), 404 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java index a62cbb7..9f16018 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java @@ -370,7 +370,6 @@ public abstract class MasterStorage { * Returns all {@link HRegionInfo} for a snapshot * * @param snapshot - * @return * @throws IOException */ public Map getSnapshotRegions(final SnapshotDescription snapshot) @@ -482,7 +481,6 @@ public abstract class MasterStorage { * @param snapshot * @param src Source {@link StorageContext} * @param dest Destination {@link StorageContext} - * @return * @throws IOException */ public abstract boolean changeSnapshotContext(SnapshotDescription snapshot, StorageContext src, @@ -518,7 +516,6 @@ public abstract class MasterStorage { * @param destHtd * @param monitor * @param status - * @return * @throws IOException */ public SnapshotRestoreMetaChanges restoreSnapshot(final SnapshotDescription snapshot, http://git-wip-us.apache.org/repos/asf/hbase/blob/81522345/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java index cbdc02f..570fb72 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java @@ -345,7 +345,6 @@ public class CloneSnapshotProcedure final TableName tableName, final List newRegions) throws IOException { -final Configuration conf = env.getMasterConfiguration(); final ForeignExceptionDispatcher monitorException = new ForeignExceptionDispatcher(); getMonitorStatus().setStatus("Clone snapshot - creating
[2/8] hbase git commit: HBASE-16904 Snapshot related changes for FS redo work
http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java deleted file mode 100644 index 572bc04..000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java +++ /dev/null @@ -1,570 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.snapshot; - -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.InvalidProtocolBufferException; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.fs.RegionStorage; -import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier; -import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor; -import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare; -import org.apache.hadoop.hbase.mob.MobUtils; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; -import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotDataManifest; -import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.Store; -import org.apache.hadoop.hbase.regionserver.StoreFile; -import org.apache.hadoop.hbase.regionserver.StoreFileInfo; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.Threads; -import org.apache.zookeeper.server.persistence.SnapShot; - -/** - * Utility class to help read/write the Snapshot Manifest. - * - * The snapshot format is transparent for the users of this class, - * once the snapshot is written, it will never be modified. - * On open() the snapshot will be loaded to the current in-memory format. - */ -@InterfaceAudience.Private -public final class SnapshotManifest { - private static final Log LOG = LogFactory.getLog(SnapshotManifest.class); - - public static final String SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY = "snapshot.manifest.size.limit"; - - public static final String DATA_MANIFEST_NAME = "data.manifest"; - - private List regionManifests; - private SnapshotDescription desc; - private HTableDescriptor htd; - - private final ForeignExceptionSnare monitor; - private final Configuration conf; - private final Path workingDir; - private final FileSystem fs; - private int manifestSizeLimit; - - private SnapshotManifest(final Configuration conf, final FileSystem fs, - final Path workingDir, final SnapshotDescription desc, - final ForeignExceptionSnare monitor) { -this.monitor = monitor; -this.desc = desc; -this.workingDir = workingDir; -this.conf = conf; -this.fs = fs; - -this.manifestSizeLimit = conf.getInt(SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 64 * 1024 * 1024); - } - - /** - * Return a SnapshotManifest instance, used for writing a snapshot. - * - * There are two usage pattern: - * - The Master will create a manifest, add the descriptor, offline regions - *and consolidate the snapshot by writing all the pendin
[4/8] hbase git commit: HBASE-16904 Snapshot related changes for FS redo work
http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java index 75a1a17..a56744b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java @@ -29,12 +29,7 @@ import java.util.concurrent.ThreadPoolExecutor; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.Stoppable; @@ -44,7 +39,7 @@ import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.errorhandling.ForeignException; import org.apache.hadoop.hbase.executor.ExecutorService; -import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier; +import org.apache.hadoop.hbase.fs.StorageContext; import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.fs.MasterStorage; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; @@ -74,12 +69,10 @@ import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException; import org.apache.hadoop.hbase.snapshot.SnapshotExistsException; -import org.apache.hadoop.hbase.snapshot.SnapshotManifest; import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; import org.apache.hadoop.hbase.snapshot.TablePartiallyOpenException; import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.KeyLocker; import org.apache.zookeeper.KeeperException; @@ -140,7 +133,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable // Snapshot handlers map, with table name as key. // The map is always accessed and modified under the object lock using synchronized. - // snapshotTable() will insert an Handler in the table. + // initiateSnapshot() will insert an Handler in the table. // isSnapshotDone() will remove the handler requested if the operation is finished. private Map snapshotHandlers = new HashMap(); @@ -154,7 +147,6 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable // snapshot using Procedure-V2. private Map restoreTableToProcIdMap = new HashMap(); - private Path rootDir; private ExecutorService executorService; /** @@ -179,131 +171,65 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable ProcedureCoordinator coordinator, ExecutorService pool) throws IOException, UnsupportedOperationException { this.master = master; - -this.rootDir = ((LegacyPathIdentifier) master.getMasterStorage().getRootContainer()).path; -checkSnapshotSupport(master.getConfiguration(), master.getMasterStorage()); +checkSnapshotSupport(master.getConfiguration()); this.coordinator = coordinator; this.executorService = pool; -resetTempDir(); +this.master.getMasterStorage().deleteAllSnapshots(StorageContext.TEMP); } /** * Gets the list of all completed snapshots. * @return list of SnapshotDescriptions - * @throws IOException File system exception + * @throws IOException Storage exception */ public List getCompletedSnapshots() throws IOException { -return getCompletedSnapshots(SnapshotDescriptionUtils.getSnapshotsDir(rootDir)); - } +List snapshotDescs = new ArrayList<>(); - /** - * Gets the list of all completed snapshots. - * @param snapshotDir snapshot directory - * @return list of SnapshotDescriptions - * @throws IOException File system exception - */ - private List getCompletedSnapshots(Path snapshotDir) throws IOException { -List snapshotDescs = new ArrayList(); -// first create the snapshot root path and check to see if it exists -FileSystem fs = master.getMasterStorage().getFileSystem(); -if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); - -// if there are no snapshots, return an empty list -if (!fs