[10/50] [abbrv] hbase git commit: HBASE-16499 slow replication for small HBase clusters
HBASE-16499 slow replication for small HBase clusters Signed-off-by: Ashish SinghiProject: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5937202f Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5937202f Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5937202f Branch: refs/heads/HBASE-19064 Commit: 5937202fd5d6c5fba74bae21846f62da4ee35583 Parents: b1b0db3 Author: Ashish Singhi Authored: Wed Apr 4 09:54:41 2018 +0530 Committer: Ashish Singhi Committed: Wed Apr 4 09:54:41 2018 +0530 -- .../regionserver/ReplicationSinkManager.java| 2 +- .../TestReplicationSinkManager.java | 36 +++- 2 files changed, 21 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/5937202f/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java index af6888c..3cd7884 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java @@ -58,7 +58,7 @@ public class ReplicationSinkManager { * Default ratio of the total number of peer cluster region servers to consider * replicating to. */ - static final float DEFAULT_REPLICATION_SOURCE_RATIO = 0.1f; + static final float DEFAULT_REPLICATION_SOURCE_RATIO = 0.5f; private final Connection conn; http://git-wip-us.apache.org/repos/asf/hbase/blob/5937202f/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java index 3be3bfb..39dabb4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java @@ -27,7 +27,6 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint; -import org.apache.hadoop.hbase.replication.ReplicationPeers; import org.apache.hadoop.hbase.replication.regionserver.ReplicationSinkManager.SinkPeer; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.testclassification.SmallTests; @@ -49,13 +48,11 @@ public class TestReplicationSinkManager { private static final String PEER_CLUSTER_ID = "PEER_CLUSTER_ID"; - private ReplicationPeers replicationPeers; private HBaseReplicationEndpoint replicationEndpoint; private ReplicationSinkManager sinkManager; @Before public void setUp() { -replicationPeers = mock(ReplicationPeers.class); replicationEndpoint = mock(HBaseReplicationEndpoint.class); sinkManager = new ReplicationSinkManager(mock(ClusterConnection.class), PEER_CLUSTER_ID, replicationEndpoint, new Configuration()); @@ -64,7 +61,8 @@ public class TestReplicationSinkManager { @Test public void testChooseSinks() { List serverNames = Lists.newArrayList(); -for (int i = 0; i < 20; i++) { +int totalServers = 20; +for (int i = 0; i < totalServers; i++) { serverNames.add(mock(ServerName.class)); } @@ -73,7 +71,8 @@ public class TestReplicationSinkManager { sinkManager.chooseSinks(); -assertEquals(2, sinkManager.getNumSinks()); +int expected = (int) (totalServers * ReplicationSinkManager.DEFAULT_REPLICATION_SOURCE_RATIO); +assertEquals(expected, sinkManager.getNumSinks()); } @@ -117,7 +116,8 @@ public class TestReplicationSinkManager { @Test public void testReportBadSink_PastThreshold() { List serverNames = Lists.newArrayList(); -for (int i = 0; i < 30; i++) { +int totalServers = 30; +for (int i = 0; i < totalServers; i++) { serverNames.add(mock(ServerName.class)); } when(replicationEndpoint.getRegionServers()) @@ -126,7 +126,8 @@ public class TestReplicationSinkManager { sinkManager.chooseSinks(); //
[21/50] [abbrv] hbase git commit: HBASE-16499 slow replication for small HBase clusters - addendum for updating in the document
HBASE-16499 slow replication for small HBase clusters - addendum for updating in the document Signed-off-by: Ashish SinghiProject: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e2b0490d Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e2b0490d Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e2b0490d Branch: refs/heads/HBASE-19064 Commit: e2b0490d18f7cc03aa59475a1b423597ddc481fb Parents: 5fed7fd Author: Ashish Singhi Authored: Thu Apr 5 11:16:52 2018 +0530 Committer: Ashish Singhi Committed: Thu Apr 5 11:16:52 2018 +0530 -- src/main/asciidoc/_chapters/upgrading.adoc | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/e2b0490d/src/main/asciidoc/_chapters/upgrading.adoc -- diff --git a/src/main/asciidoc/_chapters/upgrading.adoc b/src/main/asciidoc/_chapters/upgrading.adoc index 68adb14..f5cdff3 100644 --- a/src/main/asciidoc/_chapters/upgrading.adoc +++ b/src/main/asciidoc/_chapters/upgrading.adoc @@ -390,6 +390,7 @@ The following configuration settings changed their default value. Where applicab * hbase.client.max.perserver.tasks is now 2. Previously it was 5. * hbase.normalizer.period is now 5 minutes. Previously it was 30 minutes. * hbase.regionserver.region.split.policy is now SteppingSplitPolicy. Previously it was IncreasingToUpperBoundRegionSplitPolicy. +* replication.source.ratio is now 0.5. Previously it was 0.1. [[upgrade2.0.regions.on.master]] ."Master hosting regions" feature broken and unsupported
hbase git commit: HBASE-16499 slow replication for small HBase clusters - addendum for updating in the document
Repository: hbase Updated Branches: refs/heads/master 5fed7fd3d -> e2b0490d1 HBASE-16499 slow replication for small HBase clusters - addendum for updating in the document Signed-off-by: Ashish SinghiProject: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e2b0490d Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e2b0490d Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e2b0490d Branch: refs/heads/master Commit: e2b0490d18f7cc03aa59475a1b423597ddc481fb Parents: 5fed7fd Author: Ashish Singhi Authored: Thu Apr 5 11:16:52 2018 +0530 Committer: Ashish Singhi Committed: Thu Apr 5 11:16:52 2018 +0530 -- src/main/asciidoc/_chapters/upgrading.adoc | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/e2b0490d/src/main/asciidoc/_chapters/upgrading.adoc -- diff --git a/src/main/asciidoc/_chapters/upgrading.adoc b/src/main/asciidoc/_chapters/upgrading.adoc index 68adb14..f5cdff3 100644 --- a/src/main/asciidoc/_chapters/upgrading.adoc +++ b/src/main/asciidoc/_chapters/upgrading.adoc @@ -390,6 +390,7 @@ The following configuration settings changed their default value. Where applicab * hbase.client.max.perserver.tasks is now 2. Previously it was 5. * hbase.normalizer.period is now 5 minutes. Previously it was 30 minutes. * hbase.regionserver.region.split.policy is now SteppingSplitPolicy. Previously it was IncreasingToUpperBoundRegionSplitPolicy. +* replication.source.ratio is now 0.5. Previously it was 0.1. [[upgrade2.0.regions.on.master]] ."Master hosting regions" feature broken and unsupported
hbase git commit: HBASE-16499 slow replication for small HBase clusters
Repository: hbase Updated Branches: refs/heads/branch-2.0 fb2a0eb66 -> 79bb54ddf HBASE-16499 slow replication for small HBase clusters Signed-off-by: Ashish SinghiProject: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/79bb54dd Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/79bb54dd Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/79bb54dd Branch: refs/heads/branch-2.0 Commit: 79bb54ddf4306796ab508a0612ff17c2e4ab863c Parents: fb2a0eb Author: Ashish Singhi Authored: Wed Apr 4 10:00:44 2018 +0530 Committer: Ashish Singhi Committed: Wed Apr 4 10:00:44 2018 +0530 -- .../regionserver/ReplicationSinkManager.java| 2 +- .../TestReplicationSinkManager.java | 36 +++- 2 files changed, 21 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/79bb54dd/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java index af6888c..3cd7884 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java @@ -58,7 +58,7 @@ public class ReplicationSinkManager { * Default ratio of the total number of peer cluster region servers to consider * replicating to. */ - static final float DEFAULT_REPLICATION_SOURCE_RATIO = 0.1f; + static final float DEFAULT_REPLICATION_SOURCE_RATIO = 0.5f; private final Connection conn; http://git-wip-us.apache.org/repos/asf/hbase/blob/79bb54dd/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java index 3be3bfb..39dabb4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java @@ -27,7 +27,6 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint; -import org.apache.hadoop.hbase.replication.ReplicationPeers; import org.apache.hadoop.hbase.replication.regionserver.ReplicationSinkManager.SinkPeer; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.testclassification.SmallTests; @@ -49,13 +48,11 @@ public class TestReplicationSinkManager { private static final String PEER_CLUSTER_ID = "PEER_CLUSTER_ID"; - private ReplicationPeers replicationPeers; private HBaseReplicationEndpoint replicationEndpoint; private ReplicationSinkManager sinkManager; @Before public void setUp() { -replicationPeers = mock(ReplicationPeers.class); replicationEndpoint = mock(HBaseReplicationEndpoint.class); sinkManager = new ReplicationSinkManager(mock(ClusterConnection.class), PEER_CLUSTER_ID, replicationEndpoint, new Configuration()); @@ -64,7 +61,8 @@ public class TestReplicationSinkManager { @Test public void testChooseSinks() { List serverNames = Lists.newArrayList(); -for (int i = 0; i < 20; i++) { +int totalServers = 20; +for (int i = 0; i < totalServers; i++) { serverNames.add(mock(ServerName.class)); } @@ -73,7 +71,8 @@ public class TestReplicationSinkManager { sinkManager.chooseSinks(); -assertEquals(2, sinkManager.getNumSinks()); +int expected = (int) (totalServers * ReplicationSinkManager.DEFAULT_REPLICATION_SOURCE_RATIO); +assertEquals(expected, sinkManager.getNumSinks()); } @@ -117,7 +116,8 @@ public class TestReplicationSinkManager { @Test public void testReportBadSink_PastThreshold() { List serverNames = Lists.newArrayList(); -for (int i = 0; i < 30; i++) { +int totalServers = 30; +for (int i = 0; i < totalServers; i++) { serverNames.add(mock(ServerName.class)); } when(replicationEndpoint.getRegionServers()) @@ -126,7 +126,8 @@
hbase git commit: HBASE-16499 slow replication for small HBase clusters
Repository: hbase Updated Branches: refs/heads/branch-2 ed21f2617 -> 9a3488072 HBASE-16499 slow replication for small HBase clusters Signed-off-by: Ashish SinghiProject: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9a348807 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9a348807 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9a348807 Branch: refs/heads/branch-2 Commit: 9a3488072456809dbd139c343f849410df4cc0ee Parents: ed21f26 Author: Ashish Singhi Authored: Wed Apr 4 09:59:50 2018 +0530 Committer: Ashish Singhi Committed: Wed Apr 4 09:59:50 2018 +0530 -- .../regionserver/ReplicationSinkManager.java| 2 +- .../TestReplicationSinkManager.java | 36 +++- 2 files changed, 21 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/9a348807/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java index af6888c..3cd7884 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java @@ -58,7 +58,7 @@ public class ReplicationSinkManager { * Default ratio of the total number of peer cluster region servers to consider * replicating to. */ - static final float DEFAULT_REPLICATION_SOURCE_RATIO = 0.1f; + static final float DEFAULT_REPLICATION_SOURCE_RATIO = 0.5f; private final Connection conn; http://git-wip-us.apache.org/repos/asf/hbase/blob/9a348807/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java index 3be3bfb..39dabb4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java @@ -27,7 +27,6 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint; -import org.apache.hadoop.hbase.replication.ReplicationPeers; import org.apache.hadoop.hbase.replication.regionserver.ReplicationSinkManager.SinkPeer; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.testclassification.SmallTests; @@ -49,13 +48,11 @@ public class TestReplicationSinkManager { private static final String PEER_CLUSTER_ID = "PEER_CLUSTER_ID"; - private ReplicationPeers replicationPeers; private HBaseReplicationEndpoint replicationEndpoint; private ReplicationSinkManager sinkManager; @Before public void setUp() { -replicationPeers = mock(ReplicationPeers.class); replicationEndpoint = mock(HBaseReplicationEndpoint.class); sinkManager = new ReplicationSinkManager(mock(ClusterConnection.class), PEER_CLUSTER_ID, replicationEndpoint, new Configuration()); @@ -64,7 +61,8 @@ public class TestReplicationSinkManager { @Test public void testChooseSinks() { List serverNames = Lists.newArrayList(); -for (int i = 0; i < 20; i++) { +int totalServers = 20; +for (int i = 0; i < totalServers; i++) { serverNames.add(mock(ServerName.class)); } @@ -73,7 +71,8 @@ public class TestReplicationSinkManager { sinkManager.chooseSinks(); -assertEquals(2, sinkManager.getNumSinks()); +int expected = (int) (totalServers * ReplicationSinkManager.DEFAULT_REPLICATION_SOURCE_RATIO); +assertEquals(expected, sinkManager.getNumSinks()); } @@ -117,7 +116,8 @@ public class TestReplicationSinkManager { @Test public void testReportBadSink_PastThreshold() { List serverNames = Lists.newArrayList(); -for (int i = 0; i < 30; i++) { +int totalServers = 30; +for (int i = 0; i < totalServers; i++) { serverNames.add(mock(ServerName.class)); } when(replicationEndpoint.getRegionServers()) @@ -126,7 +126,8 @@
hbase git commit: HBASE-16499 slow replication for small HBase clusters
Repository: hbase Updated Branches: refs/heads/master b1b0db319 -> 5937202fd HBASE-16499 slow replication for small HBase clusters Signed-off-by: Ashish SinghiProject: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5937202f Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5937202f Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5937202f Branch: refs/heads/master Commit: 5937202fd5d6c5fba74bae21846f62da4ee35583 Parents: b1b0db3 Author: Ashish Singhi Authored: Wed Apr 4 09:54:41 2018 +0530 Committer: Ashish Singhi Committed: Wed Apr 4 09:54:41 2018 +0530 -- .../regionserver/ReplicationSinkManager.java| 2 +- .../TestReplicationSinkManager.java | 36 +++- 2 files changed, 21 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/5937202f/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java index af6888c..3cd7884 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java @@ -58,7 +58,7 @@ public class ReplicationSinkManager { * Default ratio of the total number of peer cluster region servers to consider * replicating to. */ - static final float DEFAULT_REPLICATION_SOURCE_RATIO = 0.1f; + static final float DEFAULT_REPLICATION_SOURCE_RATIO = 0.5f; private final Connection conn; http://git-wip-us.apache.org/repos/asf/hbase/blob/5937202f/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java index 3be3bfb..39dabb4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java @@ -27,7 +27,6 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint; -import org.apache.hadoop.hbase.replication.ReplicationPeers; import org.apache.hadoop.hbase.replication.regionserver.ReplicationSinkManager.SinkPeer; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.testclassification.SmallTests; @@ -49,13 +48,11 @@ public class TestReplicationSinkManager { private static final String PEER_CLUSTER_ID = "PEER_CLUSTER_ID"; - private ReplicationPeers replicationPeers; private HBaseReplicationEndpoint replicationEndpoint; private ReplicationSinkManager sinkManager; @Before public void setUp() { -replicationPeers = mock(ReplicationPeers.class); replicationEndpoint = mock(HBaseReplicationEndpoint.class); sinkManager = new ReplicationSinkManager(mock(ClusterConnection.class), PEER_CLUSTER_ID, replicationEndpoint, new Configuration()); @@ -64,7 +61,8 @@ public class TestReplicationSinkManager { @Test public void testChooseSinks() { List serverNames = Lists.newArrayList(); -for (int i = 0; i < 20; i++) { +int totalServers = 20; +for (int i = 0; i < totalServers; i++) { serverNames.add(mock(ServerName.class)); } @@ -73,7 +71,8 @@ public class TestReplicationSinkManager { sinkManager.chooseSinks(); -assertEquals(2, sinkManager.getNumSinks()); +int expected = (int) (totalServers * ReplicationSinkManager.DEFAULT_REPLICATION_SOURCE_RATIO); +assertEquals(expected, sinkManager.getNumSinks()); } @@ -117,7 +116,8 @@ public class TestReplicationSinkManager { @Test public void testReportBadSink_PastThreshold() { List serverNames = Lists.newArrayList(); -for (int i = 0; i < 30; i++) { +int totalServers = 30; +for (int i = 0; i < totalServers; i++) { serverNames.add(mock(ServerName.class)); } when(replicationEndpoint.getRegionServers()) @@ -126,7 +126,8 @@ public