hbase git commit: HBASE-20120 Removed unused classes/ java files from hbase-server
Repository: hbase
Updated Branches:
refs/heads/master 70240f973 -> 45bbee490
HBASE-20120 Removed unused classes/ java files from hbase-server
deleted:
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NoOpHeapMemoryTuner.java
deleted:
hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseWALEntryFilter.java
deleted:
hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSMapRUtils.java
deleted:
hbase-server/src/main/java/org/apache/hadoop/hbase/util/ProtoUtil.java
Signed-off-by: Sean Busbey
Signed-off-by: tedyu
Signed-off-by: Chia-Ping Tsai
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/45bbee49
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/45bbee49
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/45bbee49
Branch: refs/heads/master
Commit: 45bbee4905fb031259c935502444c729b32ddac8
Parents: 70240f9
Author: Umesh Agashe
Authored: Tue Feb 6 16:09:43 2018 -0800
Committer: Chia-Ping Tsai
Committed: Sat Mar 10 13:08:38 2018 +0800
--
.../hbase/regionserver/NoOpHeapMemoryTuner.java | 48 -
.../hbase/replication/BaseWALEntryFilter.java | 29 --
.../apache/hadoop/hbase/util/FSMapRUtils.java | 46
.../org/apache/hadoop/hbase/util/ProtoUtil.java | 104 ---
4 files changed, 227 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/45bbee49/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NoOpHeapMemoryTuner.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NoOpHeapMemoryTuner.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NoOpHeapMemoryTuner.java
deleted file mode 100644
index 24cf85c..000
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NoOpHeapMemoryTuner.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.regionserver;
-
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext;
-import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult;
-
-/**
- * An implementation of HeapMemoryTuner which is not doing any tuning activity
but just allows to
- * continue with old style fixed proportions.
- */
-@InterfaceAudience.Private
-public class NoOpHeapMemoryTuner implements HeapMemoryTuner {
-
- private static final TunerResult NO_OP_TUNER_RESULT = new TunerResult(false);
-
- @Override
- public Configuration getConf() {
-return null;
- }
-
- @Override
- public void setConf(Configuration conf) {
-
- }
-
- @Override
- public TunerResult tune(TunerContext context) {
-return NO_OP_TUNER_RESULT;
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/45bbee49/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseWALEntryFilter.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseWALEntryFilter.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseWALEntryFilter.java
deleted file mode 100644
index cd062a3..000
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseWALEntryFilter.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under th
hbase git commit: HBASE-20120 Removed unused classes/ java files from hbase-server
Repository: hbase
Updated Branches:
refs/heads/branch-2 1728b48c8 -> 842678c99
HBASE-20120 Removed unused classes/ java files from hbase-server
deleted:
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NoOpHeapMemoryTuner.java
deleted:
hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseWALEntryFilter.java
deleted:
hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSMapRUtils.java
deleted:
hbase-server/src/main/java/org/apache/hadoop/hbase/util/ProtoUtil.java
Signed-off-by: Sean Busbey
Signed-off-by: tedyu
Signed-off-by: Chia-Ping Tsai
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/842678c9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/842678c9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/842678c9
Branch: refs/heads/branch-2
Commit: 842678c9914dfe64fc762fee552c8216fd44f903
Parents: 1728b48
Author: Umesh Agashe
Authored: Tue Feb 6 16:09:43 2018 -0800
Committer: Chia-Ping Tsai
Committed: Sat Mar 10 12:59:48 2018 +0800
--
.../hbase/regionserver/NoOpHeapMemoryTuner.java | 48 -
.../hbase/replication/BaseWALEntryFilter.java | 29 --
.../apache/hadoop/hbase/util/FSMapRUtils.java | 46
.../org/apache/hadoop/hbase/util/ProtoUtil.java | 104 ---
4 files changed, 227 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/842678c9/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NoOpHeapMemoryTuner.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NoOpHeapMemoryTuner.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NoOpHeapMemoryTuner.java
deleted file mode 100644
index 24cf85c..000
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NoOpHeapMemoryTuner.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.regionserver;
-
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext;
-import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult;
-
-/**
- * An implementation of HeapMemoryTuner which is not doing any tuning activity
but just allows to
- * continue with old style fixed proportions.
- */
-@InterfaceAudience.Private
-public class NoOpHeapMemoryTuner implements HeapMemoryTuner {
-
- private static final TunerResult NO_OP_TUNER_RESULT = new TunerResult(false);
-
- @Override
- public Configuration getConf() {
-return null;
- }
-
- @Override
- public void setConf(Configuration conf) {
-
- }
-
- @Override
- public TunerResult tune(TunerContext context) {
-return NO_OP_TUNER_RESULT;
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/842678c9/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseWALEntryFilter.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseWALEntryFilter.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseWALEntryFilter.java
deleted file mode 100644
index cd062a3..000
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseWALEntryFilter.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed unde
hbase git commit: HBASE-20132 Change the "KV" to "Cell" for web UI
Repository: hbase
Updated Branches:
refs/heads/branch-2 b87165cbf -> 1728b48c8
HBASE-20132 Change the "KV" to "Cell" for web UI
Signed-off-by: Chia-Ping Tsai
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1728b48c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1728b48c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1728b48c
Branch: refs/heads/branch-2
Commit: 1728b48c8238a4d0372fca01ad733ce703c65874
Parents: b87165c
Author: Guangxu Cheng
Authored: Mon Mar 5 11:20:25 2018 +0800
Committer: Chia-Ping Tsai
Committed: Sat Mar 10 12:36:29 2018 +0800
--
.../hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon | 12 ++--
.../hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon | 6 +++---
.../hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon | 4 ++--
3 files changed, 11 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/1728b48c/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon
--
diff --git
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon
index 9a0e369..41fe487 100644
---
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon
+++
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon
@@ -310,9 +310,9 @@ if (master.getServerManager() != null) {
RSGroup Name
-Num. Compacting KVs
-Num. Compacted KVs
-Remaining KVs
+Num. Compacting Cells
+Num. Compacted Cells
+Remaining Cells
Compaction Progress
<%java>
@@ -321,7 +321,7 @@ if (master.getServerManager() != null) {
int numStores = 0;
long totalCompactingCells = 0;
long totalCompactedCells = 0;
- long remainingKVs = 0;
+ long remainingCells = 0;
long compactionProgress = 0;
for (Address server : rsGroupInfo.getServers()) {
ServerMetrics sl = collectServers.get(server);
@@ -332,7 +332,7 @@ if (master.getServerManager() != null) {
}
}
}
- remainingKVs = totalCompactingCells - totalCompactedCells;
+ remainingCells = totalCompactingCells - totalCompactedCells;
String percentDone = "";
if (totalCompactingCells > 0) {
percentDone = String.format("%.2f", 100 *
@@ -343,7 +343,7 @@ if (master.getServerManager() != null) {
<& rsGroupLink; rsGroupName=rsGroupName; &>
<% totalCompactingCells %>
<% totalCompactedCells %>
-<% remainingKVs %>
+<% remainingCells %>
<% percentDone %>
<%java>
http://git-wip-us.apache.org/repos/asf/hbase/blob/1728b48c/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon
--
diff --git
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon
index fb7dd54..f353d32 100644
---
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon
+++
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon
@@ -289,9 +289,9 @@ if (sl != null) {
ServerName
-Num. Compacting KVs
-Num. Compacted KVs
-Remaining KVs
+Num. Compacting Cells
+Num. Compacted Cells
+Remaining Cells
Compaction Progress
<%java>
http://git-wip-us.apache.org/repos/asf/hbase/blob/1728b48c/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon
--
diff --git
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon
index d16ce06..6a01533 100644
---
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon
+++
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon
@@ -196,8 +196,8 @@
Region Name
-Num. Compacting KVs
-Num. Compacted KVs
+Num. Compacting Cells
+Num. Compacted Cells
Compaction Progress
Last Major Compaction
hbase git commit: HBASE-20132 Change the "KV" to "Cell" for web UI
Repository: hbase
Updated Branches:
refs/heads/master dd6f4525e -> 70240f973
HBASE-20132 Change the "KV" to "Cell" for web UI
Signed-off-by: Chia-Ping Tsai
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/70240f97
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/70240f97
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/70240f97
Branch: refs/heads/master
Commit: 70240f9732da11484b47eb720a39c1cc8bb1a4e4
Parents: dd6f452
Author: Guangxu Cheng
Authored: Mon Mar 5 11:20:25 2018 +0800
Committer: Chia-Ping Tsai
Committed: Sat Mar 10 12:27:59 2018 +0800
--
.../hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon | 12 ++--
.../hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon | 6 +++---
.../hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon | 4 ++--
3 files changed, 11 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/70240f97/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon
--
diff --git
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon
index 9a0e369..41fe487 100644
---
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon
+++
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon
@@ -310,9 +310,9 @@ if (master.getServerManager() != null) {
RSGroup Name
-Num. Compacting KVs
-Num. Compacted KVs
-Remaining KVs
+Num. Compacting Cells
+Num. Compacted Cells
+Remaining Cells
Compaction Progress
<%java>
@@ -321,7 +321,7 @@ if (master.getServerManager() != null) {
int numStores = 0;
long totalCompactingCells = 0;
long totalCompactedCells = 0;
- long remainingKVs = 0;
+ long remainingCells = 0;
long compactionProgress = 0;
for (Address server : rsGroupInfo.getServers()) {
ServerMetrics sl = collectServers.get(server);
@@ -332,7 +332,7 @@ if (master.getServerManager() != null) {
}
}
}
- remainingKVs = totalCompactingCells - totalCompactedCells;
+ remainingCells = totalCompactingCells - totalCompactedCells;
String percentDone = "";
if (totalCompactingCells > 0) {
percentDone = String.format("%.2f", 100 *
@@ -343,7 +343,7 @@ if (master.getServerManager() != null) {
<& rsGroupLink; rsGroupName=rsGroupName; &>
<% totalCompactingCells %>
<% totalCompactedCells %>
-<% remainingKVs %>
+<% remainingCells %>
<% percentDone %>
<%java>
http://git-wip-us.apache.org/repos/asf/hbase/blob/70240f97/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon
--
diff --git
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon
index fb7dd54..f353d32 100644
---
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon
+++
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon
@@ -289,9 +289,9 @@ if (sl != null) {
ServerName
-Num. Compacting KVs
-Num. Compacted KVs
-Remaining KVs
+Num. Compacting Cells
+Num. Compacted Cells
+Remaining Cells
Compaction Progress
<%java>
http://git-wip-us.apache.org/repos/asf/hbase/blob/70240f97/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon
--
diff --git
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon
index d16ce06..6a01533 100644
---
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon
+++
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon
@@ -196,8 +196,8 @@
Region Name
-Num. Compacting KVs
-Num. Compacted KVs
+Num. Compacting Cells
+Num. Compacted Cells
Compaction Progress
Last Major Compaction
hbase git commit: HBASE-20148 Make serial replication as a option for a peer instead of a table
Repository: hbase
Updated Branches:
refs/heads/master 15da74cce -> dd6f4525e
HBASE-20148 Make serial replication as a option for a peer instead of a table
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dd6f4525
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dd6f4525
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dd6f4525
Branch: refs/heads/master
Commit: dd6f4525e7a9eaceb6ed4f97059b2dd3c532d323
Parents: 15da74c
Author: zhangduo
Authored: Fri Mar 9 15:00:59 2018 +0800
Committer: zhangduo
Committed: Sat Mar 10 09:04:44 2018 +0800
--
.../apache/hadoop/hbase/HTableDescriptor.java | 8 -
.../hadoop/hbase/client/TableDescriptor.java| 20 +++-
.../hbase/client/TableDescriptorBuilder.java| 9 --
.../replication/ReplicationPeerConfigUtil.java | 5 +++
.../replication/ReplicationPeerConfig.java | 32 +++-
.../ReplicationPeerConfigBuilder.java | 12
.../org/apache/hadoop/hbase/HConstants.java | 6
.../src/main/protobuf/Replication.proto | 1 +
.../hbase/replication/ReplicationUtils.java | 3 ++
.../master/assignment/RegionStateStore.java | 14 -
.../hbase/replication/ScopeWALEntryFilter.java | 32 ++--
.../regionserver/ReplicationSource.java | 4 +++
.../ReplicationSourceWALActionListener.java | 10 +-
.../ReplicationSourceWALReader.java | 6 ++--
.../regionserver/SerialReplicationChecker.java | 2 +-
.../org/apache/hadoop/hbase/wal/WALKeyImpl.java | 8 -
.../TestReplicationWALEntryFilters.java | 15 ++---
.../replication/TestSerialReplication.java | 9 +++---
18 files changed, 104 insertions(+), 92 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd6f4525/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index 3652d10..db8870d 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -537,14 +537,6 @@ public class HTableDescriptor implements TableDescriptor,
Comparablehttp://git-wip-us.apache.org/repos/asf/hbase/blob/dd6f4525/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptor.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptor.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptor.java
index 3505175..1ec61a2 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptor.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptor.java
@@ -24,7 +24,7 @@ import java.util.Comparator;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
-
+import java.util.stream.Stream;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.util.Bytes;
@@ -232,11 +232,6 @@ public interface TableDescriptor {
boolean hasRegionMemStoreReplication();
/**
- * @return true if there are at least one cf whose replication scope is
serial.
- */
- boolean hasSerialReplicationScope();
-
- /**
* Check if the compaction enable flag of the table is true. If flag is false
* then no minor/major compactions will be done in real.
*
@@ -275,6 +270,16 @@ public interface TableDescriptor {
boolean isReadOnly();
/**
+ * Check if any of the table's cfs' replication scope are set to
+ * {@link HConstants#REPLICATION_SCOPE_GLOBAL}.
+ * @return {@code true} if we have, otherwise {@code false}.
+ */
+ default boolean hasGlobalReplicationScope() {
+return Stream.of(getColumnFamilies())
+ .anyMatch(cf -> cf.getScope() == HConstants.REPLICATION_SCOPE_GLOBAL);
+ }
+
+ /**
* Check if the table's cfs' replication scope matched with the replication
state
* @param enabled replication state
* @return true if matched, otherwise false
@@ -284,8 +289,7 @@ public interface TableDescriptor {
boolean hasDisabled = false;
for (ColumnFamilyDescriptor cf : getColumnFamilies()) {
- if (cf.getScope() != HConstants.REPLICATION_SCOPE_GLOBAL &&
-cf.getScope() != HConstants.REPLICATION_SCOPE_SERIAL) {
+ if (cf.getScope() != HConstants.REPLICATION_SCOPE_GLOBAL) {
hasDisabled = true;
} else {
hasEnabled = true;
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd6f4525/hbase-client/src/ma
hbase git commit: HBASE-20164 add hadoopcheck link to table footer
Repository: hbase
Updated Branches:
refs/heads/branch-1.2 d317aa4a1 -> fd43ff465
HBASE-20164 add hadoopcheck link to table footer
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fd43ff46
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fd43ff46
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fd43ff46
Branch: refs/heads/branch-1.2
Commit: fd43ff465a32dbdbd08ba7d21d9e26ed4fa0c2ea
Parents: d317aa4
Author: Mike Drob
Authored: Thu Mar 8 21:55:25 2018 -0600
Committer: Sean Busbey
Committed: Fri Mar 9 17:27:42 2018 -0600
--
dev-support/hbase-personality.sh | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/fd43ff46/dev-support/hbase-personality.sh
--
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index f902ac1..ddf9dee 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -370,6 +370,7 @@ function hadoopcheck_rebuild
count=$(${GREP} -c '\[ERROR\]' "${logfile}")
if [[ ${count} -gt 0 ]]; then
add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors
with Hadoop v${hadoopver}."
+ add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
((result=result+1))
fi
done
@@ -384,6 +385,7 @@ function hadoopcheck_rebuild
count=$(${GREP} -c '\[ERROR\]' "${logfile}")
if [[ ${count} -gt 0 ]]; then
add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors
with Hadoop v${hadoopver}."
+ add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
((result=result+1))
fi
done
[3/7] hbase git commit: HBASE-20162 [nightly] depending on pipeline execution we sometimes refer to the wrong workspace
HBASE-20162 [nightly] depending on pipeline execution we sometimes refer to the
wrong workspace
Signed-off-by: Mike Drob
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d016d4c2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d016d4c2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d016d4c2
Branch: refs/heads/branch-2.0
Commit: d016d4c228e4df035d8e47e5499002eb8e3fc56e
Parents: 35428dc
Author: Sean Busbey
Authored: Thu Mar 8 20:27:21 2018 -0600
Committer: Sean Busbey
Committed: Fri Mar 9 17:23:19 2018 -0600
--
dev-support/Jenkinsfile| 15 ++-
dev-support/hbase_nightly_yetus.sh | 8
2 files changed, 14 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d016d4c2/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 5453fd5..c91a2b1 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -30,9 +30,6 @@ pipeline {
skipDefaultCheckout()
}
environment {
-TOOLS = "${env.WORKSPACE}/tools"
-// where we check out to across stages
-BASEDIR = "${env.WORKSPACE}/component"
YETUS_RELEASE = '0.7.0'
// where we'll write everything from different steps. Need a copy here so
the final step can check for success/failure.
OUTPUT_DIR_RELATIVE_GENERAL = 'output-general'
@@ -42,6 +39,7 @@ pipeline {
PROJECT = 'hbase'
PROJECT_PERSONALITY =
'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
+PERSONALITY_FILE = 'tools/personality.sh'
// This section of the docs tells folks not to use the javadoc tag. older
branches have our old version of the check for said tag.
AUTHOR_IGNORE_LIST =
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
WHITESPACE_IGNORE_LIST = '.*/generated/.*'
@@ -49,7 +47,6 @@ pipeline {
ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
// These tests currently have known failures. Once they burn down to 0,
remove from here so that new problems will cause a failure.
TESTS_FILTER =
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
-BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
// Flaky urls for different branches. Replace '-' and '.' in branch name
by '_' because those
// characters are not allowed in bash variable name.
// Not excluding flakies from the nightly build for now.
@@ -101,7 +98,8 @@ if [ ! -d "${YETUS_DIR}" ]; then
gunzip -c yetus.tar.gz | tar xpf - -C "${YETUS_DIR}" --strip-components 1
fi
'''
-dir ("${env.TOOLS}") {
+// Set up the file we need at PERSONALITY_FILE location
+dir ("tools") {
sh """#!/usr/bin/env bash
echo "Downloading Project personality."
curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
@@ -130,6 +128,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
// TODO does hadoopcheck need to be jdk specific?
// Should be things that work with multijdk
TESTS = 'all,-unit,-findbugs'
@@ -191,6 +190,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
branch 'branch-1*'
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK7}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK7}"
@@ -259,6 +259,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
@@ -334,6 +335,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
@@ -408,6 +410,9 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
// this needs to be set to the jdk that ought to be used to build
releases on the branch the Jenkinsfile is stored in.
jdk "JDK 1.8 (latest)"
}
[4/7] hbase git commit: HBASE-20162 [nightly] depending on pipeline execution we sometimes refer to the wrong workspace
HBASE-20162 [nightly] depending on pipeline execution we sometimes refer to the
wrong workspace
Signed-off-by: Mike Drob
Conflicts:
dev-support/Jenkinsfile
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/55b3c0c2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/55b3c0c2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/55b3c0c2
Branch: refs/heads/branch-1
Commit: 55b3c0c21b5233fa1b4cd99e9d643b9fc3426138
Parents: eef6b36
Author: Sean Busbey
Authored: Thu Mar 8 20:27:21 2018 -0600
Committer: Sean Busbey
Committed: Fri Mar 9 17:24:23 2018 -0600
--
dev-support/Jenkinsfile| 15 ++-
dev-support/hbase_nightly_yetus.sh | 8
2 files changed, 14 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/55b3c0c2/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 54a752a..bb1da7a 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -30,9 +30,6 @@ pipeline {
skipDefaultCheckout()
}
environment {
-TOOLS = "${env.WORKSPACE}/tools"
-// where we check out to across stages
-BASEDIR = "${env.WORKSPACE}/component"
YETUS_RELEASE = '0.7.0'
// where we'll write everything from different steps. Need a copy here so
the final step can check for success/failure.
OUTPUT_DIR_RELATIVE_GENERAL = 'output-general'
@@ -42,6 +39,7 @@ pipeline {
PROJECT = 'hbase'
PROJECT_PERSONALITY =
'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
+PERSONALITY_FILE = 'tools/personality.sh'
// This section of the docs tells folks not to use the javadoc tag. older
branches have our old version of the check for said tag.
AUTHOR_IGNORE_LIST =
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
WHITESPACE_IGNORE_LIST = '.*/generated/.*'
@@ -49,7 +47,6 @@ pipeline {
ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
// These tests currently have known failures. Once they burn down to 0,
remove from here so that new problems will cause a failure.
TESTS_FILTER =
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite,xml'
-BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
// Flaky urls for different branches. Replace '-' and '.' in branch name
by '_' because those
// characters are not allowed in bash variable name.
// Not excluding flakies from the nightly build for now.
@@ -101,7 +98,8 @@ if [ ! -d "${YETUS_DIR}" ]; then
gunzip -c yetus.tar.gz | tar xpf - -C "${YETUS_DIR}" --strip-components 1
fi
'''
-dir ("${env.TOOLS}") {
+// Set up the file we need at PERSONALITY_FILE location
+dir ("tools") {
sh """#!/usr/bin/env bash
echo "Downloading Project personality."
curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
@@ -130,6 +128,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
// TODO does hadoopcheck need to be jdk specific?
// Should be things that work with multijdk
TESTS = 'all,-unit,-findbugs'
@@ -191,6 +190,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
branch 'branch-1*'
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK7}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK7}"
@@ -259,6 +259,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
@@ -334,6 +335,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
@@ -408,6 +410,9 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
// this needs to be set to the jdk that ought to be used to build
releases on the branch the Jenkinsfile is stored in.
[1/7] hbase git commit: HBASE-20162 [nightly] depending on pipeline execution we sometimes refer to the wrong workspace
Repository: hbase
Updated Branches:
refs/heads/branch-1 eef6b3654 -> 55b3c0c21
refs/heads/branch-1.2 ddaea95d2 -> d317aa4a1
refs/heads/branch-1.3 e02b28f93 -> 0d81bd171
refs/heads/branch-1.4 56294ec05 -> 15d75e4fc
refs/heads/branch-2 32c5118f6 -> b87165cbf
refs/heads/branch-2.0 35428dc0a -> d016d4c22
refs/heads/master 28279697c -> 15da74cce
HBASE-20162 [nightly] depending on pipeline execution we sometimes refer to the
wrong workspace
Signed-off-by: Mike Drob
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/15da74cc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/15da74cc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/15da74cc
Branch: refs/heads/master
Commit: 15da74cceeafe29d5ea67c52770b32338e8a2489
Parents: 2827969
Author: Sean Busbey
Authored: Thu Mar 8 20:27:21 2018 -0600
Committer: Sean Busbey
Committed: Fri Mar 9 17:22:00 2018 -0600
--
dev-support/Jenkinsfile| 15 ++-
dev-support/hbase_nightly_yetus.sh | 8
2 files changed, 14 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/15da74cc/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 6f83d2f..e8bf8c9 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -30,9 +30,6 @@ pipeline {
skipDefaultCheckout()
}
environment {
-TOOLS = "${env.WORKSPACE}/tools"
-// where we check out to across stages
-BASEDIR = "${env.WORKSPACE}/component"
YETUS_RELEASE = '0.7.0'
// where we'll write everything from different steps. Need a copy here so
the final step can check for success/failure.
OUTPUT_DIR_RELATIVE_GENERAL = 'output-general'
@@ -42,6 +39,7 @@ pipeline {
PROJECT = 'hbase'
PROJECT_PERSONALITY =
'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
+PERSONALITY_FILE = 'tools/personality.sh'
// This section of the docs tells folks not to use the javadoc tag. older
branches have our old version of the check for said tag.
AUTHOR_IGNORE_LIST =
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
WHITESPACE_IGNORE_LIST = '.*/generated/.*'
@@ -49,7 +47,6 @@ pipeline {
ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
// These tests currently have known failures. Once they burn down to 0,
remove from here so that new problems will cause a failure.
TESTS_FILTER =
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
-BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
// Flaky urls for different branches. Replace '-' and '.' in branch name
by '_' because those
// characters are not allowed in bash variable name.
// Not excluding flakies from the nightly build for now.
@@ -101,7 +98,8 @@ if [ ! -d "${YETUS_DIR}" ]; then
gunzip -c yetus.tar.gz | tar xpf - -C "${YETUS_DIR}" --strip-components 1
fi
'''
-dir ("${env.TOOLS}") {
+// Set up the file we need at PERSONALITY_FILE location
+dir ("tools") {
sh """#!/usr/bin/env bash
echo "Downloading Project personality."
curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
@@ -130,6 +128,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
// TODO does hadoopcheck need to be jdk specific?
// Should be things that work with multijdk
TESTS = 'all,-unit,-findbugs'
@@ -191,6 +190,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
branch 'branch-1*'
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK7}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK7}"
@@ -259,6 +259,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
@@ -334,6 +335,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_R
[2/7] hbase git commit: HBASE-20162 [nightly] depending on pipeline execution we sometimes refer to the wrong workspace
HBASE-20162 [nightly] depending on pipeline execution we sometimes refer to the
wrong workspace
Signed-off-by: Mike Drob
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b87165cb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b87165cb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b87165cb
Branch: refs/heads/branch-2
Commit: b87165cbf734f3d6d66c568865570dc16fb5a411
Parents: 32c5118
Author: Sean Busbey
Authored: Thu Mar 8 20:27:21 2018 -0600
Committer: Sean Busbey
Committed: Fri Mar 9 17:23:13 2018 -0600
--
dev-support/Jenkinsfile| 15 ++-
dev-support/hbase_nightly_yetus.sh | 8
2 files changed, 14 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/b87165cb/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 5453fd5..c91a2b1 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -30,9 +30,6 @@ pipeline {
skipDefaultCheckout()
}
environment {
-TOOLS = "${env.WORKSPACE}/tools"
-// where we check out to across stages
-BASEDIR = "${env.WORKSPACE}/component"
YETUS_RELEASE = '0.7.0'
// where we'll write everything from different steps. Need a copy here so
the final step can check for success/failure.
OUTPUT_DIR_RELATIVE_GENERAL = 'output-general'
@@ -42,6 +39,7 @@ pipeline {
PROJECT = 'hbase'
PROJECT_PERSONALITY =
'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
+PERSONALITY_FILE = 'tools/personality.sh'
// This section of the docs tells folks not to use the javadoc tag. older
branches have our old version of the check for said tag.
AUTHOR_IGNORE_LIST =
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
WHITESPACE_IGNORE_LIST = '.*/generated/.*'
@@ -49,7 +47,6 @@ pipeline {
ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
// These tests currently have known failures. Once they burn down to 0,
remove from here so that new problems will cause a failure.
TESTS_FILTER =
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
-BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
// Flaky urls for different branches. Replace '-' and '.' in branch name
by '_' because those
// characters are not allowed in bash variable name.
// Not excluding flakies from the nightly build for now.
@@ -101,7 +98,8 @@ if [ ! -d "${YETUS_DIR}" ]; then
gunzip -c yetus.tar.gz | tar xpf - -C "${YETUS_DIR}" --strip-components 1
fi
'''
-dir ("${env.TOOLS}") {
+// Set up the file we need at PERSONALITY_FILE location
+dir ("tools") {
sh """#!/usr/bin/env bash
echo "Downloading Project personality."
curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
@@ -130,6 +128,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
// TODO does hadoopcheck need to be jdk specific?
// Should be things that work with multijdk
TESTS = 'all,-unit,-findbugs'
@@ -191,6 +190,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
branch 'branch-1*'
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK7}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK7}"
@@ -259,6 +259,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
@@ -334,6 +335,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
@@ -408,6 +410,9 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
// this needs to be set to the jdk that ought to be used to build
releases on the branch the Jenkinsfile is stored in.
jdk "JDK 1.8 (latest)"
}
+
[7/7] hbase git commit: HBASE-20162 [nightly] depending on pipeline execution we sometimes refer to the wrong workspace
HBASE-20162 [nightly] depending on pipeline execution we sometimes refer to the
wrong workspace
Signed-off-by: Mike Drob
Conflicts:
dev-support/Jenkinsfile
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d317aa4a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d317aa4a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d317aa4a
Branch: refs/heads/branch-1.2
Commit: d317aa4a10df19a106f837d98a4bfcf7938ba255
Parents: ddaea95
Author: Sean Busbey
Authored: Thu Mar 8 20:27:21 2018 -0600
Committer: Sean Busbey
Committed: Fri Mar 9 17:25:38 2018 -0600
--
dev-support/Jenkinsfile| 15 ++-
dev-support/hbase_nightly_yetus.sh | 8
2 files changed, 14 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d317aa4a/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 4b96659..eeeda83 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -30,9 +30,6 @@ pipeline {
skipDefaultCheckout()
}
environment {
-TOOLS = "${env.WORKSPACE}/tools"
-// where we check out to across stages
-BASEDIR = "${env.WORKSPACE}/component"
YETUS_RELEASE = '0.6.0'
// where we'll write everything from different steps. Need a copy here so
the final step can check for success/failure.
OUTPUT_DIR_RELATIVE_GENERAL = 'output-general'
@@ -42,6 +39,7 @@ pipeline {
PROJECT = 'hbase'
PROJECT_PERSONALITY =
'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
+PERSONALITY_FILE = 'tools/personality.sh'
// This section of the docs tells folks not to use the javadoc tag. older
branches have our old version of the check for said tag.
AUTHOR_IGNORE_LIST =
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
WHITESPACE_IGNORE_LIST = '.*/generated/.*'
@@ -49,7 +47,6 @@ pipeline {
ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
// These tests currently have known failures. Once they burn down to 0,
remove from here so that new problems will cause a failure.
TESTS_FILTER =
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite,xml'
-BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
// Flaky urls for different branches. Replace '-' and '.' in branch name
by '_' because those
// characters are not allowed in bash variable name.
// Not excluding flakies from the nightly build for now.
@@ -101,7 +98,8 @@ if [ ! -d "${YETUS_DIR}" ]; then
gunzip -c yetus.tar.gz | tar xpf - -C "${YETUS_DIR}" --strip-components 1
fi
'''
-dir ("${env.TOOLS}") {
+// Set up the file we need at PERSONALITY_FILE location
+dir ("tools") {
sh """#!/usr/bin/env bash
echo "Downloading Project personality."
curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
@@ -130,6 +128,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
// TODO does hadoopcheck need to be jdk specific?
// Should be things that work with multijdk
TESTS = 'all,-unit,-findbugs'
@@ -191,6 +190,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
branch 'branch-1*'
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK7}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK7}"
@@ -259,6 +259,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
@@ -334,6 +335,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
@@ -408,6 +410,9 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
// this needs to be set to the jdk that ought to be used to build
releases on the branch the Jenkinsfile is stored in.
[6/7] hbase git commit: HBASE-20162 [nightly] depending on pipeline execution we sometimes refer to the wrong workspace
HBASE-20162 [nightly] depending on pipeline execution we sometimes refer to the
wrong workspace
Signed-off-by: Mike Drob
Conflicts:
dev-support/Jenkinsfile
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0d81bd17
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0d81bd17
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0d81bd17
Branch: refs/heads/branch-1.3
Commit: 0d81bd171b377dc90051124a49d5f4d837e2139e
Parents: e02b28f
Author: Sean Busbey
Authored: Thu Mar 8 20:27:21 2018 -0600
Committer: Sean Busbey
Committed: Fri Mar 9 17:25:07 2018 -0600
--
dev-support/Jenkinsfile| 15 ++-
dev-support/hbase_nightly_yetus.sh | 8
2 files changed, 14 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d81bd17/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 54a752a..bb1da7a 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -30,9 +30,6 @@ pipeline {
skipDefaultCheckout()
}
environment {
-TOOLS = "${env.WORKSPACE}/tools"
-// where we check out to across stages
-BASEDIR = "${env.WORKSPACE}/component"
YETUS_RELEASE = '0.7.0'
// where we'll write everything from different steps. Need a copy here so
the final step can check for success/failure.
OUTPUT_DIR_RELATIVE_GENERAL = 'output-general'
@@ -42,6 +39,7 @@ pipeline {
PROJECT = 'hbase'
PROJECT_PERSONALITY =
'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
+PERSONALITY_FILE = 'tools/personality.sh'
// This section of the docs tells folks not to use the javadoc tag. older
branches have our old version of the check for said tag.
AUTHOR_IGNORE_LIST =
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
WHITESPACE_IGNORE_LIST = '.*/generated/.*'
@@ -49,7 +47,6 @@ pipeline {
ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
// These tests currently have known failures. Once they burn down to 0,
remove from here so that new problems will cause a failure.
TESTS_FILTER =
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite,xml'
-BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
// Flaky urls for different branches. Replace '-' and '.' in branch name
by '_' because those
// characters are not allowed in bash variable name.
// Not excluding flakies from the nightly build for now.
@@ -101,7 +98,8 @@ if [ ! -d "${YETUS_DIR}" ]; then
gunzip -c yetus.tar.gz | tar xpf - -C "${YETUS_DIR}" --strip-components 1
fi
'''
-dir ("${env.TOOLS}") {
+// Set up the file we need at PERSONALITY_FILE location
+dir ("tools") {
sh """#!/usr/bin/env bash
echo "Downloading Project personality."
curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
@@ -130,6 +128,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
// TODO does hadoopcheck need to be jdk specific?
// Should be things that work with multijdk
TESTS = 'all,-unit,-findbugs'
@@ -191,6 +190,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
branch 'branch-1*'
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK7}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK7}"
@@ -259,6 +259,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
@@ -334,6 +335,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
@@ -408,6 +410,9 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
// this needs to be set to the jdk that ought to be used to build
releases on the branch the Jenkinsfile is stored in.
[5/7] hbase git commit: HBASE-20162 [nightly] depending on pipeline execution we sometimes refer to the wrong workspace
HBASE-20162 [nightly] depending on pipeline execution we sometimes refer to the
wrong workspace
Signed-off-by: Mike Drob
Conflicts:
dev-support/Jenkinsfile
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/15d75e4f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/15d75e4f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/15d75e4f
Branch: refs/heads/branch-1.4
Commit: 15d75e4fca8555a74919bc04e0703a0210eb4977
Parents: 56294ec
Author: Sean Busbey
Authored: Thu Mar 8 20:27:21 2018 -0600
Committer: Sean Busbey
Committed: Fri Mar 9 17:25:02 2018 -0600
--
dev-support/Jenkinsfile| 15 ++-
dev-support/hbase_nightly_yetus.sh | 8
2 files changed, 14 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/15d75e4f/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 54a752a..bb1da7a 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -30,9 +30,6 @@ pipeline {
skipDefaultCheckout()
}
environment {
-TOOLS = "${env.WORKSPACE}/tools"
-// where we check out to across stages
-BASEDIR = "${env.WORKSPACE}/component"
YETUS_RELEASE = '0.7.0'
// where we'll write everything from different steps. Need a copy here so
the final step can check for success/failure.
OUTPUT_DIR_RELATIVE_GENERAL = 'output-general'
@@ -42,6 +39,7 @@ pipeline {
PROJECT = 'hbase'
PROJECT_PERSONALITY =
'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
+PERSONALITY_FILE = 'tools/personality.sh'
// This section of the docs tells folks not to use the javadoc tag. older
branches have our old version of the check for said tag.
AUTHOR_IGNORE_LIST =
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
WHITESPACE_IGNORE_LIST = '.*/generated/.*'
@@ -49,7 +47,6 @@ pipeline {
ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
// These tests currently have known failures. Once they burn down to 0,
remove from here so that new problems will cause a failure.
TESTS_FILTER =
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite,xml'
-BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
// Flaky urls for different branches. Replace '-' and '.' in branch name
by '_' because those
// characters are not allowed in bash variable name.
// Not excluding flakies from the nightly build for now.
@@ -101,7 +98,8 @@ if [ ! -d "${YETUS_DIR}" ]; then
gunzip -c yetus.tar.gz | tar xpf - -C "${YETUS_DIR}" --strip-components 1
fi
'''
-dir ("${env.TOOLS}") {
+// Set up the file we need at PERSONALITY_FILE location
+dir ("tools") {
sh """#!/usr/bin/env bash
echo "Downloading Project personality."
curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
@@ -130,6 +128,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
// TODO does hadoopcheck need to be jdk specific?
// Should be things that work with multijdk
TESTS = 'all,-unit,-findbugs'
@@ -191,6 +190,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
branch 'branch-1*'
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK7}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK7}"
@@ -259,6 +259,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
@@ -334,6 +335,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
+BASEDIR = "${env.WORKSPACE}/component"
TESTS = 'mvninstall,compile,javac,unit,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
@@ -408,6 +410,9 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
// this needs to be set to the jdk that ought to be used to build
releases on the branch the Jenkinsfile is stored in.
hbase git commit: HBASE-20164 add hadoopcheck link to table footer
Repository: hbase
Updated Branches:
refs/heads/branch-1.3 4768ca9b9 -> e02b28f93
HBASE-20164 add hadoopcheck link to table footer
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e02b28f9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e02b28f9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e02b28f9
Branch: refs/heads/branch-1.3
Commit: e02b28f9341e5df9bd8ea9190e24b15a0cc0540d
Parents: 4768ca9
Author: Mike Drob
Authored: Thu Mar 8 21:55:25 2018 -0600
Committer: Mike Drob
Committed: Fri Mar 9 15:43:40 2018 -0600
--
dev-support/hbase-personality.sh | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/e02b28f9/dev-support/hbase-personality.sh
--
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index f902ac1..ddf9dee 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -370,6 +370,7 @@ function hadoopcheck_rebuild
count=$(${GREP} -c '\[ERROR\]' "${logfile}")
if [[ ${count} -gt 0 ]]; then
add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors
with Hadoop v${hadoopver}."
+ add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
((result=result+1))
fi
done
@@ -384,6 +385,7 @@ function hadoopcheck_rebuild
count=$(${GREP} -c '\[ERROR\]' "${logfile}")
if [[ ${count} -gt 0 ]]; then
add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors
with Hadoop v${hadoopver}."
+ add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
((result=result+1))
fi
done
hbase git commit: HBASE-20164 add hadoopcheck link to table footer
Repository: hbase
Updated Branches:
refs/heads/branch-1.4 a8228eaad -> 56294ec05
HBASE-20164 add hadoopcheck link to table footer
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/56294ec0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/56294ec0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/56294ec0
Branch: refs/heads/branch-1.4
Commit: 56294ec05bec06b4426ee1cfef180934997f50a2
Parents: a8228ea
Author: Mike Drob
Authored: Thu Mar 8 21:55:25 2018 -0600
Committer: Mike Drob
Committed: Fri Mar 9 15:42:27 2018 -0600
--
dev-support/hbase-personality.sh | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/56294ec0/dev-support/hbase-personality.sh
--
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index f902ac1..ddf9dee 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -370,6 +370,7 @@ function hadoopcheck_rebuild
count=$(${GREP} -c '\[ERROR\]' "${logfile}")
if [[ ${count} -gt 0 ]]; then
add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors
with Hadoop v${hadoopver}."
+ add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
((result=result+1))
fi
done
@@ -384,6 +385,7 @@ function hadoopcheck_rebuild
count=$(${GREP} -c '\[ERROR\]' "${logfile}")
if [[ ${count} -gt 0 ]]; then
add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors
with Hadoop v${hadoopver}."
+ add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
((result=result+1))
fi
done
hbase git commit: HBASE-20164 add hadoopcheck link to table footer
Repository: hbase
Updated Branches:
refs/heads/branch-1 e81abb725 -> eef6b3654
HBASE-20164 add hadoopcheck link to table footer
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/eef6b365
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/eef6b365
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/eef6b365
Branch: refs/heads/branch-1
Commit: eef6b36540681ab4c0d948da29fdf8a96af31fc7
Parents: e81abb7
Author: Mike Drob
Authored: Thu Mar 8 21:55:25 2018 -0600
Committer: Mike Drob
Committed: Fri Mar 9 15:41:56 2018 -0600
--
dev-support/hbase-personality.sh | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/eef6b365/dev-support/hbase-personality.sh
--
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index f902ac1..ddf9dee 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -370,6 +370,7 @@ function hadoopcheck_rebuild
count=$(${GREP} -c '\[ERROR\]' "${logfile}")
if [[ ${count} -gt 0 ]]; then
add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors
with Hadoop v${hadoopver}."
+ add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
((result=result+1))
fi
done
@@ -384,6 +385,7 @@ function hadoopcheck_rebuild
count=$(${GREP} -c '\[ERROR\]' "${logfile}")
if [[ ${count} -gt 0 ]]; then
add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors
with Hadoop v${hadoopver}."
+ add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
((result=result+1))
fi
done
hbase git commit: HBASE-20164 add hadoopcheck link to table footer
Repository: hbase
Updated Branches:
refs/heads/branch-2.0 7176c7bd8 -> 35428dc0a
HBASE-20164 add hadoopcheck link to table footer
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/35428dc0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/35428dc0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/35428dc0
Branch: refs/heads/branch-2.0
Commit: 35428dc0aa317cc4c32db8a128b2194346331873
Parents: 7176c7b
Author: Mike Drob
Authored: Thu Mar 8 21:55:25 2018 -0600
Committer: Mike Drob
Committed: Fri Mar 9 15:41:38 2018 -0600
--
dev-support/hbase-personality.sh | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/35428dc0/dev-support/hbase-personality.sh
--
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index 9064f0b..b2fc76e 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -370,6 +370,7 @@ function hadoopcheck_rebuild
count=$(${GREP} -c '\[ERROR\]' "${logfile}")
if [[ ${count} -gt 0 ]]; then
add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors
with Hadoop v${hadoopver}."
+ add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
((result=result+1))
fi
done
@@ -384,6 +385,7 @@ function hadoopcheck_rebuild
count=$(${GREP} -c '\[ERROR\]' "${logfile}")
if [[ ${count} -gt 0 ]]; then
add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors
with Hadoop v${hadoopver}."
+ add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
((result=result+1))
fi
done
hbase git commit: HBASE-20164 add hadoopcheck link to table footer
Repository: hbase
Updated Branches:
refs/heads/branch-2 55e3dda25 -> 32c5118f6
HBASE-20164 add hadoopcheck link to table footer
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/32c5118f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/32c5118f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/32c5118f
Branch: refs/heads/branch-2
Commit: 32c5118f6f94afe1de323a51801a592031577a5a
Parents: 55e3dda
Author: Mike Drob
Authored: Thu Mar 8 21:55:25 2018 -0600
Committer: Mike Drob
Committed: Fri Mar 9 15:41:12 2018 -0600
--
dev-support/hbase-personality.sh | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/32c5118f/dev-support/hbase-personality.sh
--
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index 9064f0b..b2fc76e 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -370,6 +370,7 @@ function hadoopcheck_rebuild
count=$(${GREP} -c '\[ERROR\]' "${logfile}")
if [[ ${count} -gt 0 ]]; then
add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors
with Hadoop v${hadoopver}."
+ add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
((result=result+1))
fi
done
@@ -384,6 +385,7 @@ function hadoopcheck_rebuild
count=$(${GREP} -c '\[ERROR\]' "${logfile}")
if [[ ${count} -gt 0 ]]; then
add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors
with Hadoop v${hadoopver}."
+ add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
((result=result+1))
fi
done
hbase git commit: HBASE-20164 add hadoopcheck link to table footer
Repository: hbase
Updated Branches:
refs/heads/master 974200fca -> 28279697c
HBASE-20164 add hadoopcheck link to table footer
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/28279697
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/28279697
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/28279697
Branch: refs/heads/master
Commit: 28279697c27acd3c2cd311867a98c9cb2151defe
Parents: 974200f
Author: Mike Drob
Authored: Thu Mar 8 21:55:25 2018 -0600
Committer: Mike Drob
Committed: Fri Mar 9 15:40:28 2018 -0600
--
dev-support/hbase-personality.sh | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/28279697/dev-support/hbase-personality.sh
--
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index 9064f0b..b2fc76e 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -370,6 +370,7 @@ function hadoopcheck_rebuild
count=$(${GREP} -c '\[ERROR\]' "${logfile}")
if [[ ${count} -gt 0 ]]; then
add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors
with Hadoop v${hadoopver}."
+ add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
((result=result+1))
fi
done
@@ -384,6 +385,7 @@ function hadoopcheck_rebuild
count=$(${GREP} -c '\[ERROR\]' "${logfile}")
if [[ ${count} -gt 0 ]]; then
add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors
with Hadoop v${hadoopver}."
+ add_footer_table hadoopcheck "@@BASE@@/patch-javac-${hadoopver}.txt"
((result=result+1))
fi
done
hbase git commit: HBASE-20024 Fixed flakyness of TestMergeTableRegionsProcedure
Repository: hbase
Updated Branches:
refs/heads/master ee3a46964 -> 974200fca
HBASE-20024 Fixed flakyness of TestMergeTableRegionsProcedure
We assumed that we can run for loop from 0 to lastStep sequentially.
MergeTableRegionProcedure skips step 2. So, when i is 0 the procedure is
already at step 3.
Added a method StateMachineProcedure#getCurrentStateId that can be used from
test code only.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/974200fc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/974200fc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/974200fc
Branch: refs/heads/master
Commit: 974200fca1b3643f1f40bbc3df17846a568208ee
Parents: ee3a469
Author: Umesh Agashe
Authored: Tue Feb 27 08:25:37 2018 -0800
Committer: Michael Stack
Committed: Fri Mar 9 12:45:39 2018 -0800
--
.../hbase/procedure2/StateMachineProcedure.java | 13 +++
.../MasterProcedureTestingUtility.java | 23 ++--
2 files changed, 34 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/974200fc/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
index 0880238..222ae93 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
@@ -28,6 +28,9 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.StateMachineProcedureData;
/**
@@ -254,6 +257,16 @@ public abstract class StateMachineProcedure
}
/**
+ * This method is used from test code as it cannot be assumed that state
transition will happen
+ * sequentially. Some procedures may skip steps/ states, some may add
intermediate steps in
+ * future.
+ */
+ @VisibleForTesting
+ public int getCurrentStateId() {
+return getStateId(getCurrentState());
+ }
+
+ /**
* Set the next state for the procedure.
* @param stateId the ordinal() of the state enum (or state id)
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/974200fc/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
index 84b0094..9546b19 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
@@ -54,8 +54,10 @@ import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.master.TableStateManager;
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.MD5Hash;
@@ -377,11 +379,28 @@ public class MasterProcedureTestingUtility {
// execute step N - kill before store update
// restart executor/store
// execute step N - save on store
-for (int i = 0; i < numSteps; ++i) {
- LOG.info("Restart " + i + " exec state=" +
procExec.getProcedure(procId));
+// NOTE: currently we make assumption that states/ steps are sequential.
There are already
+// instances of a procedures which skip (don't use) intermediate states/
steps. In future,
+// intermediate states/ steps can be added with ordinal greater than
lastStep. If and when
+// that happens the states can not be treated as sequential steps and the
condition in
+// following while loop needs to be changed. We can use euqals/ not equals
operator to check
+// if the procedure has reached the user specified state. But
hbase git commit: HBASE-20024 Fixed flakyness of TestMergeTableRegionsProcedure
Repository: hbase
Updated Branches:
refs/heads/branch-2.0 a508f6428 -> 7176c7bd8
HBASE-20024 Fixed flakyness of TestMergeTableRegionsProcedure
We assumed that we can run for loop from 0 to lastStep sequentially.
MergeTableRegionProcedure skips step 2. So, when i is 0 the procedure is
already at step 3.
Added a method StateMachineProcedure#getCurrentStateId that can be used from
test code only.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7176c7bd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7176c7bd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7176c7bd
Branch: refs/heads/branch-2.0
Commit: 7176c7bd828679b112f42c446f7bb990cbf5040d
Parents: a508f64
Author: Umesh Agashe
Authored: Tue Feb 27 08:25:37 2018 -0800
Committer: Michael Stack
Committed: Fri Mar 9 12:45:21 2018 -0800
--
.../hbase/procedure2/StateMachineProcedure.java | 13 +++
.../MasterProcedureTestingUtility.java | 23 ++--
2 files changed, 34 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/7176c7bd/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
index 0880238..222ae93 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
@@ -28,6 +28,9 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.StateMachineProcedureData;
/**
@@ -254,6 +257,16 @@ public abstract class StateMachineProcedure
}
/**
+ * This method is used from test code as it cannot be assumed that state
transition will happen
+ * sequentially. Some procedures may skip steps/ states, some may add
intermediate steps in
+ * future.
+ */
+ @VisibleForTesting
+ public int getCurrentStateId() {
+return getStateId(getCurrentState());
+ }
+
+ /**
* Set the next state for the procedure.
* @param stateId the ordinal() of the state enum (or state id)
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/7176c7bd/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
index 84b0094..9546b19 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
@@ -54,8 +54,10 @@ import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.master.TableStateManager;
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.MD5Hash;
@@ -377,11 +379,28 @@ public class MasterProcedureTestingUtility {
// execute step N - kill before store update
// restart executor/store
// execute step N - save on store
-for (int i = 0; i < numSteps; ++i) {
- LOG.info("Restart " + i + " exec state=" +
procExec.getProcedure(procId));
+// NOTE: currently we make assumption that states/ steps are sequential.
There are already
+// instances of a procedures which skip (don't use) intermediate states/
steps. In future,
+// intermediate states/ steps can be added with ordinal greater than
lastStep. If and when
+// that happens the states can not be treated as sequential steps and the
condition in
+// following while loop needs to be changed. We can use euqals/ not equals
operator to check
+// if the procedure has reached the user specified sta
hbase git commit: HBASE-20024 Fixed flakyness of TestMergeTableRegionsProcedure
Repository: hbase
Updated Branches:
refs/heads/branch-2 cdf7be892 -> 55e3dda25
HBASE-20024 Fixed flakyness of TestMergeTableRegionsProcedure
We assumed that we can run for loop from 0 to lastStep sequentially.
MergeTableRegionProcedure skips step 2. So, when i is 0 the procedure is
already at step 3.
Added a method StateMachineProcedure#getCurrentStateId that can be used from
test code only.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/55e3dda2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/55e3dda2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/55e3dda2
Branch: refs/heads/branch-2
Commit: 55e3dda25d5e558442d8e4899010938e63a9d3f7
Parents: cdf7be8
Author: Umesh Agashe
Authored: Tue Feb 27 08:25:37 2018 -0800
Committer: Michael Stack
Committed: Fri Mar 9 12:44:49 2018 -0800
--
.../hbase/procedure2/StateMachineProcedure.java | 13 +++
.../MasterProcedureTestingUtility.java | 23 ++--
2 files changed, 34 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/55e3dda2/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
index 0880238..222ae93 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
@@ -28,6 +28,9 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.StateMachineProcedureData;
/**
@@ -254,6 +257,16 @@ public abstract class StateMachineProcedure
}
/**
+ * This method is used from test code as it cannot be assumed that state
transition will happen
+ * sequentially. Some procedures may skip steps/ states, some may add
intermediate steps in
+ * future.
+ */
+ @VisibleForTesting
+ public int getCurrentStateId() {
+return getStateId(getCurrentState());
+ }
+
+ /**
* Set the next state for the procedure.
* @param stateId the ordinal() of the state enum (or state id)
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/55e3dda2/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
index 84b0094..9546b19 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
@@ -54,8 +54,10 @@ import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.master.TableStateManager;
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.MD5Hash;
@@ -377,11 +379,28 @@ public class MasterProcedureTestingUtility {
// execute step N - kill before store update
// restart executor/store
// execute step N - save on store
-for (int i = 0; i < numSteps; ++i) {
- LOG.info("Restart " + i + " exec state=" +
procExec.getProcedure(procId));
+// NOTE: currently we make assumption that states/ steps are sequential.
There are already
+// instances of a procedures which skip (don't use) intermediate states/
steps. In future,
+// intermediate states/ steps can be added with ordinal greater than
lastStep. If and when
+// that happens the states can not be treated as sequential steps and the
condition in
+// following while loop needs to be changed. We can use euqals/ not equals
operator to check
+// if the procedure has reached the user specified state.
hbase git commit: HBASE-16179 Fix compilation errors when building hbase-spark against Spark 2.0
Repository: hbase
Updated Branches:
refs/heads/master 06550bc93 -> ee3a46964
HBASE-16179 Fix compilation errors when building hbase-spark against Spark 2.0
Signed-off-by: Mike Drob
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ee3a4696
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ee3a4696
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ee3a4696
Branch: refs/heads/master
Commit: ee3a469641ec3e7385571ab4713d559b75bf50eb
Parents: 06550bc
Author: tedyu
Authored: Fri Mar 9 07:58:53 2018 -0800
Committer: tedyu
Committed: Fri Mar 9 08:00:30 2018 -0800
--
hbase-spark-it/pom.xml | 14 ++-
.../spark/IntegrationTestSparkBulkLoad.java | 4 +-
hbase-spark/pom.xml | 8 +-
.../hbase/spark/SparkSQLPushDownFilter.java | 1 +
.../hadoop/hbase/spark/DefaultSource.scala | 1 -
.../hbase/spark/HBaseConnectionCache.scala | 1 -
.../hadoop/hbase/spark/HBaseContext.scala | 4 +-
.../hadoop/hbase/spark/JavaHBaseContext.scala | 16 +--
.../org/apache/hadoop/hbase/spark/Logging.scala | 119 +++
.../apache/hadoop/hbase/spark/NewHBaseRDD.scala | 12 +-
.../spark/datasources/HBaseTableScanRDD.scala | 5 +-
.../spark/datasources/JavaBytesEncoder.scala| 2 +-
.../hbase/spark/datasources/NaiveEncoder.scala | 2 +-
.../hbase/DataTypeParserWrapper.scala | 14 +--
.../datasources/hbase/HBaseTableCatalog.scala | 5 +-
.../spark/sql/datasources/hbase/Utils.scala | 6 +-
.../hadoop/hbase/spark/BulkLoadSuite.scala | 3 +-
.../hadoop/hbase/spark/DefaultSourceSuite.scala | 12 +-
.../spark/DynamicLogicExpressionSuite.scala | 2 +-
.../hadoop/hbase/spark/HBaseCatalogSuite.scala | 2 +-
.../hbase/spark/HBaseConnectionCacheSuite.scala | 2 +-
.../hadoop/hbase/spark/HBaseContextSuite.scala | 3 +-
.../spark/HBaseDStreamFunctionsSuite.scala | 3 +-
.../hbase/spark/HBaseRDDFunctionsSuite.scala| 3 +-
.../hbase/spark/PartitionFilterSuite.scala | 3 +-
25 files changed, 188 insertions(+), 59 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/ee3a4696/hbase-spark-it/pom.xml
--
diff --git a/hbase-spark-it/pom.xml b/hbase-spark-it/pom.xml
index c1c3967..2cac0f9 100644
--- a/hbase-spark-it/pom.xml
+++ b/hbase-spark-it/pom.xml
@@ -33,9 +33,11 @@
-1.6.0
-2.10.4
-2.10
+2.1.1
+
+2.11.8
+2.11
**/Test*.java
**/IntegrationTest*.java
@@ -264,6 +266,12 @@
+ org.scala-lang.modules
+ scala-xml_2.11
+ 1.0.4
+ provided
+
+
org.apache.spark
spark-sql_${scala.binary.version}
${spark.version}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ee3a4696/hbase-spark-it/src/test/java/org/apache/hadoop/hbase/spark/IntegrationTestSparkBulkLoad.java
--
diff --git
a/hbase-spark-it/src/test/java/org/apache/hadoop/hbase/spark/IntegrationTestSparkBulkLoad.java
b/hbase-spark-it/src/test/java/org/apache/hadoop/hbase/spark/IntegrationTestSparkBulkLoad.java
index 1273a51..d13dd17 100644
---
a/hbase-spark-it/src/test/java/org/apache/hadoop/hbase/spark/IntegrationTestSparkBulkLoad.java
+++
b/hbase-spark-it/src/test/java/org/apache/hadoop/hbase/spark/IntegrationTestSparkBulkLoad.java
@@ -319,7 +319,7 @@ public class IntegrationTestSparkBulkLoad extends
IntegrationTestBase {
PairFlatMapFunction,
SparkLinkKey, SparkLinkChain> {
@Override
-public Iterable>
call(Tuple2>
call(Tuple2 v) throws Exception {
Result value = v._2();
long longRk = Bytes.toLong(value.getRow());
@@ -334,7 +334,7 @@ public class IntegrationTestSparkBulkLoad extends
IntegrationTestBase {
new Tuple2<>(new SparkLinkKey(chainId, order), new
SparkLinkChain(longRk, next));
list.add(tuple2);
}
- return list;
+ return list.iterator();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ee3a4696/hbase-spark/pom.xml
--
diff --git a/hbase-spark/pom.xml b/hbase-spark/pom.xml
index a3a41fd..a6a04ad 100644
--- a/hbase-spark/pom.xml
+++ b/hbase-spark/pom.xml
@@ -30,9 +30,11 @@
hbase-spark
Apache HBase - Spark
-1.6.0
-2.10.4
-2.10
+2.1.1
+
+2.11.8
+2.11
${project.basedir}/..
http://git-wip-us.apache.org/repos/asf/hbase/blob/ee3a4696/hbase-spark/src/main/java/org/apache/hadoop/hbase/spark/SparkSQLPushDownFilter.java
--
diff --git
a/hbase-spark/src/main/java/org/apache/hadoop
[2/5] hbase-site git commit: Published site at 756cccecffef3a64ae328022ae47fc1d1087dc6c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/473b2db2/testdevapidocs/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.html -- diff --git a/testdevapidocs/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.html b/testdevapidocs/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.html index 5267421..4e481e4 100644 --- a/testdevapidocs/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.html +++ b/testdevapidocs/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.html @@ -109,7 +109,7 @@ var activeTableTab = "activeTableTab"; -public class TestAssignmentManagerMetrics +public class TestAssignmentManagerMetrics extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true"; title="class or interface in java.lang">Object @@ -135,33 +135,29 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html private static MiniHBaseCluster -cluster +CLUSTER -private static org.apache.hadoop.conf.Configuration -conf - - private static org.slf4j.Logger LOG - + private static org.apache.hadoop.hbase.master.HMaster -master +MASTER - + private static MetricsAssertHelper -metricsHelper +METRICS_HELPER - + private static int -msgInterval +MSG_INTERVAL - + org.junit.rules.TestName name - + private static HBaseTestingUtility TEST_UTIL @@ -237,7 +233,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html CLASS_RULE -public static final HBaseClassTestRule CLASS_RULE +public static final HBaseClassTestRule CLASS_RULE @@ -246,34 +242,34 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html LOG -private static final org.slf4j.Logger LOG +private static final org.slf4j.Logger LOG - + -metricsHelper -private static final MetricsAssertHelper metricsHelper +METRICS_HELPER +private static final MetricsAssertHelper METRICS_HELPER - + -cluster -private static MiniHBaseCluster cluster +CLUSTER +private static MiniHBaseCluster CLUSTER - + -master -private static org.apache.hadoop.hbase.master.HMaster master +MASTER +private static org.apache.hadoop.hbase.master.HMaster MASTER @@ -282,28 +278,19 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html TEST_UTIL -private static HBaseTestingUtility TEST_UTIL - - - - - - - -conf -private static org.apache.hadoop.conf.Configuration conf +private static HBaseTestingUtility TEST_UTIL - + -msgInterval -private static final int msgInterval +MSG_INTERVAL +private static final int MSG_INTERVAL See Also: -Constant Field Values +Constant Field Values @@ -330,7 +317,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html TestAssignmentManagerMetrics -public TestAssignmentManagerMetrics() +public TestAssignmentManagerMetrics() @@ -361,7 +348,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html after -public static void after() +public static void after() throws https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true"; title="class or interface in java.lang">Exception Throws: @@ -375,7 +362,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html testRITAssignmentManagerMetrics -public void testRITAssignmentManagerMetrics() +public void testRITAssignmentManagerMetrics() throws https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true"; title="class or interface in java.lang">Exception Throws: http://git-wip-us.apache.org/repos/asf/hbase-site/blob/473b2db2/testdevapidocs/org/apache/hadoop/hbase/master/TestRestartCluster.html -- diff --git a/testdevapidocs/org/apache/hadoop/hbase/master/TestRestartCluster.html b/testdevapidocs/org/apache/hadoop/hbase/master/TestRestartCluster.html index 6a82ead..5bc1f9e 100644 --- a/testdevapidocs/org/apache/hadoop/hbase/master/TestRestartCluster.html +++ b/testdevapidocs/org/apache/hadoop/hbase/master/TestRestartCluster.html @@ -109,7 +109,7 @@ var activeTableTab = "activeTableTab"; -public class TestRestartCluster +public class TestRestartCluster extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true"; title="class or interface in java.lang">Object @@ -223,7 +223,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html CLASS_RULE -public static final HBaseClassTestRule CLASS_RULE +public static final HBaseClassTestRule CLASS_RULE @@ -232,7 +232,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html LOG -private static final org.slf4j.Logger LOG +private static final org.slf4j.Logger LOG @@ -241,7 +241,7 @@ extends https://docs.oracl
[1/5] hbase-site git commit: Published site at 756cccecffef3a64ae328022ae47fc1d1087dc6c.
Repository: hbase-site
Updated Branches:
refs/heads/asf-site 5f66dfcf3 -> 473b2db23
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/473b2db2/testdevapidocs/src-html/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.html
--
diff --git
a/testdevapidocs/src-html/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.html
b/testdevapidocs/src-html/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.html
index 90c268c..64adee1 100644
---
a/testdevapidocs/src-html/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.html
+++
b/testdevapidocs/src-html/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.html
@@ -40,36 +40,36 @@
032import
org.apache.hadoop.hbase.client.Table;
033import
org.apache.hadoop.hbase.client.TableDescriptor;
034import
org.apache.hadoop.hbase.client.TableDescriptorBuilder;
-035import
org.apache.hadoop.hbase.test.MetricsAssertHelper;
-036import
org.apache.hadoop.hbase.testclassification.MasterTests;
-037import
org.apache.hadoop.hbase.testclassification.MediumTests;
-038import
org.apache.hadoop.hbase.util.Bytes;
-039import org.junit.AfterClass;
-040import org.junit.BeforeClass;
-041import org.junit.ClassRule;
-042import org.junit.Rule;
-043import org.junit.Test;
-044import
org.junit.experimental.categories.Category;
-045import org.junit.rules.TestName;
-046import org.slf4j.Logger;
-047import org.slf4j.LoggerFactory;
-048
-049@Category({ MasterTests.class,
MediumTests.class })
-050public class TestAssignmentManagerMetrics
{
-051
-052 @ClassRule
-053 public static final HBaseClassTestRule
CLASS_RULE =
-054
HBaseClassTestRule.forClass(TestAssignmentManagerMetrics.class);
-055
-056 private static final Logger LOG =
LoggerFactory.getLogger(TestAssignmentManagerMetrics.class);
-057 private static final
MetricsAssertHelper metricsHelper = CompatibilityFactory
-058
.getInstance(MetricsAssertHelper.class);
-059
-060 private static MiniHBaseCluster
cluster;
-061 private static HMaster master;
-062 private static HBaseTestingUtility
TEST_UTIL = new HBaseTestingUtility();
-063 private static Configuration conf;
-064 private static final int msgInterval =
1000;
+035import
org.apache.hadoop.hbase.master.assignment.AssignmentManager;
+036import
org.apache.hadoop.hbase.test.MetricsAssertHelper;
+037import
org.apache.hadoop.hbase.testclassification.MasterTests;
+038import
org.apache.hadoop.hbase.testclassification.MediumTests;
+039import
org.apache.hadoop.hbase.util.Bytes;
+040import org.junit.AfterClass;
+041import org.junit.BeforeClass;
+042import org.junit.ClassRule;
+043import org.junit.Rule;
+044import org.junit.Test;
+045import
org.junit.experimental.categories.Category;
+046import org.junit.rules.TestName;
+047import org.slf4j.Logger;
+048import org.slf4j.LoggerFactory;
+049
+050@Category({ MasterTests.class,
MediumTests.class })
+051public class TestAssignmentManagerMetrics
{
+052
+053 @ClassRule
+054 public static final HBaseClassTestRule
CLASS_RULE =
+055
HBaseClassTestRule.forClass(TestAssignmentManagerMetrics.class);
+056
+057 private static final Logger LOG =
LoggerFactory.getLogger(TestAssignmentManagerMetrics.class);
+058 private static final
MetricsAssertHelper METRICS_HELPER = CompatibilityFactory
+059
.getInstance(MetricsAssertHelper.class);
+060
+061 private static MiniHBaseCluster
CLUSTER;
+062 private static HMaster MASTER;
+063 private static HBaseTestingUtility
TEST_UTIL = new HBaseTestingUtility();
+064 private static final int MSG_INTERVAL =
1000;
065
066 @Rule
067 public TestName name = new
TestName();
@@ -77,7 +77,7 @@
069 @BeforeClass
070 public static void startCluster()
throws Exception {
071LOG.info("Starting cluster");
-072conf =
TEST_UTIL.getConfiguration();
+072Configuration conf =
TEST_UTIL.getConfiguration();
073
074// Disable sanity check for
coprocessor
075
conf.setBoolean("hbase.table.sanity.checks", false);
@@ -86,74 +86,76 @@
078
conf.setInt(HConstants.METRICS_RIT_STUCK_WARNING_THRESHOLD, 20);
079
080// set msgInterval to 1 second
-081
conf.setInt("hbase.regionserver.msginterval", msgInterval);
+081
conf.setInt("hbase.regionserver.msginterval", MSG_INTERVAL);
082
083// set tablesOnMaster to none
084
conf.set("hbase.balancer.tablesOnMaster", "none");
085
086// set client sync wait timeout to
5sec
-087
conf.setInt("hbase.client.sync.wait.timeout.msec", 2500);
+087
conf.setInt("hbase.client.sync.wait.timeout.msec", 5000);
088
conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
089
conf.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 2500);
-090
-091TEST_UTIL.startMiniCluster(1);
-092cluster =
TEST_UTIL.getHBaseCluster();
-093master = cluster.getMaster();
-094 }
-095
-096 @AfterClass
-097 public static void after() throws
Exception {
-098TEST_UTIL.shutdownMiniClus
[3/5] hbase-site git commit: Published site at 756cccecffef3a64ae328022ae47fc1d1087dc6c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/473b2db2/checkstyle.rss -- diff --git a/checkstyle.rss b/checkstyle.rss index af453d5..5fbc298 100644 --- a/checkstyle.rss +++ b/checkstyle.rss @@ -26,7 +26,7 @@ under the License. ©2007 - 2018 The Apache Software Foundation File: 3580, - Errors: 16177, + Errors: 16175, Warnings: 0, Infos: 0 @@ -22511,7 +22511,7 @@ under the License. 0 - 2 + 0 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/473b2db2/coc.html -- diff --git a/coc.html b/coc.html index 3500ece..7a677b0 100644 --- a/coc.html +++ b/coc.html @@ -7,7 +7,7 @@ - + Apache HBase – Code of Conduct Policy @@ -368,7 +368,7 @@ email to mailto:priv...@hbase.apache.org";>the priv https://www.apache.org/";>The Apache Software Foundation. All rights reserved. - Last Published: 2018-03-08 + Last Published: 2018-03-09 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/473b2db2/cygwin.html -- diff --git a/cygwin.html b/cygwin.html index 9942bdb..67f4d4c 100644 --- a/cygwin.html +++ b/cygwin.html @@ -7,7 +7,7 @@ - + Apache HBase – Installing Apache HBase (TM) on Windows using Cygwin @@ -667,7 +667,7 @@ Now your HBase server is running, start coding and build that next https://www.apache.org/";>The Apache Software Foundation. All rights reserved. - Last Published: 2018-03-08 + Last Published: 2018-03-09 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/473b2db2/dependencies.html -- diff --git a/dependencies.html b/dependencies.html index e0a5554..9c159fa 100644 --- a/dependencies.html +++ b/dependencies.html @@ -7,7 +7,7 @@ - + Apache HBase – Project Dependencies @@ -433,7 +433,7 @@ https://www.apache.org/";>The Apache Software Foundation. All rights reserved. - Last Published: 2018-03-08 + Last Published: 2018-03-09 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/473b2db2/dependency-convergence.html -- diff --git a/dependency-convergence.html b/dependency-convergence.html index 2f21003..a427428 100644 --- a/dependency-convergence.html +++ b/dependency-convergence.html @@ -7,7 +7,7 @@ - + Apache HBase – Reactor Dependency Convergence @@ -1040,7 +1040,7 @@ https://www.apache.org/";>The Apache Software Foundation. All rights reserved. - Last Published: 2018-03-08 + Last Published: 2018-03-09 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/473b2db2/dependency-info.html -- diff --git a/dependency-info.html b/dependency-info.html index 7aac49d..04ec196 100644 --- a/dependency-info.html +++ b/dependency-info.html @@ -7,7 +7,7 @@ - + Apache HBase – Dependency Information @@ -306,7 +306,7 @@ https://www.apache.org/";>The Apache Software Foundation. All rights reserved. - Last Published: 2018-03-08 + Last Published: 2018-03-09 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/473b2db2/dependency-management.html -- diff --git a/dependency-management.html b/dependency-management.html index 1414450..9914d0f 100644 --- a/dependency-management.html +++ b/dependency-management.html @@ -7,7 +7,7 @@ - + Apache HBase – Project Dependency Management @@ -974,7 +974,7 @@ https://www.apache.org/";>The Apache Software Foundation. All rights reserved. - Last Published: 2018-03-08 + Last Published: 2018-03-09 http://git-wip-us.apache.org/repos/asf/hb
[4/5] hbase-site git commit: Published site at 756cccecffef3a64ae328022ae47fc1d1087dc6c.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/473b2db2/checkstyle-aggregate.html -- diff --git a/checkstyle-aggregate.html b/checkstyle-aggregate.html index ceffe62..d92a5c7 100644 --- a/checkstyle-aggregate.html +++ b/checkstyle-aggregate.html @@ -7,7 +7,7 @@ - + Apache HBase – Checkstyle Results @@ -277,7 +277,7 @@ 3580 0 0 -16177 +16175 Files @@ -4977,5301 +4977,5296 @@ 0 2 -org/apache/hadoop/hbase/master/TestRestartCluster.java -0 -0 -2 - org/apache/hadoop/hbase/master/TestRollingRestart.java 0 0 1 - + org/apache/hadoop/hbase/master/TestSplitLogManager.java 0 0 7 - + org/apache/hadoop/hbase/master/TestTableStateManager.java 0 0 2 - + org/apache/hadoop/hbase/master/TestWarmupRegion.java 0 0 19 - + org/apache/hadoop/hbase/master/assignment/AssignProcedure.java 0 0 4 - + org/apache/hadoop/hbase/master/assignment/AssignmentManager.java 0 0 30 - + org/apache/hadoop/hbase/master/assignment/GCMergedRegionsProcedure.java 0 0 14 - + org/apache/hadoop/hbase/master/assignment/GCRegionProcedure.java 0 0 31 - + org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java 0 0 4 - + org/apache/hadoop/hbase/master/assignment/MockMasterServices.java 0 0 2 - + org/apache/hadoop/hbase/master/assignment/RegionStateStore.java 0 0 6 - + org/apache/hadoop/hbase/master/assignment/RegionStates.java 0 0 25 - + org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java 0 0 5 - + org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java 0 0 7 - + org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java 0 0 15 - + org/apache/hadoop/hbase/master/assignment/TestAssignmentOnRSCrash.java 0 0 1 - + org/apache/hadoop/hbase/master/assignment/TestRegionStates.java 0 0 1 - + org/apache/hadoop/hbase/master/assignment/TestSplitTableRegionProcedure.java 0 0 2 - + org/apache/hadoop/hbase/master/assignment/UnassignProcedure.java 0 0 4 - + org/apache/hadoop/hbase/master/assignment/Util.java 0 0 1 - + org/apache/hadoop/hbase/master/balancer/BalancerChore.java 0 0 1 - + org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java 0 0 75 - + org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java 0 0 61 - + org/apache/hadoop/hbase/master/balancer/ClusterLoadState.java 0 0 1 - + org/apache/hadoop/hbase/master/balancer/ClusterStatusChore.java 0 0 1 - + org/apache/hadoop/hbase/master/balancer/FavoredStochasticBalancer.java 0 0 2 - + org/apache/hadoop/hbase/master/balancer/LoadBalancerFactory.java 0 0 3 - + org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java 0 0 1 - + org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java 0 0 2 - + org/apache/hadoop/hbase/master/balancer/MetricsStochasticBalancerSourceImpl.java 0 0 7 - + org/apache/hadoop/hbase/master/balancer/RegionInfoComparator.java 0 0 2 - + org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java 0 0 3 - + org/apache/hadoop/hbase/master/balancer/ServerAndLoad.java 0 0 1 - + org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java 0 0 33 - + org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java 0 0 32 - + org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java 0 0 16 - + org/apache/hadoop/hbase/master/balancer/TestDefaultLoadBalancer.java 0 0 48 - + org/apache/hadoop/hbase/master/balancer/TestFavoredNodeTableImport.java 0 0 1 - + org/apache/hadoop/hbase/master/balancer/TestFavoredStochasticBalancerPickers.java 0 0 2 - + org/apache/hadoop/hbase/master/balancer/TestRegionsOnMasterOptions.java 0 0 13 - + org/apache/hadoop/hbase/master/balancer/TestServerAndLoad.java 0 0 1 - + org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java 0 0 33 - + org/apache/hadoop/hbase/master/cleaner/BaseFileCleanerDelegate.java 0 0 2 - + org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java 0 0 1 - + org/apache/hadoop/hbase/master/cleaner/CleanerChore.java 0 0 4 - + org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java 0 0 2 - + org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java 0 0 3 - + org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java 0 0 4 - + org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java 0 0 2 - + org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java 0 0 2 - + org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java 0 0 1 - + org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java 0 0 1 - + org/apache/hadoop/hbase/master/locking/LockManager.java 0 0 2 - + org/apache/hadoop/hbase/master/locking/LockProcedure.java 0 0 4 - + org/apache/hadoop/hbase/master/normalizer/EmptyNormalizationPlan.java 0 0 1 - + org/apache/hadoop/hba
[5/5] hbase-site git commit: Published site at 756cccecffef3a64ae328022ae47fc1d1087dc6c.
Published site at 756cccecffef3a64ae328022ae47fc1d1087dc6c. Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/473b2db2 Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/473b2db2 Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/473b2db2 Branch: refs/heads/asf-site Commit: 473b2db2351cb54df2ed6fc765ee70aec443a3e3 Parents: 5f66dfc Author: jenkins Authored: Fri Mar 9 14:53:36 2018 + Committer: jenkins Committed: Fri Mar 9 14:53:36 2018 + -- acid-semantics.html | 4 +- apache_hbase_reference_guide.pdf| 4 +- book.html | 2 +- bulk-loads.html | 4 +- checkstyle-aggregate.html | 20014 - checkstyle.rss | 4 +- coc.html| 4 +- cygwin.html | 4 +- dependencies.html | 4 +- dependency-convergence.html | 4 +- dependency-info.html| 4 +- dependency-management.html | 4 +- devapidocs/constant-values.html | 4 +- .../org/apache/hadoop/hbase/Version.html| 4 +- export_control.html | 4 +- index.html | 4 +- integration.html| 4 +- issue-tracking.html | 4 +- license.html| 4 +- mail-lists.html | 4 +- metrics.html| 4 +- old_news.html | 4 +- plugin-management.html | 4 +- plugins.html| 4 +- poweredbyhbase.html | 4 +- project-info.html | 4 +- project-reports.html| 4 +- project-summary.html| 4 +- pseudo-distributed.html | 4 +- replication.html| 4 +- resources.html | 4 +- source-repository.html | 4 +- sponsors.html | 4 +- supportingprojects.html | 4 +- team-list.html | 4 +- testdevapidocs/constant-values.html | 4 +- testdevapidocs/index-all.html |24 +- .../hadoop/hbase/backup/package-tree.html | 2 +- .../hbase/class-use/MiniHBaseCluster.html | 4 +- .../hadoop/hbase/io/hfile/package-tree.html | 2 +- .../hbase/mapreduce/TestWALRecordReader.html| 108 +- .../master/TestAssignmentManagerMetrics.html|71 +- .../hadoop/hbase/master/TestRestartCluster.html |20 +- .../org/apache/hadoop/hbase/package-tree.html | 8 +- .../hadoop/hbase/regionserver/package-tree.html | 6 +- .../test/class-use/MetricsAssertHelper.html | 2 +- .../apache/hadoop/hbase/test/package-tree.html | 4 +- .../apache/hadoop/hbase/wal/package-tree.html | 4 +- .../hbase/mapreduce/TestWALRecordReader.html| 535 +- .../master/TestAssignmentManagerMetrics.html| 184 +- .../hadoop/hbase/master/TestRestartCluster.html | 362 +- 51 files changed, 10792 insertions(+), 10696 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase-site/blob/473b2db2/acid-semantics.html -- diff --git a/acid-semantics.html b/acid-semantics.html index 8b40a13..971e3f3 100644 --- a/acid-semantics.html +++ b/acid-semantics.html @@ -7,7 +7,7 @@ - + Apache HBase – Apache HBase (TM) ACID Properties @@ -594,7 +594,7 @@ under the License. --> https://www.apache.org/";>The Apache Software Foundation. All rights reserved. - Last Published: 2018-03-08 + Last Published: 2018-03-09 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/473b2db2/apache_hbase_reference_guide.pdf -- diff --git a/apache_hbase_reference_guide.pdf b/apache_hbase_reference_guide.pdf index d26b15d..b32a264 100644 --- a/apache_hbase_reference_guide.pdf +++ b/apache_hbase_reference_guide.pdf @@ -5,8 +5,8 @@ /Author (Apache HBase Team) /Creator (Asciidoctor PDF 1.5.0.alpha.15,
hbase-site git commit: INFRA-10751 Empty commit
Repository: hbase-site Updated Branches: refs/heads/asf-site 473b2db23 -> c0fe6722c INFRA-10751 Empty commit Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/c0fe6722 Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/c0fe6722 Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/c0fe6722 Branch: refs/heads/asf-site Commit: c0fe6722c3ed025a2fae60b27b6321e899e3f5a2 Parents: 473b2db Author: jenkins Authored: Fri Mar 9 14:53:50 2018 + Committer: jenkins Committed: Fri Mar 9 14:53:50 2018 + -- --
hbase git commit: HBASE-20146 Regions are stuck while opening when WAL is disabled
Repository: hbase
Updated Branches:
refs/heads/branch-1.2 f3660b010 -> ddaea95d2
HBASE-20146 Regions are stuck while opening when WAL is disabled
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ddaea95d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ddaea95d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ddaea95d
Branch: refs/heads/branch-1.2
Commit: ddaea95d230eacb060eb9c86f9a47d6a73f5fe8c
Parents: f3660b0
Author: Ashish Singhi
Authored: Thu Mar 8 18:47:16 2018 +0530
Committer: zhangduo
Committed: Fri Mar 9 21:33:15 2018 +0800
--
.../hadoop/hbase/wal/DisabledWALProvider.java | 5 +-
.../hadoop/hbase/wal/TestDisabledWAL.java | 103 +++
2 files changed, 105 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddaea95d/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
index 33785a6..7f4fb965 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
@@ -22,8 +22,6 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -155,7 +153,8 @@ class DisabledWALProvider implements WALProvider {
@Override
public long append(HTableDescriptor htd, HRegionInfo info, WALKey key,
WALEdit edits,
- boolean inMemstore) {
+boolean inMemstore) throws IOException {
+ key.setWriteEntry(key.getMvcc().begin());
if (!this.listeners.isEmpty()) {
final long start = System.nanoTime();
long len = 0;
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddaea95d/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
new file mode 100644
index 000..d87bdea
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.wal;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+@Category({ RegionServerTests.class, MediumTests.class })
+public class TestDisabledWAL {
+
+ @Rule
+ public TestName name = new TestName();
+
+ private static final Log LOG = LogFactory.getLog(TestDisabledWAL.class);
+ static final HBaseTestingUtility TEST_UTIL = new HBaseTest
hbase git commit: HBASE-20146 Regions are stuck while opening when WAL is disabled
Repository: hbase
Updated Branches:
refs/heads/branch-1.3 35bd1580d -> 4768ca9b9
HBASE-20146 Regions are stuck while opening when WAL is disabled
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4768ca9b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4768ca9b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4768ca9b
Branch: refs/heads/branch-1.3
Commit: 4768ca9b90acc78f89b50cade81cc0d8dd4680da
Parents: 35bd158
Author: Ashish Singhi
Authored: Thu Mar 8 18:47:16 2018 +0530
Committer: zhangduo
Committed: Fri Mar 9 21:27:58 2018 +0800
--
.../hadoop/hbase/wal/DisabledWALProvider.java | 3 +-
.../hadoop/hbase/wal/TestDisabledWAL.java | 103 +++
2 files changed, 104 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/4768ca9b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
index 1816238..a8756b3 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
@@ -22,8 +22,6 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -156,6 +154,7 @@ class DisabledWALProvider implements WALProvider {
@Override
public long append(HTableDescriptor htd, HRegionInfo info, WALKey key,
WALEdit edits,
boolean inMemstore) throws IOException {
+ key.setWriteEntry(key.getMvcc().begin());
if (!this.listeners.isEmpty()) {
final long start = System.nanoTime();
long len = 0;
http://git-wip-us.apache.org/repos/asf/hbase/blob/4768ca9b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
new file mode 100644
index 000..d87bdea
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.wal;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+@Category({ RegionServerTests.class, MediumTests.class })
+public class TestDisabledWAL {
+
+ @Rule
+ public TestName name = new TestName();
+
+ private static final Log LOG = LogFactory.getLog(TestDisabledWAL.class);
+ static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private Table table;
+ private
hbase git commit: HBASE-20146 Regions are stuck while opening when WAL is disabled
Repository: hbase
Updated Branches:
refs/heads/branch-1 13519b8d6 -> e81abb725
HBASE-20146 Regions are stuck while opening when WAL is disabled
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e81abb72
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e81abb72
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e81abb72
Branch: refs/heads/branch-1
Commit: e81abb7259a5121df9c94758d2b8b4bd173e7ba4
Parents: 13519b8
Author: Ashish Singhi
Authored: Thu Mar 8 18:47:16 2018 +0530
Committer: zhangduo
Committed: Fri Mar 9 21:27:00 2018 +0800
--
.../hadoop/hbase/wal/DisabledWALProvider.java | 3 +-
.../hadoop/hbase/wal/TestDisabledWAL.java | 103 +++
2 files changed, 104 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/e81abb72/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
index 0c0f6dd..30ea651 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
@@ -23,8 +23,6 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -164,6 +162,7 @@ class DisabledWALProvider implements WALProvider {
@Override
public long append(HTableDescriptor htd, HRegionInfo info, WALKey key,
WALEdit edits,
boolean inMemstore) throws IOException {
+ key.setWriteEntry(key.getMvcc().begin());
if (!this.listeners.isEmpty()) {
final long start = System.nanoTime();
long len = 0;
http://git-wip-us.apache.org/repos/asf/hbase/blob/e81abb72/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
new file mode 100644
index 000..d87bdea
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.wal;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+@Category({ RegionServerTests.class, MediumTests.class })
+public class TestDisabledWAL {
+
+ @Rule
+ public TestName name = new TestName();
+
+ private static final Log LOG = LogFactory.getLog(TestDisabledWAL.class);
+ static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private Table table;
+ private Tabl
hbase git commit: HBASE-20146 Regions are stuck while opening when WAL is disabled
Repository: hbase
Updated Branches:
refs/heads/branch-1.4 976a24092 -> a8228eaad
HBASE-20146 Regions are stuck while opening when WAL is disabled
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a8228eaa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a8228eaa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a8228eaa
Branch: refs/heads/branch-1.4
Commit: a8228eaadb00b4ea93df30d3add5a32a414a38f3
Parents: 976a240
Author: Ashish Singhi
Authored: Thu Mar 8 18:47:16 2018 +0530
Committer: zhangduo
Committed: Fri Mar 9 21:27:37 2018 +0800
--
.../hadoop/hbase/wal/DisabledWALProvider.java | 3 +-
.../hadoop/hbase/wal/TestDisabledWAL.java | 103 +++
2 files changed, 104 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a8228eaa/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
index 0c0f6dd..30ea651 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
@@ -23,8 +23,6 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -164,6 +162,7 @@ class DisabledWALProvider implements WALProvider {
@Override
public long append(HTableDescriptor htd, HRegionInfo info, WALKey key,
WALEdit edits,
boolean inMemstore) throws IOException {
+ key.setWriteEntry(key.getMvcc().begin());
if (!this.listeners.isEmpty()) {
final long start = System.nanoTime();
long len = 0;
http://git-wip-us.apache.org/repos/asf/hbase/blob/a8228eaa/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
new file mode 100644
index 000..d87bdea
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.wal;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+@Category({ RegionServerTests.class, MediumTests.class })
+public class TestDisabledWAL {
+
+ @Rule
+ public TestName name = new TestName();
+
+ private static final Log LOG = LogFactory.getLog(TestDisabledWAL.class);
+ static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private Table table;
+ private
hbase git commit: HBASE-20146 Regions are stuck while opening when WAL is disabled
Repository: hbase
Updated Branches:
refs/heads/branch-2.0 d2e5e11f6 -> a508f6428
HBASE-20146 Regions are stuck while opening when WAL is disabled
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a508f642
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a508f642
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a508f642
Branch: refs/heads/branch-2.0
Commit: a508f642870056fb3463f2c395e0a213111443f6
Parents: d2e5e11
Author: Ashish Singhi
Authored: Thu Mar 8 18:47:16 2018 +0530
Committer: zhangduo
Committed: Fri Mar 9 21:14:48 2018 +0800
--
.../hadoop/hbase/wal/DisabledWALProvider.java | 1 +
.../hadoop/hbase/wal/TestDisabledWAL.java | 110 +++
2 files changed, 111 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a508f642/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
index 1e750e2..f858054 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
@@ -162,6 +162,7 @@ class DisabledWALProvider implements WALProvider {
@Override
public long append(RegionInfo info, WALKeyImpl key, WALEdit edits, boolean
inMemstore)
throws IOException {
+ key.setWriteEntry(key.getMvcc().begin());
if (!this.listeners.isEmpty()) {
final long start = System.nanoTime();
long len = 0;
http://git-wip-us.apache.org/repos/asf/hbase/blob/a508f642/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
new file mode 100644
index 000..bd59ce9
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.wal;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+@Category({ RegionServerTests.class, MediumTests.class })
+public class TestDisabledWAL {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestDisabledWAL.class);
+
+ @Rule
+ public TestName name = new TestName();
+
+ private static final Log LOG = LogFactory.getLog(TestDisabledWAL.class);
+ static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private Table table;
+ private TableName tableName;
+ private byte[] fam = Bytes.toBytes("f1");
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+Configuration conf = TEST_UT
hbase git commit: HBASE-20146 Regions are stuck while opening when WAL is disabled
Repository: hbase
Updated Branches:
refs/heads/branch-2 d4e0fc74d -> cdf7be892
HBASE-20146 Regions are stuck while opening when WAL is disabled
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cdf7be89
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cdf7be89
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cdf7be89
Branch: refs/heads/branch-2
Commit: cdf7be89291eb4260db3dcbd3ed5bf4231adecf7
Parents: d4e0fc7
Author: Ashish Singhi
Authored: Thu Mar 8 18:47:16 2018 +0530
Committer: zhangduo
Committed: Fri Mar 9 21:14:42 2018 +0800
--
.../hadoop/hbase/wal/DisabledWALProvider.java | 1 +
.../hadoop/hbase/wal/TestDisabledWAL.java | 110 +++
2 files changed, 111 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/cdf7be89/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
index 1e750e2..f858054 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
@@ -162,6 +162,7 @@ class DisabledWALProvider implements WALProvider {
@Override
public long append(RegionInfo info, WALKeyImpl key, WALEdit edits, boolean
inMemstore)
throws IOException {
+ key.setWriteEntry(key.getMvcc().begin());
if (!this.listeners.isEmpty()) {
final long start = System.nanoTime();
long len = 0;
http://git-wip-us.apache.org/repos/asf/hbase/blob/cdf7be89/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
new file mode 100644
index 000..bd59ce9
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.wal;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+@Category({ RegionServerTests.class, MediumTests.class })
+public class TestDisabledWAL {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestDisabledWAL.class);
+
+ @Rule
+ public TestName name = new TestName();
+
+ private static final Log LOG = LogFactory.getLog(TestDisabledWAL.class);
+ static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private Table table;
+ private TableName tableName;
+ private byte[] fam = Bytes.toBytes("f1");
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+Configuration conf = TEST_UTIL.g
hbase git commit: HBASE-20146 Regions are stuck while opening when WAL is disabled
Repository: hbase
Updated Branches:
refs/heads/master 756cccecf -> 06550bc93
HBASE-20146 Regions are stuck while opening when WAL is disabled
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/06550bc9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/06550bc9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/06550bc9
Branch: refs/heads/master
Commit: 06550bc93b2ed0f39086511cc8dfb1ca279f0d44
Parents: 756ccce
Author: Ashish Singhi
Authored: Thu Mar 8 18:47:16 2018 +0530
Committer: zhangduo
Committed: Fri Mar 9 21:10:16 2018 +0800
--
.../hadoop/hbase/wal/DisabledWALProvider.java | 1 +
.../hadoop/hbase/wal/TestDisabledWAL.java | 110 +++
2 files changed, 111 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/06550bc9/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
index 1e750e2..f858054 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
@@ -162,6 +162,7 @@ class DisabledWALProvider implements WALProvider {
@Override
public long append(RegionInfo info, WALKeyImpl key, WALEdit edits, boolean
inMemstore)
throws IOException {
+ key.setWriteEntry(key.getMvcc().begin());
if (!this.listeners.isEmpty()) {
final long start = System.nanoTime();
long len = 0;
http://git-wip-us.apache.org/repos/asf/hbase/blob/06550bc9/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
new file mode 100644
index 000..bd59ce9
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDisabledWAL.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.wal;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+@Category({ RegionServerTests.class, MediumTests.class })
+public class TestDisabledWAL {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestDisabledWAL.class);
+
+ @Rule
+ public TestName name = new TestName();
+
+ private static final Log LOG = LogFactory.getLog(TestDisabledWAL.class);
+ static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private Table table;
+ private TableName tableName;
+ private byte[] fam = Bytes.toBytes("f1");
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+Configuration conf = TEST_UTIL.getCo
[13/40] hbase git commit: HBASE-19543 Abstract a replication storage interface to extract the zk specific code
HBASE-19543 Abstract a replication storage interface to extract the zk specific
code
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1de08ded
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1de08ded
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1de08ded
Branch: refs/heads/branch-2
Commit: 1de08ded58780ba767f2596c50081f5a93a13e5f
Parents: 3fd4176
Author: zhangduo
Authored: Fri Dec 22 14:37:28 2017 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../hadoop/hbase/util/CollectionUtils.java | 3 +
hbase-replication/pom.xml | 12 +
.../replication/ReplicationPeerStorage.java | 74
.../replication/ReplicationQueueStorage.java| 164 +++
.../replication/ReplicationStateZKBase.java | 1 -
.../replication/ReplicationStorageFactory.java | 49 +++
.../replication/ZKReplicationPeerStorage.java | 164 +++
.../replication/ZKReplicationQueueStorage.java | 425 +++
.../replication/ZKReplicationStorageBase.java | 75
.../TestZKReplicationPeerStorage.java | 171
.../TestZKReplicationQueueStorage.java | 171
.../org/apache/hadoop/hbase/master/HMaster.java | 34 +-
.../hadoop/hbase/master/MasterServices.java | 6 +-
.../master/procedure/MasterProcedureEnv.java| 24 +-
.../master/replication/AddPeerProcedure.java| 6 +-
.../replication/DisablePeerProcedure.java | 7 +-
.../master/replication/EnablePeerProcedure.java | 6 +-
.../master/replication/ModifyPeerProcedure.java | 41 +-
.../master/replication/RemovePeerProcedure.java | 6 +-
.../master/replication/ReplicationManager.java | 199 -
.../replication/ReplicationPeerManager.java | 331 +++
.../replication/UpdatePeerConfigProcedure.java | 7 +-
.../replication/TestReplicationAdmin.java | 64 ++-
.../hbase/master/MockNoopMasterServices.java| 13 +-
.../hbase/master/TestMasterNoCluster.java | 3 +-
.../TestReplicationDisableInactivePeer.java | 6 +-
26 files changed, 1749 insertions(+), 313 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/1de08ded/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java
index 875b124..8bbb6f1 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java
@@ -107,6 +107,9 @@ public class CollectionUtils {
return list.get(list.size() - 1);
}
+ public static List nullToEmpty(List list) {
+return list != null ? list : Collections.emptyList();
+ }
/**
* In HBASE-16648 we found that ConcurrentHashMap.get is much faster than
computeIfAbsent if the
* value already exists. Notice that the implementation does not guarantee
that the supplier will
http://git-wip-us.apache.org/repos/asf/hbase/blob/1de08ded/hbase-replication/pom.xml
--
diff --git a/hbase-replication/pom.xml b/hbase-replication/pom.xml
index af1145b..4952009 100644
--- a/hbase-replication/pom.xml
+++ b/hbase-replication/pom.xml
@@ -97,6 +97,18 @@
org.apache.hbase
hbase-zookeeper
+
+ org.apache.hbase
+ hbase-common
+ test-jar
+ test
+
+
+ org.apache.hbase
+ hbase-zookeeper
+ test-jar
+ test
+
org.apache.commons
http://git-wip-us.apache.org/repos/asf/hbase/blob/1de08ded/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java
new file mode 100644
index 000..e00cd0d
--- /dev/null
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.
[19/40] hbase git commit: HBASE-19592 Add UTs to test retry on update zk failure
HBASE-19592 Add UTs to test retry on update zk failure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ac07e079
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ac07e079
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ac07e079
Branch: refs/heads/branch-2
Commit: ac07e079883d14053301ffb9ae60b2315b6eca69
Parents: 712b5a8
Author: zhangduo
Authored: Tue Dec 26 20:39:00 2017 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../replication/ReplicationPeerManager.java | 5 +-
.../TestReplicationProcedureRetry.java | 200 +++
2 files changed, 202 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/ac07e079/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index b78cbce..f4ccce8 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -53,7 +53,7 @@ import org.apache.yetus.audience.InterfaceAudience;
* Used to add/remove a replication peer.
*/
@InterfaceAudience.Private
-public final class ReplicationPeerManager {
+public class ReplicationPeerManager {
private final ReplicationPeerStorage peerStorage;
@@ -61,8 +61,7 @@ public final class ReplicationPeerManager {
private final ConcurrentMap peers;
- private ReplicationPeerManager(ReplicationPeerStorage peerStorage,
- ReplicationQueueStorage queueStorage,
+ ReplicationPeerManager(ReplicationPeerStorage peerStorage,
ReplicationQueueStorage queueStorage,
ConcurrentMap peers) {
this.peerStorage = peerStorage;
this.queueStorage = queueStorage;
http://git-wip-us.apache.org/repos/asf/hbase/blob/ac07e079/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
new file mode 100644
index 000..ab35b46
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.spy;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.ReplicationTests;
+import org.apache.zookeeper.KeeperException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.invocation.InvocationOnMock;
+
+/**
+ * All the modification method will fail once in the test and should finally
succeed.
+ */
+@Category({ ReplicationTests.cl
[07/40] hbase git commit: HBASE-19524 Master side changes for moving peer modification from zk watcher to procedure
HBASE-19524 Master side changes for moving peer modification from zk watcher to
procedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5e410d81
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5e410d81
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5e410d81
Branch: refs/heads/branch-2
Commit: 5e410d814004288fe25320d8caf31880903d3e05
Parents: 95af14f
Author: zhangduo
Authored: Mon Dec 18 15:22:36 2017 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../procedure2/RemoteProcedureDispatcher.java | 3 +-
.../src/main/protobuf/MasterProcedure.proto | 21 -
.../src/main/protobuf/RegionServerStatus.proto | 3 +-
.../src/main/protobuf/Replication.proto | 5 +
.../replication/ReplicationPeersZKImpl.java | 4 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 93 ---
.../hadoop/hbase/master/MasterRpcServices.java | 4 +-
.../hadoop/hbase/master/MasterServices.java | 26 +++---
.../assignment/RegionTransitionProcedure.java | 13 +--
.../master/procedure/MasterProcedureEnv.java| 5 +
.../master/procedure/ProcedurePrepareLatch.java | 2 +-
.../master/replication/AddPeerProcedure.java| 97
.../replication/DisablePeerProcedure.java | 70 ++
.../master/replication/EnablePeerProcedure.java | 69 ++
.../master/replication/ModifyPeerProcedure.java | 97 +---
.../master/replication/RefreshPeerCallable.java | 67 --
.../replication/RefreshPeerProcedure.java | 28 --
.../master/replication/RemovePeerProcedure.java | 69 ++
.../master/replication/ReplicationManager.java | 76 ---
.../replication/UpdatePeerConfigProcedure.java | 92 +++
.../hbase/regionserver/HRegionServer.java | 5 +-
.../regionserver/RefreshPeerCallable.java | 70 ++
.../hbase/master/MockNoopMasterServices.java| 23 +++--
.../replication/DummyModifyPeerProcedure.java | 13 ++-
24 files changed, 733 insertions(+), 222 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/5e410d81/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
index 7e3dde6..fb852c3 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
@@ -247,9 +247,8 @@ public abstract class RemoteProcedureDispatcherhttp://git-wip-us.apache.org/repos/asf/hbase/blob/5e410d81/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
index 8aa5c78..a9521ad 100644
--- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
@@ -27,6 +27,7 @@ option optimize_for = SPEED;
import "HBase.proto";
import "RPC.proto";
import "Snapshot.proto";
+import "Replication.proto";
//
// WARNING - Compatibility rules
@@ -370,9 +371,10 @@ message GCMergedRegionsStateData {
}
enum PeerModificationState {
- UPDATE_PEER_STORAGE = 1;
- REFRESH_PEER_ON_RS = 2;
- POST_PEER_MODIFICATION = 3;
+ PRE_PEER_MODIFICATION = 1;
+ UPDATE_PEER_STORAGE = 2;
+ REFRESH_PEER_ON_RS = 3;
+ POST_PEER_MODIFICATION = 4;
}
message PeerModificationStateData {
@@ -397,4 +399,17 @@ message RefreshPeerParameter {
required string peer_id = 1;
required PeerModificationType type = 2;
required ServerName target_server = 3;
+}
+
+message ModifyPeerStateData {
+ required string peer_id = 1;
+}
+
+message AddPeerStateData {
+ required ReplicationPeer peer_config = 1;
+ required bool enabled = 2;
+}
+
+message UpdatePeerConfigStateData {
+ required ReplicationPeer peer_config = 1;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/5e410d81/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
index eb396ac..4f75941 100644
--- a/hbase-protocol-shaded/src/main
[09/40] hbase git commit: HBASE-19580 Use slf4j instead of commons-logging in new, just-added Peer Procedure classes
HBASE-19580 Use slf4j instead of commons-logging in new, just-added Peer
Procedure classes
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/da078709
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/da078709
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/da078709
Branch: refs/heads/branch-2
Commit: da07870995fa45a5faa87c495c705cbc681f
Parents: 9ead593
Author: zhangduo
Authored: Thu Dec 21 21:59:46 2017 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../hadoop/hbase/master/replication/AddPeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/DisablePeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/EnablePeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/ModifyPeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/RefreshPeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/RemovePeerProcedure.java | 6 +++---
.../hbase/master/replication/UpdatePeerConfigProcedure.java| 6 +++---
7 files changed, 21 insertions(+), 21 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/da078709/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
index c3862d8..066c3e7 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
@@ -19,8 +19,6 @@ package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
@@ -28,6 +26,8 @@ import
org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.AddPeerStateData;
@@ -37,7 +37,7 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.A
@InterfaceAudience.Private
public class AddPeerProcedure extends ModifyPeerProcedure {
- private static final Log LOG = LogFactory.getLog(AddPeerProcedure.class);
+ private static final Logger LOG =
LoggerFactory.getLogger(AddPeerProcedure.class);
private ReplicationPeerConfig peerConfig;
http://git-wip-us.apache.org/repos/asf/hbase/blob/da078709/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
index 0b32db9..9a28de6 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
@@ -19,11 +19,11 @@ package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The procedure for disabling a replication peer.
@@ -31,7 +31,7 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public class DisablePeerProcedure extends ModifyPeerProcedure {
- private static final Log LOG = LogFactory.getLog(DisablePeerProcedure.class);
+ private static final Logger LOG =
LoggerFactory.getLogger(DisablePeerProcedure.class);
public DisablePeerProcedure() {
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/da078709/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/ap
[24/40] hbase git commit: HBASE-19622 Reimplement ReplicationPeers with the new replication storage interface
http://git-wip-us.apache.org/repos/asf/hbase/blob/1e36a84a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 5b7bab8..ce9882a 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -160,7 +160,6 @@ public class ReplicationSourceManager implements
ReplicationListener {
this.clusterId = clusterId;
this.walFileLengthProvider = walFileLengthProvider;
this.replicationTracker.registerListener(this);
-this.replicationPeers.getAllPeerIds();
// It's preferable to failover 1 RS at a time, but with good zk servers
// more could be processed at the same time.
int nbWorkers = conf.getInt("replication.executor.workers", 1);
@@ -260,8 +259,8 @@ public class ReplicationSourceManager implements
ReplicationListener {
}
List otherRegionServers =
replicationTracker.getListOfRegionServers().stream()
.map(ServerName::valueOf).collect(Collectors.toList());
-LOG.info(
- "Current list of replicators: " + currentReplicators + " other RSs: " +
otherRegionServers);
+LOG.info("Current list of replicators: " + currentReplicators + " other
RSs: "
++ otherRegionServers);
// Look if there's anything to process after a restart
for (ServerName rs : currentReplicators) {
@@ -278,7 +277,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
* The returned future is for adoptAbandonedQueues task.
*/
Future init() throws IOException, ReplicationException {
-for (String id : this.replicationPeers.getConnectedPeerIds()) {
+for (String id : this.replicationPeers.getAllPeerIds()) {
addSource(id);
if (replicationForBulkLoadDataEnabled) {
// Check if peer exists in hfile-refs queue, if not add it. This can
happen in the case
@@ -297,8 +296,8 @@ public class ReplicationSourceManager implements
ReplicationListener {
*/
@VisibleForTesting
ReplicationSourceInterface addSource(String id) throws IOException,
ReplicationException {
-ReplicationPeerConfig peerConfig =
replicationPeers.getReplicationPeerConfig(id);
-ReplicationPeer peer = replicationPeers.getConnectedPeer(id);
+ReplicationPeerConfig peerConfig = replicationPeers.getPeerConfig(id);
+ReplicationPeer peer = replicationPeers.getPeer(id);
ReplicationSourceInterface src = getReplicationSource(id, peerConfig,
peer);
synchronized (this.walsById) {
this.sources.add(src);
@@ -344,7 +343,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
public void deleteSource(String peerId, boolean closeConnection) {
abortWhenFail(() -> this.queueStorage.removeQueue(server.getServerName(),
peerId));
if (closeConnection) {
- this.replicationPeers.peerDisconnected(peerId);
+ this.replicationPeers.removePeer(peerId);
}
}
@@ -437,12 +436,12 @@ public class ReplicationSourceManager implements
ReplicationListener {
// update replication queues on ZK
// synchronize on replicationPeers to avoid adding source for the
to-be-removed peer
synchronized (replicationPeers) {
- for (String id : replicationPeers.getConnectedPeerIds()) {
+ for (String id : replicationPeers.getAllPeerIds()) {
try {
this.queueStorage.addWAL(server.getServerName(), id, logName);
} catch (ReplicationException e) {
- throw new IOException("Cannot add log to replication queue" +
-" when creating a new source, queueId=" + id + ", filename=" +
logName, e);
+ throw new IOException("Cannot add log to replication queue"
+ + " when creating a new source, queueId=" + id + ", filename=" +
logName, e);
}
}
}
@@ -587,7 +586,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
public void addPeer(String id) throws ReplicationException, IOException {
LOG.info("Trying to add peer, peerId: " + id);
-boolean added = this.replicationPeers.peerConnected(id);
+boolean added = this.replicationPeers.addPeer(id);
if (added) {
LOG.info("Peer " + id + " connected success, trying to start the
replication source thread.");
addSource(id);
@@ -723,16 +722,21 @@ public class ReplicationSourceManager implements
ReplicationListener {
// there is not an actual peer defined corresponding to peerId for
the failover.
ReplicationQueueInfo replicationQueueInfo = new
ReplicationQueueInfo(peerId);
[32/40] hbase git commit: HBASE-19634 Add permission check for executeProcedures in AccessController
HBASE-19634 Add permission check for executeProcedures in AccessController
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/53b18fe0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/53b18fe0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/53b18fe0
Branch: refs/heads/branch-2
Commit: 53b18fe0ac6062de6e0bd002cd5ed55b2fecf27e
Parents: 029406c
Author: zhangduo
Authored: Thu Jan 4 16:18:21 2018 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../hbase/coprocessor/RegionServerObserver.java | 14 ++
.../hbase/regionserver/RSRpcServices.java | 52 +++-
.../RegionServerCoprocessorHost.java| 18 +++
.../hbase/security/access/AccessController.java | 30 ++-
.../hadoop/hbase/TestJMXConnectorServer.java| 7 +++
.../security/access/TestAccessController.java | 18 +--
6 files changed, 100 insertions(+), 39 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/53b18fe0/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
index c1af3fb..5b751df 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
@@ -126,4 +126,18 @@ public interface RegionServerObserver {
default void postClearCompactionQueues(
final ObserverContext ctx)
throws IOException {}
+
+ /**
+ * This will be called before executing procedures
+ * @param ctx the environment to interact with the framework and region
server.
+ */
+ default void
preExecuteProcedures(ObserverContext ctx)
+ throws IOException {}
+
+ /**
+ * This will be called after executing procedures
+ * @param ctx the environment to interact with the framework and region
server.
+ */
+ default void
postExecuteProcedures(ObserverContext ctx)
+ throws IOException {}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/53b18fe0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 5b0ce4e..cceae28 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -3508,36 +3508,40 @@ public class RSRpcServices implements
HBaseRPCErrorHandler,
}
@Override
+ @QosPriority(priority = HConstants.ADMIN_QOS)
public ExecuteProceduresResponse executeProcedures(RpcController controller,
ExecuteProceduresRequest request) throws ServiceException {
-if (request.getOpenRegionCount() > 0) {
- for (OpenRegionRequest req : request.getOpenRegionList()) {
-openRegion(controller, req);
+try {
+ checkOpen();
+ regionServer.getRegionServerCoprocessorHost().preExecuteProcedures();
+ if (request.getOpenRegionCount() > 0) {
+for (OpenRegionRequest req : request.getOpenRegionList()) {
+ openRegion(controller, req);
+}
}
-}
-if (request.getCloseRegionCount() > 0) {
- for (CloseRegionRequest req : request.getCloseRegionList()) {
-closeRegion(controller, req);
+ if (request.getCloseRegionCount() > 0) {
+for (CloseRegionRequest req : request.getCloseRegionList()) {
+ closeRegion(controller, req);
+}
}
-}
-if (request.getProcCount() > 0) {
- for (RemoteProcedureRequest req : request.getProcList()) {
-RSProcedureCallable callable;
-try {
- callable =
-
Class.forName(req.getProcClass()).asSubclass(RSProcedureCallable.class).newInstance();
-} catch (Exception e) {
- // here we just ignore the error as this should not happen and we do
not provide a general
- // way to report errors for all types of remote procedure. The
procedure will hang at
- // master side but after you solve the problem and restart master it
will be executed
- // again and pass.
- LOG.warn("create procedure of type " + req.getProcClass() + "
failed, give up", e);
- continue;
+ if (request.getProcCount() > 0) {
+for (RemoteProcedureRequest req : request.getProcList()) {
+ RSPr
[23/40] hbase git commit: HBASE-19635 Introduce a thread at RS side to call reportProcedureDone
HBASE-19635 Introduce a thread at RS side to call reportProcedureDone
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4c6942df
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4c6942df
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4c6942df
Branch: refs/heads/branch-2
Commit: 4c6942df58df6f148ea88733914df9299e5946c5
Parents: d9b5eb3
Author: zhangduo
Authored: Wed Dec 27 20:13:42 2017 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../src/main/protobuf/RegionServerStatus.proto | 5 +-
.../hadoop/hbase/master/MasterRpcServices.java | 15 ++-
.../hbase/regionserver/HRegionServer.java | 72
.../RemoteProcedureResultReporter.java | 111 +++
.../handler/RSProcedureHandler.java | 2 +-
5 files changed, 149 insertions(+), 56 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/4c6942df/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
index 4f75941..3f836cd 100644
--- a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
@@ -146,7 +146,7 @@ message RegionSpaceUseReportRequest {
message RegionSpaceUseReportResponse {
}
-message ReportProcedureDoneRequest {
+message RemoteProcedureResult {
required uint64 proc_id = 1;
enum Status {
SUCCESS = 1;
@@ -155,6 +155,9 @@ message ReportProcedureDoneRequest {
required Status status = 2;
optional ForeignExceptionMessage error = 3;
}
+message ReportProcedureDoneRequest {
+ repeated RemoteProcedureResult result = 1;
+}
message ReportProcedureDoneResponse {
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/4c6942df/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 6ce0d39..8f92041 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -268,6 +268,7 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProto
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportResponse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RemoteProcedureResult;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportProcedureDoneRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportProcedureDoneResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorRequest;
@@ -2236,12 +2237,14 @@ public class MasterRpcServices extends RSRpcServices
@Override
public ReportProcedureDoneResponse reportProcedureDone(RpcController
controller,
ReportProcedureDoneRequest request) throws ServiceException {
-if (request.getStatus() == ReportProcedureDoneRequest.Status.SUCCESS) {
- master.remoteProcedureCompleted(request.getProcId());
-} else {
- master.remoteProcedureFailed(request.getProcId(),
-RemoteProcedureException.fromProto(request.getError()));
-}
+request.getResultList().forEach(result -> {
+ if (result.getStatus() == RemoteProcedureResult.Status.SUCCESS) {
+master.remoteProcedureCompleted(result.getProcId());
+ } else {
+master.remoteProcedureFailed(result.getProcId(),
+ RemoteProcedureException.fromProto(result.getError()));
+ }
+});
return ReportProcedureDoneResponse.getDefaultInstance();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/4c6942df/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index c95ac37..81febc0 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegio
[29/40] hbase git commit: HBASE-19686 Use KeyLocker instead of ReentrantLock in PeerProcedureHandlerImpl
HBASE-19686 Use KeyLocker instead of ReentrantLock in PeerProcedureHandlerImpl
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b0de0ec4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b0de0ec4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b0de0ec4
Branch: refs/heads/branch-2
Commit: b0de0ec4396414ab5b130e2b6db9f226163ca5a3
Parents: 5d922a5
Author: zhangduo
Authored: Tue Jan 2 16:13:55 2018 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../regionserver/PeerProcedureHandlerImpl.java | 41 ++--
1 file changed, 29 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/b0de0ec4/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
index 1efe180..c09c6a0 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
@@ -19,10 +19,10 @@
package org.apache.hadoop.hbase.replication.regionserver;
import java.io.IOException;
-import java.util.concurrent.locks.ReentrantLock;
-
+import java.util.concurrent.locks.Lock;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
+import org.apache.hadoop.hbase.util.KeyLocker;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,7 +32,7 @@ public class PeerProcedureHandlerImpl implements
PeerProcedureHandler {
private static final Logger LOG =
LoggerFactory.getLogger(PeerProcedureHandlerImpl.class);
private final ReplicationSourceManager replicationSourceManager;
- private final ReentrantLock peersLock = new ReentrantLock();
+ private final KeyLocker peersLock = new KeyLocker<>();
public PeerProcedureHandlerImpl(ReplicationSourceManager
replicationSourceManager) {
this.replicationSourceManager = replicationSourceManager;
@@ -40,40 +40,57 @@ public class PeerProcedureHandlerImpl implements
PeerProcedureHandler {
@Override
public void addPeer(String peerId) throws ReplicationException, IOException {
-peersLock.lock();
+Lock peerLock = peersLock.acquireLock(peerId);
try {
replicationSourceManager.addPeer(peerId);
} finally {
- peersLock.unlock();
+ peerLock.unlock();
}
}
@Override
public void removePeer(String peerId) throws ReplicationException,
IOException {
-peersLock.lock();
+Lock peerLock = peersLock.acquireLock(peerId);
try {
if (replicationSourceManager.getReplicationPeers().getPeer(peerId) !=
null) {
replicationSourceManager.removePeer(peerId);
}
} finally {
- peersLock.unlock();
+ peerLock.unlock();
}
}
@Override
public void disablePeer(String peerId) throws ReplicationException,
IOException {
-PeerState newState =
replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
-LOG.info("disable replication peer, id: " + peerId + ", new state: " +
newState);
+PeerState newState;
+Lock peerLock = peersLock.acquireLock(peerId);
+try {
+ newState =
replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
+} finally {
+ peerLock.unlock();
+}
+LOG.info("disable replication peer, id: {}, new state: {}", peerId,
newState);
}
@Override
public void enablePeer(String peerId) throws ReplicationException,
IOException {
-PeerState newState =
replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
-LOG.info("enable replication peer, id: " + peerId + ", new state: " +
newState);
+PeerState newState;
+Lock peerLock = peersLock.acquireLock(peerId);
+try {
+ newState =
replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
+} finally {
+ peerLock.unlock();
+}
+LOG.info("enable replication peer, id: {}, new state: {}", peerId,
newState);
}
@Override
public void updatePeerConfig(String peerId) throws ReplicationException,
IOException {
-replicationSourceManager.getReplicationPeers().refreshPeerConfig(peerId);
+Lock peerLock = peersLock.acquireLock(peerId);
+try {
+ replicationSourceManager.getReplicationPeers().refreshPeerConfig(peerId);
+} finally {
+ peerLock.unlock();
+}
[01/40] hbase git commit: HBASE-20082 Fix findbugs errors only on master which are introduced by HBASE-19397
Repository: hbase
Updated Branches:
refs/heads/branch-2 3576eb6bd -> d4e0fc74d
HBASE-20082 Fix findbugs errors only on master which are introduced by
HBASE-19397
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d4e0fc74
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d4e0fc74
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d4e0fc74
Branch: refs/heads/branch-2
Commit: d4e0fc74d481417713ea0496c97ea7db20bf0324
Parents: 55d0d30
Author: zhangduo
Authored: Mon Feb 26 10:22:09 2018 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../hadoop/hbase/master/replication/RefreshPeerProcedure.java | 2 ++
.../hbase/replication/regionserver/ReplicationSourceManager.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d4e0fc74/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
index 1253ef9..ba9bcdc 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
@@ -50,6 +50,8 @@ public class RefreshPeerProcedure extends
Procedure
private PeerOperationType type;
+ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value =
"IS2_INCONSISTENT_SYNC",
+ justification = "Will never change after construction")
private ServerName targetServer;
private boolean dispatched;
http://git-wip-us.apache.org/repos/asf/hbase/blob/d4e0fc74/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 6965f55..d11dc8e 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -402,7 +402,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
toStartup.add(replicationSource);
}
}
-for (ReplicationSourceInterface replicationSource : oldsources) {
+for (ReplicationSourceInterface replicationSource : toStartup) {
replicationSource.startup();
}
}
[17/40] hbase git commit: HBASE-19573 Rewrite ReplicationPeer with the new replication storage interface
HBASE-19573 Rewrite ReplicationPeer with the new replication storage interface
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/eae251d2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/eae251d2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/eae251d2
Branch: refs/heads/branch-2
Commit: eae251d2037f2fd53a893f2c18717f56c41351d3
Parents: 59cad95
Author: Guanghao Zhang
Authored: Tue Dec 26 11:39:34 2017 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../replication/VerifyReplication.java | 5 -
.../hbase/replication/ReplicationPeer.java | 42 ++--
.../hbase/replication/ReplicationPeerImpl.java | 169 ++
.../replication/ReplicationPeerZKImpl.java | 233 ---
.../hbase/replication/ReplicationPeers.java | 4 +-
.../replication/ReplicationPeersZKImpl.java | 23 +-
.../replication/TestReplicationStateBasic.java | 7 +-
.../regionserver/PeerProcedureHandlerImpl.java | 29 +--
8 files changed, 216 insertions(+), 296 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/eae251d2/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
--
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index 9065f4e..09d4b4b 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -51,7 +51,6 @@ import org.apache.hadoop.hbase.mapreduce.TableSplit;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationFactory;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
-import org.apache.hadoop.hbase.replication.ReplicationPeerZKImpl;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
import org.apache.hadoop.hbase.util.Bytes;
@@ -333,7 +332,6 @@ public class VerifyReplication extends Configured
implements Tool {
private static Pair
getPeerQuorumConfig(
final Configuration conf, String peerId) throws IOException {
ZKWatcher localZKW = null;
-ReplicationPeerZKImpl peer = null;
try {
localZKW = new ZKWatcher(conf, "VerifyReplication",
new Abortable() {
@@ -354,9 +352,6 @@ public class VerifyReplication extends Configured
implements Tool {
throw new IOException(
"An error occurred while trying to connect to the remove peer
cluster", e);
} finally {
- if (peer != null) {
-peer.close();
- }
if (localZKW != null) {
localZKW.close();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/eae251d2/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
index b66d76d..4846018 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.TableName;
import org.apache.yetus.audience.InterfaceAudience;
-
/**
* ReplicationPeer manages enabled / disabled state for the peer.
*/
@@ -49,65 +48,52 @@ public interface ReplicationPeer {
String getId();
/**
- * Get the peer config object
- * @return the ReplicationPeerConfig for this peer
- */
- public ReplicationPeerConfig getPeerConfig();
-
- /**
- * Get the peer config object. if loadFromBackingStore is true, it will load
from backing store
- * directly and update its load peer config. otherwise, just return the
local cached peer config.
- * @return the ReplicationPeerConfig for this peer
- */
- public ReplicationPeerConfig getPeerConfig(boolean loadFromBackingStore)
- throws ReplicationException;
-
- /**
* Returns the state of the peer by reading local cache.
* @return the enabled state
*/
PeerState getPeerState();
/**
- * Returns the state of peer, if loadFromBackingStore is true, it will load
from backing store
- * directly and update its local peer state. otherwise, just return the
local cac
[40/40] hbase git commit: HBASE-19873 addendum add missing rule for new tests
HBASE-19873 addendum add missing rule for new tests
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/abaa80f2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/abaa80f2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/abaa80f2
Branch: refs/heads/branch-2
Commit: abaa80f26ea71e5112b2c36bf33ef92cae18cb5b
Parents: db4cb01
Author: zhangduo
Authored: Tue Jan 30 09:40:23 2018 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../hbase/replication/TestZKReplicationPeerStorage.java | 7 ++-
.../hbase/replication/TestZKReplicationQueueStorage.java | 8 +++-
.../hbase/replication/TestReplicationProcedureRetry.java | 7 ++-
.../apache/hadoop/hbase/util/TestHBaseFsckReplication.java | 6 ++
4 files changed, 25 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/abaa80f2/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
--
diff --git
a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
index 3eb11da..3290fb0 100644
---
a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
+++
b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
@@ -33,19 +33,24 @@ import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.stream.Stream;
-
+import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseZKTestingUtility;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category({ ReplicationTests.class, MediumTests.class })
public class TestZKReplicationPeerStorage {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestZKReplicationPeerStorage.class);
+
private static final HBaseZKTestingUtility UTIL = new
HBaseZKTestingUtility();
private static ZKReplicationPeerStorage STORAGE;
http://git-wip-us.apache.org/repos/asf/hbase/blob/abaa80f2/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
--
diff --git
a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
index 786730f..2c01a26 100644
---
a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
+++
b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
@@ -27,8 +27,8 @@ import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.SortedSet;
-
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseZKTestingUtility;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -38,11 +38,17 @@ import org.apache.zookeeper.KeeperException;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category({ ReplicationTests.class, MediumTests.class })
public class TestZKReplicationQueueStorage {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestZKReplicationQueueStorage.class);
+
private static final HBaseZKTestingUtility UTIL = new
HBaseZKTestingUtility();
private static ZKReplicationQueueStorage STORAGE;
http://git-wip-us.apache.org/repos/asf/hbase/blob/abaa80f2/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
index ab35b46..a2ae0b4 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
+++
b/hbase
[33/40] hbase git commit: HBASE-19636 All rs should already start work with the new peer change when replication peer procedure is finished
http://git-wip-us.apache.org/repos/asf/hbase/blob/d36aacdf/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
index 4c1b94f..6d2b578 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
@@ -28,12 +28,11 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
@@ -41,6 +40,8 @@ import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
@@ -71,9 +72,6 @@ public class TestNamespaceReplication extends
TestReplicationBase {
private static final byte[] val = Bytes.toBytes("myval");
- private static HTableDescriptor tabA;
- private static HTableDescriptor tabB;
-
private static Connection connection1;
private static Connection connection2;
private static Admin admin1;
@@ -93,23 +91,21 @@ public class TestNamespaceReplication extends
TestReplicationBase {
admin2.createNamespace(NamespaceDescriptor.create(ns1).build());
admin2.createNamespace(NamespaceDescriptor.create(ns2).build());
-tabA = new HTableDescriptor(tabAName);
-HColumnDescriptor fam = new HColumnDescriptor(f1Name);
-fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
-tabA.addFamily(fam);
-fam = new HColumnDescriptor(f2Name);
-fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
-tabA.addFamily(fam);
+TableDescriptorBuilder builder =
TableDescriptorBuilder.newBuilder(tabAName);
+builder.addColumnFamily(ColumnFamilyDescriptorBuilder
+
.newBuilder(f1Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
+builder.addColumnFamily(ColumnFamilyDescriptorBuilder
+
.newBuilder(f2Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
+TableDescriptor tabA = builder.build();
admin1.createTable(tabA);
admin2.createTable(tabA);
-tabB = new HTableDescriptor(tabBName);
-fam = new HColumnDescriptor(f1Name);
-fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
-tabB.addFamily(fam);
-fam = new HColumnDescriptor(f2Name);
-fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
-tabB.addFamily(fam);
+builder = TableDescriptorBuilder.newBuilder(tabBName);
+builder.addColumnFamily(ColumnFamilyDescriptorBuilder
+
.newBuilder(f1Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
+builder.addColumnFamily(ColumnFamilyDescriptorBuilder
+
.newBuilder(f2Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
+TableDescriptor tabB = builder.build();
admin1.createTable(tabB);
admin2.createTable(tabB);
}
@@ -137,22 +133,24 @@ public class TestNamespaceReplication extends
TestReplicationBase {
@Test
public void testNamespaceReplication() throws Exception {
+String peerId = "2";
+
Table htab1A = connection1.getTable(tabAName);
Table htab2A = connection2.getTable(tabAName);
Table htab1B = connection1.getTable(tabBName);
Table htab2B = connection2.getTable(tabBName);
-ReplicationPeerConfig rpc = admin.getPeerConfig("2");
-rpc.setReplicateAllUserTables(false);
-admin.updatePeerConfig("2", rpc);
+ReplicationPeerConfig rpc = admin1.getReplicationPeerConfig(peerId);
+admin1.updateReplicationPeerConfig(peerId,
+
ReplicationPeerConfig.newBuilder(rpc).setReplicateAllUserTables(false).build());
// add ns1 to peer config which replicate to cluster2
-rpc = admin.getPeerConfig("2");
+rpc = admin1.getReplicationPeerConfig(peerId);
Set namespaces = new HashSet<>();
namespaces.add(ns1);
-rpc.setNamespaces(namespaces);
-admin.updatePeerConfig("2", rpc);
+admin1.updateReplicationPeerConfig(peerId,
+ Replication
[12/40] hbase git commit: HBASE-19543 Abstract a replication storage interface to extract the zk specific code
http://git-wip-us.apache.org/repos/asf/hbase/blob/1de08ded/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java
deleted file mode 100644
index b6f8784..000
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.replication;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Pattern;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.replication.BaseReplicationEndpoint;
-import org.apache.hadoop.hbase.replication.ReplicationException;
-import org.apache.hadoop.hbase.replication.ReplicationFactory;
-import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
-import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
-import org.apache.hadoop.hbase.replication.ReplicationPeers;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.yetus.audience.InterfaceAudience;
-
-/**
- * Manages and performs all replication admin operations.
- *
- * Used to add/remove a replication peer.
- */
-@InterfaceAudience.Private
-public class ReplicationManager {
- private final ReplicationQueuesClient replicationQueuesClient;
- private final ReplicationPeers replicationPeers;
-
- public ReplicationManager(Configuration conf, ZKWatcher zkw, Abortable
abortable)
- throws IOException {
-try {
- this.replicationQueuesClient = ReplicationFactory
- .getReplicationQueuesClient(new
ReplicationQueuesClientArguments(conf, abortable, zkw));
- this.replicationQueuesClient.init();
- this.replicationPeers = ReplicationFactory.getReplicationPeers(zkw, conf,
-this.replicationQueuesClient, abortable);
- this.replicationPeers.init();
-} catch (Exception e) {
- throw new IOException("Failed to construct ReplicationManager", e);
-}
- }
-
- public void addReplicationPeer(String peerId, ReplicationPeerConfig
peerConfig, boolean enabled)
- throws ReplicationException {
-checkPeerConfig(peerConfig);
-replicationPeers.registerPeer(peerId, peerConfig, enabled);
-replicationPeers.peerConnected(peerId);
- }
-
- public void removeReplicationPeer(String peerId) throws ReplicationException
{
-replicationPeers.peerDisconnected(peerId);
-replicationPeers.unregisterPeer(peerId);
- }
-
- public void enableReplicationPeer(String peerId) throws ReplicationException
{
-this.replicationPeers.enablePeer(peerId);
- }
-
- public void disableReplicationPeer(String peerId) throws
ReplicationException {
-this.replicationPeers.disablePeer(peerId);
- }
-
- public ReplicationPeerConfig getPeerConfig(String peerId)
- throws ReplicationException, ReplicationPeerNotFoundException {
-ReplicationPeerConfig peerConfig =
replicationPeers.getReplicationPeerConfig(peerId);
-if (peerConfig == null) {
- throw new ReplicationPeerNotFoundException(peerId);
-}
-return peerConfig;
- }
-
- public void updatePeerConfig(String peerId, ReplicationPeerConfig peerConfig)
- throws ReplicationException, IOException {
-checkPeerConfig(peerConfig);
-this.replicationPeers.updatePeerConfig(peerId, peerConfig);
- }
-
- public List listReplicationPeers(Pattern pattern)
- throws ReplicationException {
-List peers = new ArrayList<>();
-List peerIds = replicationPeers.getAllPeerIds();
-for (String peerId : peer
[36/40] hbase git commit: HBASE-19748 TestRegionReplicaFailover and TestRegionReplicaReplicationEndpoint UT hangs
HBASE-19748 TestRegionReplicaFailover and TestRegionReplicaReplicationEndpoint
UT hangs
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a7dce367
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a7dce367
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a7dce367
Branch: refs/heads/branch-2
Commit: a7dce3676643afb19f27ab0ce22b8427d98b2c83
Parents: 1c32241
Author: huzheng
Authored: Wed Jan 10 15:00:30 2018 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../main/java/org/apache/hadoop/hbase/master/HMaster.java | 9 -
1 file changed, 4 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a7dce367/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 71af62b..cd967ed 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -40,7 +40,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
-import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
@@ -71,6 +70,7 @@ import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.PleaseHoldException;
+import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
import org.apache.hadoop.hbase.ServerMetricsBuilder;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableDescriptors;
@@ -3407,13 +3407,12 @@ public class HMaster extends HRegionServer implements
MasterServices {
cpHost.preGetReplicationPeerConfig(peerId);
}
LOG.info(getClientIdAuditPrefix() + " get replication peer config, id=" +
peerId);
-Optional peerConfig =
- this.replicationPeerManager.getPeerConfig(peerId);
-
+ReplicationPeerConfig peerConfig =
this.replicationPeerManager.getPeerConfig(peerId)
+.orElseThrow(() -> new ReplicationPeerNotFoundException(peerId));
if (cpHost != null) {
cpHost.postGetReplicationPeerConfig(peerId);
}
-return peerConfig.orElse(null);
+return peerConfig;
}
@Override
[37/40] hbase git commit: HBASE-19711 TestReplicationAdmin.testConcurrentPeerOperations hangs
HBASE-19711 TestReplicationAdmin.testConcurrentPeerOperations hangs
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1e69d7fa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1e69d7fa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1e69d7fa
Branch: refs/heads/branch-2
Commit: 1e69d7fa240f22af62d5934dd2eb7f9e767f572c
Parents: 0165455
Author: Guanghao Zhang
Authored: Fri Jan 5 15:39:06 2018 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../procedure/MasterProcedureScheduler.java | 23
1 file changed, 19 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/1e69d7fa/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
index b18dd6c..eab06a2 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
@@ -290,7 +290,7 @@ public class MasterProcedureScheduler extends
AbstractProcedureScheduler {
@Override
public void completionCleanup(final Procedure proc) {
if (proc instanceof TableProcedureInterface) {
- TableProcedureInterface iProcTable = (TableProcedureInterface)proc;
+ TableProcedureInterface iProcTable = (TableProcedureInterface) proc;
boolean tableDeleted;
if (proc.hasException()) {
Exception procEx = proc.getException().unwrapRemoteException();
@@ -311,9 +311,7 @@ public class MasterProcedureScheduler extends
AbstractProcedureScheduler {
}
} else if (proc instanceof PeerProcedureInterface) {
PeerProcedureInterface iProcPeer = (PeerProcedureInterface) proc;
- if (iProcPeer.getPeerOperationType() == PeerOperationType.REMOVE) {
-removePeerQueue(iProcPeer.getPeerId());
- }
+ tryCleanupPeerQueue(iProcPeer.getPeerId(), proc);
} else {
// No cleanup for ServerProcedureInterface types, yet.
return;
@@ -402,6 +400,23 @@ public class MasterProcedureScheduler extends
AbstractProcedureScheduler {
locking.removePeerLock(peerId);
}
+ private void tryCleanupPeerQueue(String peerId, Procedure procedure) {
+schedLock();
+try {
+ PeerQueue queue = AvlTree.get(peerMap, peerId,
PEER_QUEUE_KEY_COMPARATOR);
+ if (queue == null) {
+return;
+ }
+
+ final LockAndQueue lock = locking.getPeerLock(peerId);
+ if (queue.isEmpty() && lock.tryExclusiveLock(procedure)) {
+removeFromRunQueue(peerRunQueue, queue);
+removePeerQueue(peerId);
+ }
+} finally {
+ schedUnlock();
+}
+ }
private static boolean isPeerProcedure(Procedure proc) {
return proc instanceof PeerProcedureInterface;
[20/40] hbase git commit: HBASE-19642 Fix locking for peer modification procedure
HBASE-19642 Fix locking for peer modification procedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b40c4268
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b40c4268
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b40c4268
Branch: refs/heads/branch-2
Commit: b40c42680656fd37a15936bdee1abfdb380c0ff5
Parents: ac07e07
Author: zhangduo
Authored: Wed Dec 27 18:27:13 2017 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../master/replication/ModifyPeerProcedure.java | 21 +---
1 file changed, 18 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/b40c4268/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
index 279fbc7..a682606 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
@@ -46,6 +46,8 @@ public abstract class ModifyPeerProcedure
protected String peerId;
+ private volatile boolean locked;
+
// used to keep compatible with old client where we can only returns after
updateStorage.
protected ProcedurePrepareLatch latch;
@@ -145,17 +147,30 @@ public abstract class ModifyPeerProcedure
@Override
protected LockState acquireLock(MasterProcedureEnv env) {
-return env.getProcedureScheduler().waitPeerExclusiveLock(this, peerId)
- ? LockState.LOCK_EVENT_WAIT
- : LockState.LOCK_ACQUIRED;
+if (env.getProcedureScheduler().waitPeerExclusiveLock(this, peerId)) {
+ return LockState.LOCK_EVENT_WAIT;
+}
+locked = true;
+return LockState.LOCK_ACQUIRED;
}
@Override
protected void releaseLock(MasterProcedureEnv env) {
+locked = false;
env.getProcedureScheduler().wakePeerExclusiveLock(this, peerId);
}
@Override
+ protected boolean holdLock(MasterProcedureEnv env) {
+return true;
+ }
+
+ @Override
+ protected boolean hasLock(MasterProcedureEnv env) {
+return locked;
+ }
+
+ @Override
protected void rollbackState(MasterProcedureEnv env, PeerModificationState
state)
throws IOException, InterruptedException {
if (state == PeerModificationState.PRE_PEER_MODIFICATION) {
[15/40] hbase git commit: HBASE-19599 Remove ReplicationQueuesClient, use ReplicationQueueStorage directly
HBASE-19599 Remove ReplicationQueuesClient, use ReplicationQueueStorage directly
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/15055a42
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/15055a42
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/15055a42
Branch: refs/heads/branch-2
Commit: 15055a421bbc789684af5bffadab04ed95b09568
Parents: 1de08de
Author: zhangduo
Authored: Mon Dec 25 18:49:56 2017 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../hbase/replication/ReplicationFactory.java | 19 +-
.../replication/ReplicationPeersZKImpl.java | 21 +-
.../replication/ReplicationQueueStorage.java| 26 +-
.../replication/ReplicationQueuesClient.java| 93 -
.../ReplicationQueuesClientArguments.java | 40 --
.../ReplicationQueuesClientZKImpl.java | 181 -
.../replication/ZKReplicationQueueStorage.java | 90 -
.../replication/TestReplicationStateBasic.java | 378 +++
.../replication/TestReplicationStateZKImpl.java | 153
.../TestZKReplicationQueueStorage.java | 74
.../cleaner/ReplicationZKNodeCleaner.java | 71 ++--
.../cleaner/ReplicationZKNodeCleanerChore.java | 5 +-
.../replication/ReplicationPeerManager.java | 31 +-
.../master/ReplicationHFileCleaner.java | 109 ++
.../master/ReplicationLogCleaner.java | 44 +--
.../regionserver/DumpReplicationQueues.java | 78 ++--
.../hbase/util/hbck/ReplicationChecker.java | 14 +-
.../client/TestAsyncReplicationAdminApi.java| 31 +-
.../replication/TestReplicationAdmin.java | 2 +
.../hbase/master/cleaner/TestLogsCleaner.java | 81 ++--
.../cleaner/TestReplicationHFileCleaner.java| 29 --
.../cleaner/TestReplicationZKNodeCleaner.java | 12 +-
.../replication/TestReplicationStateBasic.java | 378 ---
.../replication/TestReplicationStateZKImpl.java | 232
.../TestReplicationSourceManagerZkImpl.java | 41 --
25 files changed, 890 insertions(+), 1343 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/15055a42/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
index 9f4ad18..6c1c213 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -37,20 +36,14 @@ public class ReplicationFactory {
args);
}
- public static ReplicationQueuesClient
- getReplicationQueuesClient(ReplicationQueuesClientArguments args) throws
Exception {
-return (ReplicationQueuesClient) ConstructorUtils
-.invokeConstructor(ReplicationQueuesClientZKImpl.class, args);
- }
-
- public static ReplicationPeers getReplicationPeers(final ZKWatcher zk,
Configuration conf,
- Abortable abortable) {
+ public static ReplicationPeers getReplicationPeers(ZKWatcher zk,
Configuration conf,
+ Abortable abortable) {
return getReplicationPeers(zk, conf, null, abortable);
}
- public static ReplicationPeers getReplicationPeers(final ZKWatcher zk,
Configuration conf,
- final
ReplicationQueuesClient queuesClient, Abortable abortable) {
-return new ReplicationPeersZKImpl(zk, conf, queuesClient, abortable);
+ public static ReplicationPeers getReplicationPeers(ZKWatcher zk,
Configuration conf,
+ ReplicationQueueStorage queueStorage, Abortable abortable) {
+return new ReplicationPeersZKImpl(zk, conf, queueStorage, abortable);
}
public static ReplicationTracker getReplicationTracker(ZKWatcher zookeeper,
http://git-wip-us.apache.org/repos/asf/hbase/blob/15055a42/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
index 419e289..4e5f757 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replicat
[18/40] hbase git commit: HBASE-19630 Add peer cluster key check when add new replication peer
HBASE-19630 Add peer cluster key check when add new replication peer
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/712b5a80
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/712b5a80
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/712b5a80
Branch: refs/heads/branch-2
Commit: 712b5a80dc6c24016ee9111418df52bae56eef8a
Parents: eae251d
Author: Guanghao Zhang
Authored: Tue Dec 26 21:10:00 2017 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../replication/ReplicationPeerManager.java | 54
.../replication/TestReplicationAdmin.java | 22
2 files changed, 54 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/712b5a80/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index 84abfeb..b78cbce 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.master.replication;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -42,6 +43,7 @@ import
org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.zookeeper.ZKConfig;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
@@ -216,36 +218,36 @@ public final class ReplicationPeerManager {
return desc != null ? Optional.of(desc.getPeerConfig()) : Optional.empty();
}
- /**
- * If replicate_all flag is true, it means all user tables will be
replicated to peer cluster.
- * Then allow config exclude namespaces or exclude table-cfs which can't be
replicated to peer
- * cluster.
- *
- * If replicate_all flag is false, it means all user tables can't be
replicated to peer cluster.
- * Then allow to config namespaces or table-cfs which will be replicated to
peer cluster.
- */
- private static void checkPeerConfig(ReplicationPeerConfig peerConfig)
- throws DoNotRetryIOException {
+ private void checkPeerConfig(ReplicationPeerConfig peerConfig) throws
DoNotRetryIOException {
+checkClusterKey(peerConfig.getClusterKey());
+
if (peerConfig.replicateAllUserTables()) {
- if ((peerConfig.getNamespaces() != null &&
!peerConfig.getNamespaces().isEmpty()) ||
-(peerConfig.getTableCFsMap() != null &&
!peerConfig.getTableCFsMap().isEmpty())) {
-throw new DoNotRetryIOException("Need clean namespaces or table-cfs
config firstly " +
- "when you want replicate all cluster");
+ // If replicate_all flag is true, it means all user tables will be
replicated to peer cluster.
+ // Then allow config exclude namespaces or exclude table-cfs which can't
be replicated to peer
+ // cluster.
+ if ((peerConfig.getNamespaces() != null &&
!peerConfig.getNamespaces().isEmpty())
+ || (peerConfig.getTableCFsMap() != null &&
!peerConfig.getTableCFsMap().isEmpty())) {
+throw new DoNotRetryIOException("Need clean namespaces or table-cfs
config firstly "
++ "when you want replicate all cluster");
}
checkNamespacesAndTableCfsConfigConflict(peerConfig.getExcludeNamespaces(),
peerConfig.getExcludeTableCFsMap());
} else {
- if ((peerConfig.getExcludeNamespaces() != null &&
-!peerConfig.getExcludeNamespaces().isEmpty()) ||
-(peerConfig.getExcludeTableCFsMap() != null &&
- !peerConfig.getExcludeTableCFsMap().isEmpty())) {
+ // If replicate_all flag is false, it means all user tables can't be
replicated to peer
+ // cluster. Then allow to config namespaces or table-cfs which will be
replicated to peer
+ // cluster.
+ if ((peerConfig.getExcludeNamespaces() != null
+ && !peerConfig.getExcludeNamespaces().isEmpty())
+ || (peerConfig.getExcludeTableCFsMap() != null
+ && !peerConfig.getExcludeTableCFsMap().isEmpty())) {
throw new DoNotRetryIOException(
-"Need clean exclude-namespaces or exclude-table-cfs config
firstly" +
-
[38/40] hbase git commit: HBASE-19783 Change replication peer cluster key/endpoint from a not-null value to null is not allowed
HBASE-19783 Change replication peer cluster key/endpoint from a not-null value
to null is not allowed
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/db4cb01a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/db4cb01a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/db4cb01a
Branch: refs/heads/branch-2
Commit: db4cb01a9e933e600743f52c681eaa6d15e219a4
Parents: a7dce36
Author: Guanghao Zhang
Authored: Fri Jan 12 22:04:38 2018 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../replication/ReplicationPeerManager.java | 28 +---
1 file changed, 19 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/db4cb01a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index 696b2d7..19fc7f4 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -132,20 +132,19 @@ public class ReplicationPeerManager {
checkPeerConfig(peerConfig);
ReplicationPeerDescription desc = checkPeerExists(peerId);
ReplicationPeerConfig oldPeerConfig = desc.getPeerConfig();
-if (!StringUtils.isBlank(peerConfig.getClusterKey()) &&
- !peerConfig.getClusterKey().equals(oldPeerConfig.getClusterKey())) {
+if (!isStringEquals(peerConfig.getClusterKey(),
oldPeerConfig.getClusterKey())) {
throw new DoNotRetryIOException(
"Changing the cluster key on an existing peer is not allowed.
Existing key '" +
-oldPeerConfig.getClusterKey() + "' for peer " + peerId + " does
not match new key '" +
-peerConfig.getClusterKey() + "'");
+ oldPeerConfig.getClusterKey() + "' for peer " + peerId + " does
not match new key '" +
+ peerConfig.getClusterKey() + "'");
}
-if (!StringUtils.isBlank(peerConfig.getReplicationEndpointImpl()) &&
-
!peerConfig.getReplicationEndpointImpl().equals(oldPeerConfig.getReplicationEndpointImpl()))
{
+if (!isStringEquals(peerConfig.getReplicationEndpointImpl(),
+ oldPeerConfig.getReplicationEndpointImpl())) {
throw new DoNotRetryIOException("Changing the replication endpoint
implementation class " +
-"on an existing peer is not allowed. Existing class '" +
-oldPeerConfig.getReplicationEndpointImpl() + "' for peer " + peerId +
-" does not match new class '" +
peerConfig.getReplicationEndpointImpl() + "'");
+ "on an existing peer is not allowed. Existing class '" +
+ oldPeerConfig.getReplicationEndpointImpl() + "' for peer " + peerId +
+ " does not match new class '" +
peerConfig.getReplicationEndpointImpl() + "'");
}
}
@@ -341,4 +340,15 @@ public class ReplicationPeerManager {
return new ReplicationPeerManager(peerStorage,
ReplicationStorageFactory.getReplicationQueueStorage(zk, conf), peers);
}
+
+ /**
+ * For replication peer cluster key or endpoint class, null and empty string
is same. So here
+ * don't use {@link StringUtils#equals(CharSequence, CharSequence)} directly.
+ */
+ private boolean isStringEquals(String s1, String s2) {
+if (StringUtils.isBlank(s1)) {
+ return StringUtils.isBlank(s2);
+}
+return s1.equals(s2);
+ }
}
[03/40] hbase git commit: HBASE-19719 Fix checkstyle issues
HBASE-19719 Fix checkstyle issues
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1c32241a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1c32241a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1c32241a
Branch: refs/heads/branch-2
Commit: 1c32241a0066f19abdbd70d53f321beed39cac28
Parents: 1e69d7f
Author: zhangduo
Authored: Sat Jan 6 08:30:55 2018 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../hbase/replication/ReplicationStorageFactory.java | 2 +-
.../master/assignment/RegionTransitionProcedure.java | 4 ++--
.../hbase/master/procedure/RSProcedureDispatcher.java | 13 ++---
.../master/ReplicationPeerConfigUpgrader.java | 8
4 files changed, 13 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/1c32241a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
index 60d0749..462cfed 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
@@ -27,7 +27,7 @@ import org.apache.yetus.audience.InterfaceAudience;
* For now we only have zk based implementation.
*/
@InterfaceAudience.Private
-public class ReplicationStorageFactory {
+public final class ReplicationStorageFactory {
private ReplicationStorageFactory() {
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/1c32241a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
index a0e58f3..7ce7454 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
@@ -36,11 +36,11 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionTransitionState;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
-import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-
/**
* Base class for the Assign and Unassign Procedure.
*
http://git-wip-us.apache.org/repos/asf/hbase/blob/1c32241a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
index 57a4535..6c78914 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.master.procedure;
import java.io.IOException;
@@ -36,6 +35,12 @@ import org.apache.hadoop.ipc.RemoteException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.collect.ArrayListMultimap;
+import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
+import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback;
+import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
+import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
@@ -47,12 +52,6 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionR
import
org.apache.hadoop.hbase.shaded.protobuf.generated.A
[31/40] hbase git commit: HBASE-19697 Remove TestReplicationAdminUsingProcedure
HBASE-19697 Remove TestReplicationAdminUsingProcedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/029406c4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/029406c4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/029406c4
Branch: refs/heads/branch-2
Commit: 029406c45baec1f07a1524877f7c7d26c7c01363
Parents: 5655b3c
Author: zhangduo
Authored: Wed Jan 3 21:13:57 2018 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../TestReplicationAdminUsingProcedure.java | 225 ---
1 file changed, 225 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/029406c4/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
deleted file mode 100644
index 1300376..000
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.client.replication;
-
-import java.io.IOException;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
-import org.apache.hadoop.hbase.replication.TestReplicationBase;
-import org.apache.hadoop.hbase.testclassification.ClientTests;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.log4j.Logger;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
-import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
-
-@Category({ MediumTests.class, ClientTests.class })
-public class TestReplicationAdminUsingProcedure extends TestReplicationBase {
-
- private static final String PEER_ID = "2";
- private static final Logger LOG =
Logger.getLogger(TestReplicationAdminUsingProcedure.class);
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
-conf1.setInt("hbase.multihconnection.threads.max", 10);
-
-// Start the master & slave mini cluster.
-TestReplicationBase.setUpBeforeClass();
-
-// Remove the replication peer
-hbaseAdmin.removeReplicationPeer(PEER_ID);
- }
-
- private void loadData(int startRowKey, int endRowKey) throws IOException {
-for (int i = startRowKey; i < endRowKey; i++) {
- byte[] rowKey = Bytes.add(row, Bytes.toBytes(i));
- Put put = new Put(rowKey);
- put.addColumn(famName, null, Bytes.toBytes(i));
- htable1.put(put);
-}
- }
-
- private void waitForReplication(int expectedRows, int retries)
- throws IOException, InterruptedException {
-Scan scan;
-for (int i = 0; i < retries; i++) {
- scan = new Scan();
- if (i == retries - 1) {
-throw new IOException("Waited too much time for normal batch
replication");
- }
- try (ResultScanner scanner = htable2.getScanner(scan)) {
-int count = 0;
-for (Result res : scanner) {
- count++;
-}
-if (count != expectedRows) {
- LOG.info("Only got " + count + " rows, expected rows: " +
expectedRows);
- Thread.sleep(SLEEP_TIME);
-} else {
- return;
-}
- }
-}
- }
-
- @Before
- public void setUp() throws IOException {
-ReplicationPeerConfig rpc = new ReplicationPeerConfig();
-rpc.setCl
[22/40] hbase git commit: HBASE-19617 Remove ReplicationQueues, use ReplicationQueueStorage directly
HBASE-19617 Remove ReplicationQueues, use ReplicationQueueStorage directly
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d9b5eb3a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d9b5eb3a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d9b5eb3a
Branch: refs/heads/branch-2
Commit: d9b5eb3abbdd8f4c66cbd8e5811bd49a5599d4f4
Parents: b40c426
Author: zhangduo
Authored: Wed Dec 27 22:03:51 2017 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../hbase/replication/ReplicationFactory.java | 9 +-
.../hbase/replication/ReplicationQueues.java| 161 ---
.../replication/ReplicationQueuesArguments.java | 70 ---
.../replication/ReplicationQueuesZKImpl.java| 417 -
.../hbase/replication/ReplicationTableBase.java | 442 ---
.../replication/ReplicationTrackerZKImpl.java | 21 +-
.../replication/ZKReplicationQueueStorage.java | 22 +
.../replication/TestReplicationStateBasic.java | 131 +++---
.../replication/TestReplicationStateZKImpl.java | 41 +-
.../regionserver/DumpReplicationQueues.java | 15 +-
.../RecoveredReplicationSource.java | 17 +-
.../RecoveredReplicationSourceShipper.java | 22 +-
.../replication/regionserver/Replication.java | 20 +-
.../regionserver/ReplicationSource.java | 16 +-
.../ReplicationSourceInterface.java | 11 +-
.../regionserver/ReplicationSourceManager.java | 265 ++-
.../regionserver/ReplicationSyncUp.java | 13 +-
.../hbase/master/cleaner/TestLogsCleaner.java | 12 +-
.../cleaner/TestReplicationHFileCleaner.java| 23 +-
.../cleaner/TestReplicationZKNodeCleaner.java | 22 +-
.../replication/ReplicationSourceDummy.java | 6 +-
.../replication/TestReplicationSyncUpTool.java | 4 +-
.../TestReplicationSourceManager.java | 97 ++--
.../TestReplicationSourceManagerZkImpl.java | 57 +--
24 files changed, 356 insertions(+), 1558 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d9b5eb3a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
index 6c1c213..5e70e57 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
@@ -17,12 +17,11 @@
*/
package org.apache.hadoop.hbase.replication;
-import org.apache.commons.lang3.reflect.ConstructorUtils;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.yetus.audience.InterfaceAudience;
/**
* A factory class for instantiating replication objects that deal with
replication state.
@@ -30,12 +29,6 @@ import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
@InterfaceAudience.Private
public class ReplicationFactory {
- public static ReplicationQueues
getReplicationQueues(ReplicationQueuesArguments args)
- throws Exception {
-return (ReplicationQueues)
ConstructorUtils.invokeConstructor(ReplicationQueuesZKImpl.class,
- args);
- }
-
public static ReplicationPeers getReplicationPeers(ZKWatcher zk,
Configuration conf,
Abortable abortable) {
return getReplicationPeers(zk, conf, null, abortable);
http://git-wip-us.apache.org/repos/asf/hbase/blob/d9b5eb3a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
deleted file mode 100644
index a2d21f7..000
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licen
[16/40] hbase git commit: HBASE-19579 Add peer lock test for shell command list_locks
HBASE-19579 Add peer lock test for shell command list_locks
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/59cad95b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/59cad95b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/59cad95b
Branch: refs/heads/branch-2
Commit: 59cad95b5857658754614e65a96a800f902e8161
Parents: 15055a4
Author: Guanghao Zhang
Authored: Sat Dec 23 21:04:27 2017 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../src/main/protobuf/LockService.proto | 1 +
.../src/test/ruby/shell/list_locks_test.rb | 19 +++
2 files changed, 20 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/59cad95b/hbase-protocol-shaded/src/main/protobuf/LockService.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/LockService.proto
b/hbase-protocol-shaded/src/main/protobuf/LockService.proto
index b8d180c..0675070 100644
--- a/hbase-protocol-shaded/src/main/protobuf/LockService.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/LockService.proto
@@ -77,6 +77,7 @@ enum LockedResourceType {
NAMESPACE = 2;
TABLE = 3;
REGION = 4;
+ PEER = 5;
}
message LockedResource {
http://git-wip-us.apache.org/repos/asf/hbase/blob/59cad95b/hbase-shell/src/test/ruby/shell/list_locks_test.rb
--
diff --git a/hbase-shell/src/test/ruby/shell/list_locks_test.rb
b/hbase-shell/src/test/ruby/shell/list_locks_test.rb
index f465a6b..ef1c0ce 100644
--- a/hbase-shell/src/test/ruby/shell/list_locks_test.rb
+++ b/hbase-shell/src/test/ruby/shell/list_locks_test.rb
@@ -67,6 +67,25 @@ module Hbase
proc_id)
end
+define_test 'list peer locks' do
+ lock = create_exclusive_lock(0)
+ peer_id = '1'
+
+ @scheduler.waitPeerExclusiveLock(lock, peer_id)
+ output = capture_stdout { @list_locks.command }
+ @scheduler.wakePeerExclusiveLock(lock, peer_id)
+
+ assert_equal(
+"PEER(1)\n" \
+"Lock type: EXCLUSIVE, procedure: {" \
+
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
+ "\"procId\"=>\"0\", \"submittedTime\"=>\"0\",
\"state\"=>\"RUNNABLE\", " \
+ "\"lastUpdate\"=>\"0\", " \
+ "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\",
\"description\"=>\"description\"}]" \
+"}\n\n",
+output)
+end
+
define_test 'list server locks' do
lock = create_exclusive_lock(0)
[26/40] hbase git commit: HBASE-19633 Clean up the replication queues in the postPeerModification stage when removing a peer
HBASE-19633 Clean up the replication queues in the postPeerModification stage when removing a peer Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/525fef57 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/525fef57 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/525fef57 Branch: refs/heads/branch-2 Commit: 525fef572e73c99c6acb98f8daabdda5fd2a36f3 Parents: 1e36a84 Author: zhangduo Authored: Tue Jan 2 09:57:23 2018 +0800 Committer: zhangduo Committed: Fri Mar 9 20:55:48 2018 +0800 -- .../replication/ReplicationPeerConfig.java | 2 +- .../replication/VerifyReplication.java | 34 ++ .../hbase/replication/ReplicationPeers.java | 32 +++-- .../replication/ZKReplicationQueueStorage.java | 3 +- .../replication/ZKReplicationStorageBase.java | 4 +-- .../replication/TestReplicationStateBasic.java | 10 ++ .../master/replication/AddPeerProcedure.java| 5 ++- .../replication/DisablePeerProcedure.java | 3 +- .../master/replication/EnablePeerProcedure.java | 3 +- .../master/replication/ModifyPeerProcedure.java | 34 ++ .../replication/RefreshPeerProcedure.java | 17 - .../master/replication/RemovePeerProcedure.java | 7 ++-- .../replication/ReplicationPeerManager.java | 31 +++- .../replication/UpdatePeerConfigProcedure.java | 3 +- .../RemoteProcedureResultReporter.java | 3 +- .../regionserver/RefreshPeerCallable.java | 5 +-- .../regionserver/ReplicationSourceManager.java | 38 +++- .../TestReplicationAdminUsingProcedure.java | 7 ++-- 18 files changed, 124 insertions(+), 117 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/525fef57/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java index b80ee16..fdae288 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java @@ -27,8 +27,8 @@ import java.util.Set; import java.util.TreeMap; import org.apache.hadoop.hbase.TableName; -import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.yetus.audience.InterfaceAudience; /** * A configuration for the replication peer cluster. http://git-wip-us.apache.org/repos/asf/hbase/blob/525fef57/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java -- diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java index f0070f0..fe45762 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java @@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.mapreduce.replication; import java.io.IOException; import java.util.Arrays; import java.util.UUID; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileSystem; @@ -45,13 +44,14 @@ import org.apache.hadoop.hbase.filter.PrefixFilter; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.TableInputFormat; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; -import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat; import org.apache.hadoop.hbase.mapreduce.TableMapper; +import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat; import org.apache.hadoop.hbase.mapreduce.TableSplit; import org.apache.hadoop.hbase.replication.ReplicationException; -import org.apache.hadoop.hbase.replication.ReplicationFactory; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; +import org.apache.hadoop.hbase.replication.ReplicationPeerStorage; import org.apache.hadoop.hbase.replication.ReplicationPeers; +import org.apache.hadoop.hbase.replication.ReplicationStorageFactory; import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; @@ -66,6 +66,7 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.slf4j
[21/40] hbase git commit: HBASE-19617 Remove ReplicationQueues, use ReplicationQueueStorage directly
http://git-wip-us.apache.org/repos/asf/hbase/blob/d9b5eb3a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index b8d80d2..5b7bab8 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -33,17 +33,20 @@ import java.util.TreeSet;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
@@ -56,7 +59,7 @@ import
org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
-import org.apache.hadoop.hbase.replication.ReplicationQueues;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationTracker;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
@@ -91,7 +94,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
private final List sources;
// List of all the sources we got from died RSs
private final List oldsources;
- private final ReplicationQueues replicationQueues;
+ private final ReplicationQueueStorage queueStorage;
private final ReplicationTracker replicationTracker;
private final ReplicationPeers replicationPeers;
// UUID for this cluster
@@ -124,7 +127,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
/**
* Creates a replication manager and sets the watch on all the other
registered region servers
- * @param replicationQueues the interface for manipulating replication queues
+ * @param queueStorage the interface for manipulating replication queues
* @param replicationPeers
* @param replicationTracker
* @param conf the configuration to use
@@ -134,14 +137,14 @@ public class ReplicationSourceManager implements
ReplicationListener {
* @param oldLogDir the directory where old logs are archived
* @param clusterId
*/
- public ReplicationSourceManager(ReplicationQueues replicationQueues,
+ public ReplicationSourceManager(ReplicationQueueStorage queueStorage,
ReplicationPeers replicationPeers, ReplicationTracker
replicationTracker, Configuration conf,
Server server, FileSystem fs, Path logDir, Path oldLogDir, UUID
clusterId,
WALFileLengthProvider walFileLengthProvider) throws IOException {
//CopyOnWriteArrayList is thread-safe.
//Generally, reading is more than modifying.
this.sources = new CopyOnWriteArrayList<>();
-this.replicationQueues = replicationQueues;
+this.queueStorage = queueStorage;
this.replicationPeers = replicationPeers;
this.replicationTracker = replicationTracker;
this.server = server;
@@ -174,6 +177,19 @@ public class ReplicationSourceManager implements
ReplicationListener {
HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT);
}
+ @FunctionalInterface
+ private interface ReplicationQueueOperation {
+void exec() throws ReplicationException;
+ }
+
+ private void abortWhenFail(ReplicationQueueOperation op) {
+try {
+ op.exec();
+} catch (ReplicationException e) {
+ server.abort("Failed to operate on replication queue", e);
+}
+ }
+
/**
* Provide the id of the peer and a log key and this method will figure which
* wal it belongs to and will log, for this region server, the current
@@ -185,12 +201,13 @@ public class ReplicationSourceManager implements
ReplicationListener {
* @param queueRecovered indicates if this queue comes from another region
server
* @param holdLogInZK if true then the log is retained in
[34/40] hbase git commit: HBASE-19636 All rs should already start work with the new peer change when replication peer procedure is finished
HBASE-19636 All rs should already start work with the new peer change when
replication peer procedure is finished
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d36aacdf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d36aacdf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d36aacdf
Branch: refs/heads/branch-2
Commit: d36aacdf9ea856fbe981073ac079181143e81c09
Parents: 53b18fe
Author: Guanghao Zhang
Authored: Thu Jan 4 16:58:01 2018 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../replication/ReplicationPeerConfig.java | 1 -
.../hbase/replication/ReplicationPeerImpl.java | 4 +-
.../hbase/replication/ReplicationQueueInfo.java | 23 +-
.../hbase/replication/ReplicationUtils.java | 56 ++
.../replication/TestReplicationStateZKImpl.java | 21 -
.../regionserver/ReplicationSourceService.java | 3 +-
.../regionserver/PeerProcedureHandler.java | 3 +
.../regionserver/PeerProcedureHandlerImpl.java | 50 +-
.../RecoveredReplicationSource.java | 6 +-
.../RecoveredReplicationSourceShipper.java | 8 +-
.../replication/regionserver/Replication.java | 11 +-
.../regionserver/ReplicationSource.java | 34 +-
.../regionserver/ReplicationSourceFactory.java | 4 +-
.../ReplicationSourceInterface.java | 8 +-
.../regionserver/ReplicationSourceManager.java | 827 ++-
.../regionserver/ReplicationSourceShipper.java | 6 +-
.../ReplicationSourceWALReader.java | 2 +-
.../replication/ReplicationSourceDummy.java | 2 +-
.../replication/TestNamespaceReplication.java | 57 +-
.../TestReplicationSourceManager.java | 5 +-
20 files changed, 622 insertions(+), 509 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d36aacdf/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
index fdae288..bf8d030 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
-
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.yetus.audience.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/hbase/blob/d36aacdf/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
index 3e17025..604e0bb 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -28,6 +27,7 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public class ReplicationPeerImpl implements ReplicationPeer {
+
private final Configuration conf;
private final String id;
http://git-wip-us.apache.org/repos/asf/hbase/blob/d36aacdf/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
index ecd888f..cd65f9b 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
@@ -29,7 +29,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.ServerName;
/**
- * This class is responsible for the parsing logic for a znode representing a
queue.
+ * This class is responsible for the parsing logic for a queue id representing
a queue.
* It will extract the peerId if it's recovered as well as
[28/40] hbase git commit: HBASE-19687 Move the logic in ReplicationZKNodeCleaner to ReplicationChecker and remove ReplicationZKNodeCleanerChore
HBASE-19687 Move the logic in ReplicationZKNodeCleaner to ReplicationChecker
and remove ReplicationZKNodeCleanerChore
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b84fbde1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b84fbde1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b84fbde1
Branch: refs/heads/branch-2
Commit: b84fbde17564878050ebb6fd7ca8dd5ce583dd93
Parents: 13318dd
Author: zhangduo
Authored: Wed Jan 3 09:39:44 2018 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../replication/VerifyReplication.java | 6 +-
.../hbase/replication/ReplicationPeers.java | 26 +--
.../hbase/replication/ReplicationUtils.java | 38
.../replication/TestReplicationStateBasic.java | 2 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 13 --
.../cleaner/ReplicationZKNodeCleaner.java | 192 ---
.../cleaner/ReplicationZKNodeCleanerChore.java | 54 --
.../replication/ReplicationPeerManager.java | 18 +-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 11 +-
.../hbase/util/hbck/ReplicationChecker.java | 109 +++
.../cleaner/TestReplicationZKNodeCleaner.java | 115 ---
.../hbase/util/TestHBaseFsckReplication.java| 101 ++
.../hadoop/hbase/util/hbck/HbckTestingUtil.java | 6 +-
13 files changed, 226 insertions(+), 465 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/b84fbde1/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
--
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index fe45762..fac4875 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -50,8 +50,8 @@ import org.apache.hadoop.hbase.mapreduce.TableSplit;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
-import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.replication.ReplicationUtils;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -345,10 +345,10 @@ public class VerifyReplication extends Configured
implements Tool {
}
});
ReplicationPeerStorage storage =
- ReplicationStorageFactory.getReplicationPeerStorage(localZKW, conf);
+ReplicationStorageFactory.getReplicationPeerStorage(localZKW, conf);
ReplicationPeerConfig peerConfig = storage.getPeerConfig(peerId);
return Pair.newPair(peerConfig,
-ReplicationPeers.getPeerClusterConfiguration(peerConfig, conf));
+ReplicationUtils.getPeerClusterConfiguration(peerConfig, conf));
} catch (ReplicationException e) {
throw new IOException("An error occurred while trying to connect to the
remove peer cluster",
e);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b84fbde1/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
index 45940a5..fcbc350 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
@@ -17,14 +17,11 @@
*/
package org.apache.hadoop.hbase.replication;
-import java.io.IOException;
import java.util.Collections;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.CompoundConfiguration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
@@ -106,25 +103,6 @@ public class ReplicationPeers {
return Collections.unmodifiableSet(peerCache.keySet());
}
- public static Config
[10/40] hbase git commit: HBASE-19564 Procedure id is missing in the response of peer related operations
HBASE-19564 Procedure id is missing in the response of peer related operations
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7c2a0d7e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7c2a0d7e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7c2a0d7e
Branch: refs/heads/branch-2
Commit: 7c2a0d7e217e93768751bed6ea22fa6dba9ef5f6
Parents: 750b7d8
Author: zhangduo
Authored: Wed Dec 20 20:57:37 2017 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../hadoop/hbase/master/MasterRpcServices.java | 24 ++--
.../master/replication/ModifyPeerProcedure.java | 4 +---
2 files changed, 13 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/7c2a0d7e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 7bd355a..6ce0d39 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -1868,10 +1868,10 @@ public class MasterRpcServices extends RSRpcServices
public AddReplicationPeerResponse addReplicationPeer(RpcController
controller,
AddReplicationPeerRequest request) throws ServiceException {
try {
- master.addReplicationPeer(request.getPeerId(),
-ReplicationPeerConfigUtil.convert(request.getPeerConfig()),
request.getPeerState()
-.getState().equals(ReplicationState.State.ENABLED));
- return AddReplicationPeerResponse.newBuilder().build();
+ long procId = master.addReplicationPeer(request.getPeerId(),
+ReplicationPeerConfigUtil.convert(request.getPeerConfig()),
+
request.getPeerState().getState().equals(ReplicationState.State.ENABLED));
+ return AddReplicationPeerResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
throw new ServiceException(e);
}
@@ -1881,8 +1881,8 @@ public class MasterRpcServices extends RSRpcServices
public RemoveReplicationPeerResponse removeReplicationPeer(RpcController
controller,
RemoveReplicationPeerRequest request) throws ServiceException {
try {
- master.removeReplicationPeer(request.getPeerId());
- return RemoveReplicationPeerResponse.newBuilder().build();
+ long procId = master.removeReplicationPeer(request.getPeerId());
+ return
RemoveReplicationPeerResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
throw new ServiceException(e);
}
@@ -1892,8 +1892,8 @@ public class MasterRpcServices extends RSRpcServices
public EnableReplicationPeerResponse enableReplicationPeer(RpcController
controller,
EnableReplicationPeerRequest request) throws ServiceException {
try {
- master.enableReplicationPeer(request.getPeerId());
- return EnableReplicationPeerResponse.newBuilder().build();
+ long procId = master.enableReplicationPeer(request.getPeerId());
+ return
EnableReplicationPeerResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
throw new ServiceException(e);
}
@@ -1903,8 +1903,8 @@ public class MasterRpcServices extends RSRpcServices
public DisableReplicationPeerResponse disableReplicationPeer(RpcController
controller,
DisableReplicationPeerRequest request) throws ServiceException {
try {
- master.disableReplicationPeer(request.getPeerId());
- return DisableReplicationPeerResponse.newBuilder().build();
+ long procId = master.disableReplicationPeer(request.getPeerId());
+ return
DisableReplicationPeerResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
throw new ServiceException(e);
}
@@ -1930,9 +1930,9 @@ public class MasterRpcServices extends RSRpcServices
public UpdateReplicationPeerConfigResponse
updateReplicationPeerConfig(RpcController controller,
UpdateReplicationPeerConfigRequest request) throws ServiceException {
try {
- master.updateReplicationPeerConfig(request.getPeerId(),
+ long procId = master.updateReplicationPeerConfig(request.getPeerId(),
ReplicationPeerConfigUtil.convert(request.getPeerConfig()));
- return UpdateReplicationPeerConfigResponse.newBuilder().build();
+ return
UpdateReplicationPeerConfigResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
throw new Servic
[27/40] hbase git commit: HBASE-19623 Create replication endpoint asynchronously when adding a replication source
HBASE-19623 Create replication endpoint asynchronously when adding a
replication source
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5d922a5e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5d922a5e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5d922a5e
Branch: refs/heads/branch-2
Commit: 5d922a5e0c3fd5a71a9d036ac3d5f34e270e7733
Parents: 525fef5
Author: zhangduo
Authored: Tue Jan 2 13:25:58 2018 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../hbase/replication/ReplicationPeer.java | 8 ++
.../hbase/replication/ReplicationPeers.java | 18 +--
.../replication/ZKReplicationPeerStorage.java | 7 +-
.../replication/TestReplicationStateBasic.java | 20 +---
.../TestZKReplicationPeerStorage.java | 14 +--
.../HBaseInterClusterReplicationEndpoint.java | 17 ++-
.../RecoveredReplicationSource.java | 13 +--
.../regionserver/ReplicationSource.java | 110 +++
.../ReplicationSourceInterface.java | 8 +-
.../regionserver/ReplicationSourceManager.java | 47 +---
.../client/TestAsyncReplicationAdminApi.java| 2 -
.../replication/TestReplicationAdmin.java | 2 -
.../replication/ReplicationSourceDummy.java | 7 +-
.../replication/TestReplicationSource.java | 5 +-
.../TestReplicationSourceManager.java | 8 +-
15 files changed, 116 insertions(+), 170 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/5d922a5e/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
index 4846018..2da3cce 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
@@ -54,6 +54,14 @@ public interface ReplicationPeer {
PeerState getPeerState();
/**
+ * Test whether the peer is enabled.
+ * @return {@code true} if enabled, otherwise {@code false}.
+ */
+ default boolean isPeerEnabled() {
+return getPeerState() == PeerState.ENABLED;
+ }
+
+ /**
* Get the peer config object
* @return the ReplicationPeerConfig for this peer
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/5d922a5e/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
index 422801b..45940a5 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.replication;
import java.io.IOException;
+import java.util.Collections;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -86,21 +87,6 @@ public class ReplicationPeers {
}
/**
- * Get the peer state for the specified connected remote slave cluster. The
value might be read
- * from cache, so it is recommended to use {@link #peerStorage } to read
storage directly if
- * reading the state after enabling or disabling it.
- * @param peerId a short that identifies the cluster
- * @return true if replication is enabled, false otherwise.
- */
- public boolean isPeerEnabled(String peerId) {
-ReplicationPeer replicationPeer = this.peerCache.get(peerId);
-if (replicationPeer == null) {
- throw new IllegalArgumentException("Peer with id= " + peerId + " is not
cached");
-}
-return replicationPeer.getPeerState() == PeerState.ENABLED;
- }
-
- /**
* Returns the ReplicationPeerImpl for the specified cached peer. This
ReplicationPeer will
* continue to track changes to the Peer's state and config. This method
returns null if no peer
* has been cached with the given peerId.
@@ -117,7 +103,7 @@ public class ReplicationPeers {
* @return a Set of Strings for peerIds
*/
public Set getAllPeerIds() {
-return peerCache.keySet();
+return Collections.unmodifiableSet(peerCache.keySet());
}
public static Configuration
getPeerClusterConfiguration(ReplicationPeerConfig peerConfig,
http://git-wip-us.apache.org/repos/asf/hbase/blob/5d922a5e/hbase-replication/
[39/40] hbase git commit: HBASE-19923 Reset peer state and config when refresh replication source failed
HBASE-19923 Reset peer state and config when refresh replication source failed
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/55d0d302
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/55d0d302
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/55d0d302
Branch: refs/heads/branch-2
Commit: 55d0d3026a3d2733fbffd48237c5d7729e1e234d
Parents: d1e775e
Author: Guanghao Zhang
Authored: Tue Feb 6 14:58:39 2018 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../hbase/replication/ReplicationPeerImpl.java | 4 ++--
.../regionserver/PeerProcedureHandlerImpl.java | 24
2 files changed, 22 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/55d0d302/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
index 604e0bb..d656466 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
@@ -54,11 +54,11 @@ public class ReplicationPeerImpl implements ReplicationPeer
{
this.peerConfigListeners = new ArrayList<>();
}
- void setPeerState(boolean enabled) {
+ public void setPeerState(boolean enabled) {
this.peerState = enabled ? PeerState.ENABLED : PeerState.DISABLED;
}
- void setPeerConfig(ReplicationPeerConfig peerConfig) {
+ public void setPeerConfig(ReplicationPeerConfig peerConfig) {
this.peerConfig = peerConfig;
peerConfigListeners.forEach(listener ->
listener.peerConfigUpdated(peerConfig));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/55d0d302/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
index ce8fdae..a02d181 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
@@ -62,18 +62,26 @@ public class PeerProcedureHandlerImpl implements
PeerProcedureHandler {
private void refreshPeerState(String peerId) throws ReplicationException,
IOException {
PeerState newState;
Lock peerLock = peersLock.acquireLock(peerId);
+ReplicationPeerImpl peer = null;
+PeerState oldState = null;
+boolean success = false;
try {
- ReplicationPeerImpl peer =
replicationSourceManager.getReplicationPeers().getPeer(peerId);
+ peer = replicationSourceManager.getReplicationPeers().getPeer(peerId);
if (peer == null) {
throw new ReplicationException("Peer with id=" + peerId + " is not
cached.");
}
- PeerState oldState = peer.getPeerState();
+ oldState = peer.getPeerState();
newState =
replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
// RS need to start work with the new replication state change
if (oldState.equals(PeerState.ENABLED) &&
newState.equals(PeerState.DISABLED)) {
replicationSourceManager.refreshSources(peerId);
}
+ success = true;
} finally {
+ if (!success && peer != null) {
+// Reset peer state if refresh source failed
+peer.setPeerState(oldState.equals(PeerState.ENABLED));
+ }
peerLock.unlock();
}
}
@@ -91,19 +99,27 @@ public class PeerProcedureHandlerImpl implements
PeerProcedureHandler {
@Override
public void updatePeerConfig(String peerId) throws ReplicationException,
IOException {
Lock peerLock = peersLock.acquireLock(peerId);
+ReplicationPeerImpl peer = null;
+ReplicationPeerConfig oldConfig = null;
+boolean success = false;
try {
- ReplicationPeerImpl peer =
replicationSourceManager.getReplicationPeers().getPeer(peerId);
+ peer = replicationSourceManager.getReplicationPeers().getPeer(peerId);
if (peer == null) {
throw new ReplicationException("Peer with id=" + peerId + " is not
cached.");
}
- ReplicationPeerConfig oldConfig = peer.getPeerConfig();
+ oldConfig = peer.getPeerConfig();
ReplicationPeerConfig newConfig =
replicationSource
[25/40] hbase git commit: HBASE-19622 Reimplement ReplicationPeers with the new replication storage interface
HBASE-19622 Reimplement ReplicationPeers with the new replication storage
interface
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1e36a84a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1e36a84a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1e36a84a
Branch: refs/heads/branch-2
Commit: 1e36a84afc388987592f1ec003bfe6a665da4ffd
Parents: 4c6942d
Author: huzheng
Authored: Tue Dec 26 16:46:10 2017 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../replication/ReplicationPeerConfigUtil.java | 10 +-
.../replication/VerifyReplication.java | 9 +-
.../hbase/replication/ReplicationFactory.java | 10 +-
.../hbase/replication/ReplicationPeerImpl.java | 60 +-
.../replication/ReplicationPeerStorage.java | 3 +-
.../hbase/replication/ReplicationPeers.java | 238
.../replication/ReplicationPeersZKImpl.java | 552 ---
.../replication/ZKReplicationPeerStorage.java | 12 +-
.../replication/ZKReplicationStorageBase.java | 3 +-
.../replication/TestReplicationStateBasic.java | 125 ++---
.../replication/TestReplicationStateZKImpl.java | 2 +-
.../TestZKReplicationPeerStorage.java | 12 +-
.../cleaner/ReplicationZKNodeCleaner.java | 57 +-
.../replication/ReplicationPeerManager.java | 6 +-
.../regionserver/DumpReplicationQueues.java | 2 +-
.../regionserver/PeerProcedureHandlerImpl.java | 49 +-
.../replication/regionserver/Replication.java | 2 +-
.../regionserver/ReplicationSource.java | 7 +-
.../regionserver/ReplicationSourceManager.java | 44 +-
.../cleaner/TestReplicationHFileCleaner.java| 7 +-
.../replication/TestMultiSlaveReplication.java | 2 -
.../TestReplicationTrackerZKImpl.java | 26 +-
.../TestReplicationSourceManager.java | 17 +-
.../hadoop/hbase/HBaseZKTestingUtility.java | 3 +-
24 files changed, 307 insertions(+), 951 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/1e36a84a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
index 022bf64..a234a9b 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
@@ -247,22 +247,22 @@ public final class ReplicationPeerConfigUtil {
public static ReplicationPeerConfig parsePeerFrom(final byte[] bytes)
throws DeserializationException {
if (ProtobufUtil.isPBMagicPrefix(bytes)) {
- int pblen = ProtobufUtil.lengthOfPBMagic();
+ int pbLen = ProtobufUtil.lengthOfPBMagic();
ReplicationProtos.ReplicationPeer.Builder builder =
ReplicationProtos.ReplicationPeer.newBuilder();
ReplicationProtos.ReplicationPeer peer;
try {
-ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
+ProtobufUtil.mergeFrom(builder, bytes, pbLen, bytes.length - pbLen);
peer = builder.build();
} catch (IOException e) {
throw new DeserializationException(e);
}
return convert(peer);
} else {
- if (bytes.length > 0) {
-return
ReplicationPeerConfig.newBuilder().setClusterKey(Bytes.toString(bytes)).build();
+ if (bytes == null || bytes.length <= 0) {
+throw new DeserializationException("Bytes to deserialize should not be
empty.");
}
- return ReplicationPeerConfig.newBuilder().setClusterKey("").build();
+ return
ReplicationPeerConfig.newBuilder().setClusterKey(Bytes.toString(bytes)).build();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/1e36a84a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
--
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index 09d4b4b..f0070f0 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -339,15 +339,10 @@ public class VerifyReplication extends Configured
implements Tool {
@Override public boolean isAborted()
[30/40] hbase git commit: HBASE-19661 Replace ReplicationStateZKBase with ZKReplicationStorageBase
HBASE-19661 Replace ReplicationStateZKBase with ZKReplicationStorageBase
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5655b3c0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5655b3c0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5655b3c0
Branch: refs/heads/branch-2
Commit: 5655b3c0ca86dda899de4115959a0168e8bc9eee
Parents: b84fbde
Author: huzheng
Authored: Fri Dec 29 15:55:28 2017 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../hbase/replication/ReplicationFactory.java | 5 +-
.../replication/ReplicationStateZKBase.java | 159 ---
.../replication/ReplicationTrackerZKImpl.java | 21 +--
.../replication/ZKReplicationPeerStorage.java | 24 ++-
.../replication/ZKReplicationStorageBase.java | 13 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 4 +-
.../master/ReplicationPeerConfigUpgrader.java | 128 +++
.../regionserver/DumpReplicationQueues.java | 18 +--
.../replication/regionserver/Replication.java | 3 +-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 3 +-
.../TestReplicationTrackerZKImpl.java | 3 +-
.../replication/master/TestTableCFsUpdater.java | 41 ++---
.../TestReplicationSourceManager.java | 6 +-
13 files changed, 136 insertions(+), 292 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/5655b3c0/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
index 6c66aff..2a970ba 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
@@ -33,9 +33,8 @@ public class ReplicationFactory {
return new ReplicationPeers(zk, conf);
}
- public static ReplicationTracker getReplicationTracker(ZKWatcher zookeeper,
- final ReplicationPeers replicationPeers, Configuration conf, Abortable
abortable,
+ public static ReplicationTracker getReplicationTracker(ZKWatcher zookeeper,
Abortable abortable,
Stoppable stopper) {
-return new ReplicationTrackerZKImpl(zookeeper, replicationPeers, conf,
abortable, stopper);
+return new ReplicationTrackerZKImpl(zookeeper, abortable, stopper);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/5655b3c0/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
deleted file mode 100644
index a48683e..000
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hbase.thirdparty.com.google.protobuf.CodedOutputStream;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
-import org.apache.hadoop.hbase.zookeeper.ZKConfig;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
-import org.apache.yetus.audience.Interfac
[05/40] hbase git commit: HBASE-19520 Add UTs for the new lock type PEER
HBASE-19520 Add UTs for the new lock type PEER
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9ead5934
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9ead5934
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9ead5934
Branch: refs/heads/branch-2
Commit: 9ead5934c0bd82328fdd15516944811008891949
Parents: 7c2a0d7
Author: Guanghao Zhang
Authored: Wed Dec 20 16:43:38 2017 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../procedure/MasterProcedureScheduler.java | 9 +-
.../procedure/TestMasterProcedureScheduler.java | 65 -
...TestMasterProcedureSchedulerConcurrency.java | 135 +++
3 files changed, 201 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/9ead5934/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
index 58f9ccd..b18dd6c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
@@ -277,6 +277,13 @@ public class MasterProcedureScheduler extends
AbstractProcedureScheduler {
while (tableIter.hasNext()) {
count += tableIter.next().size();
}
+
+// Peer queues
+final AvlTreeIterator peerIter = new AvlTreeIterator<>(peerMap);
+while (peerIter.hasNext()) {
+ count += peerIter.next().size();
+}
+
return count;
}
@@ -807,7 +814,7 @@ public class MasterProcedureScheduler extends
AbstractProcedureScheduler {
* @see #wakePeerExclusiveLock(Procedure, String)
* @param procedure the procedure trying to acquire the lock
* @param peerId peer to lock
- * @return true if the procedure has to wait for the per to be available
+ * @return true if the procedure has to wait for the peer to be available
*/
public boolean waitPeerExclusiveLock(Procedure procedure, String peerId) {
schedLock();
http://git-wip-us.apache.org/repos/asf/hbase/blob/9ead5934/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
index 160f4d2..65757db 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
@@ -908,6 +908,27 @@ public class TestMasterProcedureScheduler {
}
}
+ public static class TestPeerProcedure extends TestProcedure implements
PeerProcedureInterface {
+private final String peerId;
+private final PeerOperationType opType;
+
+public TestPeerProcedure(long procId, String peerId, PeerOperationType
opType) {
+ super(procId);
+ this.peerId = peerId;
+ this.opType = opType;
+}
+
+@Override
+public String getPeerId() {
+ return peerId;
+}
+
+@Override
+public PeerOperationType getPeerOperationType() {
+ return opType;
+}
+ }
+
private static LockProcedure createLockProcedure(LockType lockType, long
procId) throws Exception {
LockProcedure procedure = new LockProcedure();
@@ -930,22 +951,19 @@ public class TestMasterProcedureScheduler {
return createLockProcedure(LockType.SHARED, procId);
}
- private static void assertLockResource(LockedResource resource,
- LockedResourceType resourceType, String resourceName)
- {
+ private static void assertLockResource(LockedResource resource,
LockedResourceType resourceType,
+ String resourceName) {
assertEquals(resourceType, resource.getResourceType());
assertEquals(resourceName, resource.getResourceName());
}
- private static void assertExclusiveLock(LockedResource resource,
Procedure procedure)
- {
+ private static void assertExclusiveLock(LockedResource resource,
Procedure procedure) {
assertEquals(LockType.EXCLUSIVE, resource.getLockType());
assertEquals(procedure, resource.getExclusiveLockOwnerProcedure());
assertEquals(0, resource.getSharedLockCount());
}
- private static void assertSharedLock(LockedResource resource, int lockCount)
- {
+ private
[14/40] hbase git commit: HBASE-19599 Remove ReplicationQueuesClient, use ReplicationQueueStorage directly
http://git-wip-us.apache.org/repos/asf/hbase/blob/15055a42/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
index 6e27a21..d8f9625 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
@@ -21,13 +21,13 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.stream.Collectors;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
@@ -48,17 +48,18 @@ import
org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationQueues;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments;
+import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.ReplicationTracker;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import
org.apache.hbase.thirdparty.com.google.common.util.concurrent.AtomicLongMap;
/**
@@ -303,57 +304,53 @@ public class DumpReplicationQueues extends Configured
implements Tool {
}
public String dumpQueues(ClusterConnection connection, ZKWatcher zkw,
Set peerIds,
- boolean hdfs) throws Exception {
-ReplicationQueuesClient queuesClient;
+ boolean hdfs) throws Exception {
+ReplicationQueueStorage queueStorage;
ReplicationPeers replicationPeers;
ReplicationQueues replicationQueues;
ReplicationTracker replicationTracker;
-ReplicationQueuesClientArguments replicationArgs =
-new ReplicationQueuesClientArguments(getConf(), new
WarnOnlyAbortable(), zkw);
+ReplicationQueuesArguments replicationArgs =
+new ReplicationQueuesArguments(getConf(), new WarnOnlyAbortable(),
zkw);
StringBuilder sb = new StringBuilder();
-queuesClient =
ReplicationFactory.getReplicationQueuesClient(replicationArgs);
-queuesClient.init();
+queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw,
getConf());
replicationQueues =
ReplicationFactory.getReplicationQueues(replicationArgs);
-replicationPeers = ReplicationFactory.getReplicationPeers(zkw, getConf(),
queuesClient, connection);
+replicationPeers =
+ReplicationFactory.getReplicationPeers(zkw, getConf(), queueStorage,
connection);
replicationTracker = ReplicationFactory.getReplicationTracker(zkw,
replicationPeers, getConf(),
new WarnOnlyAbortable(), new WarnOnlyStoppable());
-List liveRegionServers =
replicationTracker.getListOfRegionServers();
+Set liveRegionServers = new
HashSet<>(replicationTracker.getListOfRegionServers());
// Loops each peer on each RS and dumps the queues
-try {
- List regionservers = queuesClient.getListOfReplicators();
- if (regionservers == null || regionservers.isEmpty()) {
-return sb.toString();
+List regionservers = queueStorage.getListOfReplicators();
+if (regionservers == null || regionservers.isEmpty()) {
+ return sb.toString();
+}
+for (ServerName regionserver : regionservers) {
+ List queueIds = queueStorage.getAllQueues(regionserver);
+ replicationQueues.init(regionserver.getServerName());
+ if (!liveRegionServers.contains(regionserver.getServerName())) {
+deadRegionServers.add(regionserver.getServerName());
}
- for (String regionserver : regionservers) {
-List queueIds = queuesClient.getAllQueues(regionserver);
-replicationQueues.init(regionserver);
-if (!liveRegionServers.contains(regionserver)) {
- deadRegionServers.add(regionserver);
-}
-for (String
[35/40] hbase git commit: HBASE-19707 Race in start and terminate of a replication source after we async start replicatione endpoint
HBASE-19707 Race in start and terminate of a replication source after we async
start replicatione endpoint
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0165455d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0165455d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0165455d
Branch: refs/heads/branch-2
Commit: 0165455d3490f9215fa75e8857cc4869632f2ac1
Parents: d36aacd
Author: zhangduo
Authored: Fri Jan 5 18:28:44 2018 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../RecoveredReplicationSource.java | 16 +-
.../regionserver/ReplicationSource.java | 203 ++-
2 files changed, 116 insertions(+), 103 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/0165455d/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
index 1be9a88..3cae0f2 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
@@ -68,7 +68,7 @@ public class RecoveredReplicationSource extends
ReplicationSource {
LOG.debug("Someone has beat us to start a worker thread for wal group "
+ walGroupId);
} else {
LOG.debug("Starting up worker for wal group " + walGroupId);
- worker.startup(getUncaughtExceptionHandler());
+ worker.startup(this::uncaughtException);
worker.setWALReader(
startNewWALReader(worker.getName(), walGroupId, queue,
worker.getStartPosition()));
workerThreads.put(walGroupId, worker);
@@ -76,13 +76,13 @@ public class RecoveredReplicationSource extends
ReplicationSource {
}
@Override
- protected ReplicationSourceWALReader startNewWALReader(String threadName,
- String walGroupId, PriorityBlockingQueue queue, long
startPosition) {
-ReplicationSourceWALReader walReader = new
RecoveredReplicationSourceWALReader(fs,
-conf, queue, startPosition, walEntryFilter, this);
-Threads.setDaemonThreadRunning(walReader, threadName
-+ ".replicationSource.replicationWALReaderThread." + walGroupId + ","
+ queueId,
- getUncaughtExceptionHandler());
+ protected ReplicationSourceWALReader startNewWALReader(String threadName,
String walGroupId,
+ PriorityBlockingQueue queue, long startPosition) {
+ReplicationSourceWALReader walReader =
+ new RecoveredReplicationSourceWALReader(fs, conf, queue, startPosition,
walEntryFilter, this);
+Threads.setDaemonThreadRunning(walReader,
+ threadName + ".replicationSource.replicationWALReaderThread." +
walGroupId + "," + queueId,
+ this::uncaughtException);
return walReader;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0165455d/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 0092251..09b6cc1 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -75,7 +75,7 @@ import
org.apache.hbase.thirdparty.com.google.common.collect.Lists;
*
*/
@InterfaceAudience.Private
-public class ReplicationSource extends Thread implements
ReplicationSourceInterface {
+public class ReplicationSource implements ReplicationSourceInterface {
private static final Logger LOG =
LoggerFactory.getLogger(ReplicationSource.class);
// Queues of logs to process, entry in format of walGroupId->queue,
@@ -114,10 +114,8 @@ public class ReplicationSource extends Thread implements
ReplicationSourceInterf
private MetricsSource metrics;
// WARN threshold for the number of queued logs, defaults to 2
private int logQueueWarnThreshold;
- // whether the replication endpoint has been initialized
- private volatile boolean endpointInitialized = false;
// ReplicationEndpoint which will handle the actual replication
- private ReplicationEndpoint replicationEndpoint;
+ private volatile ReplicationEndpoint replicationEndpoint;
// A filter (or a chain of fil
[08/40] hbase git commit: HBASE-19536 Client side changes for moving peer modification from zk watcher to procedure
HBASE-19536 Client side changes for moving peer modification from zk watcher to
procedure
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/750b7d8d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/750b7d8d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/750b7d8d
Branch: refs/heads/branch-2
Commit: 750b7d8dbe3d0f3a4d5eb37f26990d1932c75f02
Parents: 5e410d8
Author: Guanghao Zhang
Authored: Tue Dec 19 15:50:57 2017 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../org/apache/hadoop/hbase/client/Admin.java | 87 ++-
.../apache/hadoop/hbase/client/HBaseAdmin.java | 149 ++-
.../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 82 +-
3 files changed, 238 insertions(+), 80 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/750b7d8d/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index 40dac2f..b8546fa 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -2473,7 +2473,7 @@ public interface Admin extends Abortable, Closeable {
/**
* Add a new replication peer for replicating data to slave cluster.
* @param peerId a short name that identifies the peer
- * @param peerConfig configuration for the replication slave cluster
+ * @param peerConfig configuration for the replication peer
* @throws IOException if a remote or network exception occurs
*/
default void addReplicationPeer(String peerId, ReplicationPeerConfig
peerConfig)
@@ -2484,7 +2484,7 @@ public interface Admin extends Abortable, Closeable {
/**
* Add a new replication peer for replicating data to slave cluster.
* @param peerId a short name that identifies the peer
- * @param peerConfig configuration for the replication slave cluster
+ * @param peerConfig configuration for the replication peer
* @param enabled peer state, true if ENABLED and false if DISABLED
* @throws IOException if a remote or network exception occurs
*/
@@ -2492,6 +2492,37 @@ public interface Admin extends Abortable, Closeable {
throws IOException;
/**
+ * Add a new replication peer but does not block and wait for it.
+ *
+ * You can use Future.get(long, TimeUnit) to wait on the operation to
complete. It may throw
+ * ExecutionException if there was an error while executing the operation or
TimeoutException in
+ * case the wait timeout was not long enough to allow the operation to
complete.
+ * @param peerId a short name that identifies the peer
+ * @param peerConfig configuration for the replication peer
+ * @return the result of the async operation
+ * @throws IOException IOException if a remote or network exception occurs
+ */
+ default Future addReplicationPeerAsync(String peerId,
ReplicationPeerConfig peerConfig)
+ throws IOException {
+return addReplicationPeerAsync(peerId, peerConfig, true);
+ }
+
+ /**
+ * Add a new replication peer but does not block and wait for it.
+ *
+ * You can use Future.get(long, TimeUnit) to wait on the operation to
complete. It may throw
+ * ExecutionException if there was an error while executing the operation or
TimeoutException in
+ * case the wait timeout was not long enough to allow the operation to
complete.
+ * @param peerId a short name that identifies the peer
+ * @param peerConfig configuration for the replication peer
+ * @param enabled peer state, true if ENABLED and false if DISABLED
+ * @return the result of the async operation
+ * @throws IOException IOException if a remote or network exception occurs
+ */
+ Future addReplicationPeerAsync(String peerId, ReplicationPeerConfig
peerConfig,
+ boolean enabled) throws IOException;
+
+ /**
* Remove a peer and stop the replication.
* @param peerId a short name that identifies the peer
* @throws IOException if a remote or network exception occurs
@@ -2499,6 +2530,18 @@ public interface Admin extends Abortable, Closeable {
void removeReplicationPeer(String peerId) throws IOException;
/**
+ * Remove a replication peer but does not block and wait for it.
+ *
+ * You can use Future.get(long, TimeUnit) to wait on the operation to
complete. It may throw
+ * ExecutionException if there was an error while executing the operation or
TimeoutException in
+ * case the wait timeout was not long enough to allow the operation to
complete.
+ * @param peerId a short name that i
[02/40] hbase git commit: HBASE-19936 Introduce a new base class for replication peer procedure
HBASE-19936 Introduce a new base class for replication peer procedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d1e775e3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d1e775e3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d1e775e3
Branch: refs/heads/branch-2
Commit: d1e775e35e5a2ee497111c1ba4d947882b2e690e
Parents: abaa80f
Author: zhangduo
Authored: Mon Feb 5 16:14:25 2018 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../src/main/protobuf/MasterProcedure.proto | 2 +-
.../replication/AbstractPeerProcedure.java | 97
.../master/replication/ModifyPeerProcedure.java | 67 +-
3 files changed, 102 insertions(+), 64 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d1e775e3/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
index a9521ad..9377988 100644
--- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
@@ -401,7 +401,7 @@ message RefreshPeerParameter {
required ServerName target_server = 3;
}
-message ModifyPeerStateData {
+message PeerProcedureStateData {
required string peer_id = 1;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d1e775e3/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
new file mode 100644
index 000..0ad8a63
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import java.io.IOException;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface;
+import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
+import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
+import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.PeerProcedureStateData;
+
+/**
+ * The base class for all replication peer related procedure.
+ */
+@InterfaceAudience.Private
+public abstract class AbstractPeerProcedure
+extends StateMachineProcedure implements
PeerProcedureInterface {
+
+ protected String peerId;
+
+ private volatile boolean locked;
+
+ // used to keep compatible with old client where we can only returns after
updateStorage.
+ protected ProcedurePrepareLatch latch;
+
+ protected AbstractPeerProcedure() {
+ }
+
+ protected AbstractPeerProcedure(String peerId) {
+this.peerId = peerId;
+this.latch = ProcedurePrepareLatch.createLatch(2, 0);
+ }
+
+ public ProcedurePrepareLatch getLatch() {
+return latch;
+ }
+
+ @Override
+ public String getPeerId() {
+return peerId;
+ }
+
+ @Override
+ protected LockState acquireLock(MasterProcedureEnv env) {
+if (env.getProcedureScheduler().waitPeerExclusiveLock(this, peerId)) {
+ return LockState.LOCK_EVENT_WAIT;
+}
+locked = true;
+return LockState.LOCK_ACQUIRED;
+ }
+
+ @Override
+ protected void releaseLock(MasterProcedureEnv env) {
+locked = false;
+env.getProcedureScheduler().wakePeerExclusiveLock(this, peerId);
+ }
+
+ @Override
+ protected boolean holdLock(MasterProcedureEnv env) {
+return true;
+ }
+
+ @Override
+ protected boolean has
[06/40] hbase git commit: HBASE-19216 Implement a general framework to execute remote procedure on RS
HBASE-19216 Implement a general framework to execute remote procedure on RS
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/95af14fe
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/95af14fe
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/95af14fe
Branch: refs/heads/branch-2
Commit: 95af14fea6e3ba139452f6e9525bb58c24d534d9
Parents: 3576eb6
Author: zhangduo
Authored: Fri Dec 15 21:06:44 2017 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../hbase/procedure2/LockedResourceType.java| 4 +-
.../procedure2/RemoteProcedureDispatcher.java | 23 ++-
.../src/main/protobuf/Admin.proto | 9 +-
.../src/main/protobuf/MasterProcedure.proto | 30 +++
.../src/main/protobuf/RegionServerStatus.proto | 15 ++
.../apache/hadoop/hbase/executor/EventType.java | 26 ++-
.../hadoop/hbase/executor/ExecutorType.java | 3 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 30 ++-
.../hadoop/hbase/master/MasterRpcServices.java | 13 ++
.../assignment/RegionTransitionProcedure.java | 18 +-
.../procedure/MasterProcedureScheduler.java | 113 ++-
.../procedure/PeerProcedureInterface.java | 34
.../hbase/master/procedure/PeerQueue.java | 54 +
.../master/procedure/RSProcedureDispatcher.java | 101 ++
.../hbase/master/procedure/SchemaLocking.java | 5 +
.../master/replication/ModifyPeerProcedure.java | 127
.../master/replication/RefreshPeerCallable.java | 67 +++
.../replication/RefreshPeerProcedure.java | 197 +++
.../hbase/procedure2/RSProcedureCallable.java | 43
.../hbase/regionserver/HRegionServer.java | 61 ++
.../hbase/regionserver/RSRpcServices.java | 58 --
.../handler/RSProcedureHandler.java | 51 +
.../assignment/TestAssignmentManager.java | 20 +-
.../replication/DummyModifyPeerProcedure.java | 41
.../TestDummyModifyPeerProcedure.java | 80
.../security/access/TestAccessController.java | 1 +
26 files changed, 1109 insertions(+), 115 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/95af14fe/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
index c5fe62b..dc9b5d4 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -22,5 +22,5 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public enum LockedResourceType {
- SERVER, NAMESPACE, TABLE, REGION
+ SERVER, NAMESPACE, TABLE, REGION, PEER
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/95af14fe/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
index 6238e10..7e3dde6 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
@@ -226,13 +226,30 @@ public abstract class RemoteProcedureDispatcher
- * @param
*/
public interface RemoteProcedure {
+/**
+ * For building the remote operation.
+ */
RemoteOperation remoteCallBuild(TEnv env, TRemote remote);
-void remoteCallCompleted(TEnv env, TRemote remote, RemoteOperation
response);
+
+/**
+ * Called when the executeProcedure call is failed.
+ */
void remoteCallFailed(TEnv env, TRemote remote, IOException exception);
+
+/**
+ * Called when RS tells the remote procedure is succeeded through the
+ * {@code reportProcedureDone} method.
+ */
+void remoteOperationCompleted(TEnv env);
+
+/**
+ * Called when RS tells the remote procedure is failed through the {@code
reportProcedureDone}
+ * method.
+ * @param error the error message
+ */
+void remoteOperationFailed(TEnv env, String error);
}
/**
[04/40] hbase git commit: HBASE-19544 Add UTs for testing concurrent modifications on replication peer
HBASE-19544 Add UTs for testing concurrent modifications on replication peer
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/13318dd3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/13318dd3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/13318dd3
Branch: refs/heads/branch-2
Commit: 13318dd3503f110549b456eb485e993829463fab
Parents: b0de0ec
Author: Guanghao Zhang
Authored: Tue Jan 2 17:07:41 2018 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../replication/TestReplicationAdmin.java | 69
1 file changed, 69 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/13318dd3/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
index 772a9d6..a753d23 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
@@ -31,6 +31,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
@@ -58,6 +59,8 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Unit testing of ReplicationAdmin
@@ -69,6 +72,8 @@ public class TestReplicationAdmin {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestReplicationAdmin.class);
+ private static final Logger LOG =
LoggerFactory.getLogger(TestReplicationAdmin.class);
+
private final static HBaseTestingUtility TEST_UTIL =
new HBaseTestingUtility();
@@ -118,6 +123,70 @@ public class TestReplicationAdmin {
}
@Test
+ public void testConcurrentPeerOperations() throws Exception {
+int threadNum = 5;
+AtomicLong successCount = new AtomicLong(0);
+
+// Test concurrent add peer operation
+Thread[] addPeers = new Thread[threadNum];
+for (int i = 0; i < threadNum; i++) {
+ addPeers[i] = new Thread(() -> {
+try {
+ hbaseAdmin.addReplicationPeer(ID_ONE,
+ReplicationPeerConfig.newBuilder().setClusterKey(KEY_ONE).build());
+ successCount.incrementAndGet();
+} catch (Exception e) {
+ LOG.debug("Got exception when add replication peer", e);
+}
+ });
+ addPeers[i].start();
+}
+for (Thread addPeer : addPeers) {
+ addPeer.join();
+}
+assertEquals(1, successCount.get());
+
+// Test concurrent remove peer operation
+successCount.set(0);
+Thread[] removePeers = new Thread[threadNum];
+for (int i = 0; i < threadNum; i++) {
+ removePeers[i] = new Thread(() -> {
+try {
+ hbaseAdmin.removeReplicationPeer(ID_ONE);
+ successCount.incrementAndGet();
+} catch (Exception e) {
+ LOG.debug("Got exception when remove replication peer", e);
+}
+ });
+ removePeers[i].start();
+}
+for (Thread removePeer : removePeers) {
+ removePeer.join();
+}
+assertEquals(1, successCount.get());
+
+// Test concurrent add peer operation again
+successCount.set(0);
+addPeers = new Thread[threadNum];
+for (int i = 0; i < threadNum; i++) {
+ addPeers[i] = new Thread(() -> {
+try {
+ hbaseAdmin.addReplicationPeer(ID_ONE,
+ReplicationPeerConfig.newBuilder().setClusterKey(KEY_ONE).build());
+ successCount.incrementAndGet();
+} catch (Exception e) {
+ LOG.debug("Got exception when add replication peer", e);
+}
+ });
+ addPeers[i].start();
+}
+for (Thread addPeer : addPeers) {
+ addPeer.join();
+}
+assertEquals(1, successCount.get());
+ }
+
+ @Test
public void testAddInvalidPeer() {
ReplicationPeerConfigBuilder builder = ReplicationPeerConfig.newBuilder();
builder.setClusterKey(KEY_ONE);
[11/40] hbase git commit: HBASE-19525 RS side changes for moving peer modification from zk watcher to procedure
HBASE-19525 RS side changes for moving peer modification from zk watcher to
procedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3fd41760
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3fd41760
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3fd41760
Branch: refs/heads/branch-2
Commit: 3fd417600e0716c72a2f09cb06629f62bd1d5b20
Parents: da07870
Author: huzheng
Authored: Wed Dec 20 10:47:18 2017 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:55:48 2018 +0800
--
.../hadoop/hbase/protobuf/ProtobufUtil.java | 11 +-
.../hbase/shaded/protobuf/ProtobufUtil.java | 13 +-
.../hbase/replication/ReplicationListener.java | 14 --
.../hbase/replication/ReplicationPeer.java | 28 ++-
.../replication/ReplicationPeerZKImpl.java | 186 ---
.../replication/ReplicationPeersZKImpl.java | 19 +-
.../replication/ReplicationTrackerZKImpl.java | 73 +-
.../regionserver/ReplicationSourceService.java | 6 +
.../handler/RSProcedureHandler.java | 3 +
.../replication/BaseReplicationEndpoint.java| 2 +-
.../regionserver/PeerProcedureHandler.java | 38
.../regionserver/PeerProcedureHandlerImpl.java | 81 +++
.../regionserver/RefreshPeerCallable.java | 39 +++-
.../replication/regionserver/Replication.java | 9 +
.../regionserver/ReplicationSource.java | 8 +-
.../regionserver/ReplicationSourceManager.java | 37 ++-
.../TestReplicationAdminUsingProcedure.java | 226 +++
.../replication/DummyModifyPeerProcedure.java | 48
.../TestDummyModifyPeerProcedure.java | 80 ---
.../TestReplicationTrackerZKImpl.java | 61 -
.../TestReplicationSourceManager.java | 32 ++-
21 files changed, 532 insertions(+), 482 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/3fd41760/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 3033da7..f11fcf6 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.protobuf;
+import static org.apache.hadoop.hbase.protobuf.ProtobufMagic.PB_MAGIC;
+
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.InvalidProtocolBufferException;
@@ -191,7 +193,7 @@ public final class ProtobufUtil {
* byte array that is bytes.length plus {@link
ProtobufMagic#PB_MAGIC}.length.
*/
public static byte [] prependPBMagic(final byte [] bytes) {
-return Bytes.add(ProtobufMagic.PB_MAGIC, bytes);
+return Bytes.add(PB_MAGIC, bytes);
}
/**
@@ -216,10 +218,11 @@ public final class ProtobufUtil {
* @param bytes bytes to check
* @throws DeserializationException if we are missing the pb magic prefix
*/
- public static void expectPBMagicPrefix(final byte [] bytes) throws
DeserializationException {
+ public static void expectPBMagicPrefix(final byte[] bytes) throws
DeserializationException {
if (!isPBMagicPrefix(bytes)) {
- throw new DeserializationException("Missing pb magic " +
- Bytes.toString(ProtobufMagic.PB_MAGIC) + " prefix");
+ String bytesPrefix = bytes == null ? "null" :
Bytes.toStringBinary(bytes, 0, PB_MAGIC.length);
+ throw new DeserializationException(
+ "Missing pb magic " + Bytes.toString(PB_MAGIC) + " prefix, bytes: "
+ bytesPrefix);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3fd41760/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index 4472846..520a4cd 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.shaded.protobuf;
+import static org.apache.hadoop.hbase.protobuf.ProtobufMagic.PB_MAGIC;
+
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -277,7 +279,7 @@ public final class ProtobufUtil {
* byte array that is bytes.length plus {@link
ProtobufMagic#PB_MAGIC}.
hbase git commit: HBASE-19598 Fix TestAssignmentManagerMetrics flaky test
Repository: hbase
Updated Branches:
refs/heads/branch-2.0 58d840199 -> d2e5e11f6
HBASE-19598 Fix TestAssignmentManagerMetrics flaky test
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d2e5e11f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d2e5e11f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d2e5e11f
Branch: refs/heads/branch-2.0
Commit: d2e5e11f6567e9a1bfb131e2b9b5416d7b871669
Parents: 58d8401
Author: zhangduo
Authored: Fri Mar 9 10:39:47 2018 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:32:16 2018 +0800
--
.../master/TestAssignmentManagerMetrics.java| 36 +++-
1 file changed, 19 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d2e5e11f/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java
index 287fc70..cee7a4a 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.test.MetricsAssertHelper;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -54,14 +55,13 @@ public class TestAssignmentManagerMetrics {
HBaseClassTestRule.forClass(TestAssignmentManagerMetrics.class);
private static final Logger LOG =
LoggerFactory.getLogger(TestAssignmentManagerMetrics.class);
- private static final MetricsAssertHelper metricsHelper = CompatibilityFactory
+ private static final MetricsAssertHelper METRICS_HELPER =
CompatibilityFactory
.getInstance(MetricsAssertHelper.class);
- private static MiniHBaseCluster cluster;
- private static HMaster master;
+ private static MiniHBaseCluster CLUSTER;
+ private static HMaster MASTER;
private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
- private static Configuration conf;
- private static final int msgInterval = 1000;
+ private static final int MSG_INTERVAL = 1000;
@Rule
public TestName name = new TestName();
@@ -69,7 +69,7 @@ public class TestAssignmentManagerMetrics {
@BeforeClass
public static void startCluster() throws Exception {
LOG.info("Starting cluster");
-conf = TEST_UTIL.getConfiguration();
+Configuration conf = TEST_UTIL.getConfiguration();
// Disable sanity check for coprocessor
conf.setBoolean("hbase.table.sanity.checks", false);
@@ -78,19 +78,21 @@ public class TestAssignmentManagerMetrics {
conf.setInt(HConstants.METRICS_RIT_STUCK_WARNING_THRESHOLD, 20);
// set msgInterval to 1 second
-conf.setInt("hbase.regionserver.msginterval", msgInterval);
+conf.setInt("hbase.regionserver.msginterval", MSG_INTERVAL);
// set tablesOnMaster to none
conf.set("hbase.balancer.tablesOnMaster", "none");
// set client sync wait timeout to 5sec
-conf.setInt("hbase.client.sync.wait.timeout.msec", 2500);
+conf.setInt("hbase.client.sync.wait.timeout.msec", 5000);
conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
conf.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 2500);
+// set a small interval for updating rit metrics
+conf.setInt(AssignmentManager.RIT_CHORE_INTERVAL_MSEC_CONF_KEY,
MSG_INTERVAL);
TEST_UTIL.startMiniCluster(1);
-cluster = TEST_UTIL.getHBaseCluster();
-master = cluster.getMaster();
+CLUSTER = TEST_UTIL.getHBaseCluster();
+MASTER = CLUSTER.getMaster();
}
@AfterClass
@@ -112,14 +114,14 @@ public class TestAssignmentManagerMetrics {
table.put(put);
// Sleep 3 seconds, wait for doMetrics chore catching up
- Thread.sleep(msgInterval * 3);
+ Thread.sleep(MSG_INTERVAL * 3);
// check the RIT is 0
MetricsAssignmentManagerSource amSource =
-
master.getAssignmentManager().getAssignmentManagerMetrics().getMetricsProcSource();
+
MASTER.getAssignmentManager().getAssignmentManagerMetrics().getMetricsProcSource();
- metricsHelper.assertGauge(MetricsAssignmentManagerSource.RIT_COUNT_NAME,
0, amSource);
-
metricsHelper.assertGauge(MetricsAssignmentManagerSource.RIT_COUNT_OVE
hbase git commit: HBASE-19598 Fix TestAssignmentManagerMetrics flaky test
Repository: hbase
Updated Branches:
refs/heads/branch-2 b4e297f3f -> 3576eb6bd
HBASE-19598 Fix TestAssignmentManagerMetrics flaky test
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3576eb6b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3576eb6b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3576eb6b
Branch: refs/heads/branch-2
Commit: 3576eb6bd824300bef2efbf55ce6a5dea4a26214
Parents: b4e297f
Author: zhangduo
Authored: Fri Mar 9 10:39:47 2018 +0800
Committer: zhangduo
Committed: Fri Mar 9 20:31:45 2018 +0800
--
.../master/TestAssignmentManagerMetrics.java| 36 +++-
1 file changed, 19 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/3576eb6b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java
index 287fc70..cee7a4a 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.test.MetricsAssertHelper;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -54,14 +55,13 @@ public class TestAssignmentManagerMetrics {
HBaseClassTestRule.forClass(TestAssignmentManagerMetrics.class);
private static final Logger LOG =
LoggerFactory.getLogger(TestAssignmentManagerMetrics.class);
- private static final MetricsAssertHelper metricsHelper = CompatibilityFactory
+ private static final MetricsAssertHelper METRICS_HELPER =
CompatibilityFactory
.getInstance(MetricsAssertHelper.class);
- private static MiniHBaseCluster cluster;
- private static HMaster master;
+ private static MiniHBaseCluster CLUSTER;
+ private static HMaster MASTER;
private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
- private static Configuration conf;
- private static final int msgInterval = 1000;
+ private static final int MSG_INTERVAL = 1000;
@Rule
public TestName name = new TestName();
@@ -69,7 +69,7 @@ public class TestAssignmentManagerMetrics {
@BeforeClass
public static void startCluster() throws Exception {
LOG.info("Starting cluster");
-conf = TEST_UTIL.getConfiguration();
+Configuration conf = TEST_UTIL.getConfiguration();
// Disable sanity check for coprocessor
conf.setBoolean("hbase.table.sanity.checks", false);
@@ -78,19 +78,21 @@ public class TestAssignmentManagerMetrics {
conf.setInt(HConstants.METRICS_RIT_STUCK_WARNING_THRESHOLD, 20);
// set msgInterval to 1 second
-conf.setInt("hbase.regionserver.msginterval", msgInterval);
+conf.setInt("hbase.regionserver.msginterval", MSG_INTERVAL);
// set tablesOnMaster to none
conf.set("hbase.balancer.tablesOnMaster", "none");
// set client sync wait timeout to 5sec
-conf.setInt("hbase.client.sync.wait.timeout.msec", 2500);
+conf.setInt("hbase.client.sync.wait.timeout.msec", 5000);
conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
conf.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 2500);
+// set a small interval for updating rit metrics
+conf.setInt(AssignmentManager.RIT_CHORE_INTERVAL_MSEC_CONF_KEY,
MSG_INTERVAL);
TEST_UTIL.startMiniCluster(1);
-cluster = TEST_UTIL.getHBaseCluster();
-master = cluster.getMaster();
+CLUSTER = TEST_UTIL.getHBaseCluster();
+MASTER = CLUSTER.getMaster();
}
@AfterClass
@@ -112,14 +114,14 @@ public class TestAssignmentManagerMetrics {
table.put(put);
// Sleep 3 seconds, wait for doMetrics chore catching up
- Thread.sleep(msgInterval * 3);
+ Thread.sleep(MSG_INTERVAL * 3);
// check the RIT is 0
MetricsAssignmentManagerSource amSource =
-
master.getAssignmentManager().getAssignmentManagerMetrics().getMetricsProcSource();
+
MASTER.getAssignmentManager().getAssignmentManagerMetrics().getMetricsProcSource();
- metricsHelper.assertGauge(MetricsAssignmentManagerSource.RIT_COUNT_NAME,
0, amSource);
-
metricsHelper.assertGauge(MetricsAssignmentManagerSource.RIT_COUNT_OVER_TH
