[12/20] hbase git commit: HBASE-20117 Cleanup the unused replication barriers in meta table

[zhangduo]

HBASE-20117 Cleanup the unused replication barriers in meta table


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/644bfe36
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/644bfe36
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/644bfe36

Branch: refs/heads/branch-2
Commit: 644bfe36b297b2787bf07a46eb6f5085322edfa9
Parents: fedf3ca
Author: zhangduo 
Authored: Tue Mar 13 21:36:06 2018 +0800
Committer: zhangduo 
Committed: Mon Apr 9 15:18:44 2018 +0800

--
 .../apache/hadoop/hbase/MetaTableAccessor.java  |   2 +-
 .../hbase/replication/ReplicationUtils.java |  56 +++-
 .../org/apache/hadoop/hbase/master/HMaster.java |  91 +++---
 .../cleaner/ReplicationBarrierCleaner.java  | 162 ++
 .../replication/ReplicationPeerManager.java |  10 +
 .../NamespaceTableCfWALEntryFilter.java |  39 +--
 .../cleaner/TestReplicationBarrierCleaner.java  | 293 +++
 .../TestSerialReplicationChecker.java   |   2 +-
 8 files changed, 565 insertions(+), 90 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/644bfe36/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index 2a88b56..a800c1c 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -2053,7 +2053,7 @@ public class MetaTableAccessor {
 return Bytes.toLong(c.getValueArray(), c.getValueOffset(), 
c.getValueLength());
   }
 
-  private static long[] getReplicationBarriers(Result result) {
+  public static long[] getReplicationBarriers(Result result) {
 return result.getColumnCells(HConstants.REPLICATION_BARRIER_FAMILY, 
HConstants.SEQNUM_QUALIFIER)
   
.stream().mapToLong(MetaTableAccessor::getReplicationBarrier).sorted().distinct().toArray();
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/644bfe36/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
--
diff --git 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
index 857b385..e2479e0 100644
--- 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
+++ 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
@@ -39,15 +39,6 @@ public final class ReplicationUtils {
   private ReplicationUtils() {
   }
 
-  /**
-   * @param c Configuration to look at
-   * @return True if replication for bulk load data is enabled.
-   */
-  public static boolean isReplicationForBulkLoadDataEnabled(final 
Configuration c) {
-return c.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
-  HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT);
-  }
-
   public static Configuration 
getPeerClusterConfiguration(ReplicationPeerConfig peerConfig,
   Configuration baseConf) throws ReplicationException {
 Configuration otherConf;
@@ -135,4 +126,51 @@ public final class ReplicationUtils {
 isTableCFsEqual(rpc1.getTableCFsMap(), rpc2.getTableCFsMap());
 }
   }
+
+  /**
+   * @param c Configuration to look at
+   * @return True if replication for bulk load data is enabled.
+   */
+  public static boolean isReplicationForBulkLoadDataEnabled(final 
Configuration c) {
+return c.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
+  HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT);
+  }
+
+  /**
+   * Returns whether we should replicate the given table.
+   */
+  public static boolean contains(ReplicationPeerConfig peerConfig, TableName 
tableName) {
+String namespace = tableName.getNamespaceAsString();
+if (peerConfig.replicateAllUserTables()) {
+  // replicate all user tables, but filter by exclude namespaces and 
table-cfs config
+  Set excludeNamespaces = peerConfig.getExcludeNamespaces();
+  if (excludeNamespaces != null && excludeNamespaces.contains(namespace)) {
+return false;
+  }
+  Map excludedTableCFs = 
peerConfig.getTableCFsMap();
+  // trap here, must check existence first since HashMap allows null value.
+  if (excludedTableCFs == null || 
!excludedTableCFs.containsKey(tableName)) {
+return true;
+  }
+  List cfs = excludedTableCFs.get(tableName);
+  // if cfs is null or empty then we can make sure that we 

[14/22] hbase git commit: HBASE-20117 Cleanup the unused replication barriers in meta table

[zhangduo]

HBASE-20117 Cleanup the unused replication barriers in meta table


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/644bfe36
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/644bfe36
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/644bfe36

Branch: refs/heads/HBASE-20046-branch-2
Commit: 644bfe36b297b2787bf07a46eb6f5085322edfa9
Parents: fedf3ca
Author: zhangduo 
Authored: Tue Mar 13 21:36:06 2018 +0800
Committer: zhangduo 
Committed: Mon Apr 9 15:18:44 2018 +0800

--
 .../apache/hadoop/hbase/MetaTableAccessor.java  |   2 +-
 .../hbase/replication/ReplicationUtils.java |  56 +++-
 .../org/apache/hadoop/hbase/master/HMaster.java |  91 +++---
 .../cleaner/ReplicationBarrierCleaner.java  | 162 ++
 .../replication/ReplicationPeerManager.java |  10 +
 .../NamespaceTableCfWALEntryFilter.java |  39 +--
 .../cleaner/TestReplicationBarrierCleaner.java  | 293 +++
 .../TestSerialReplicationChecker.java   |   2 +-
 8 files changed, 565 insertions(+), 90 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/644bfe36/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index 2a88b56..a800c1c 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -2053,7 +2053,7 @@ public class MetaTableAccessor {
 return Bytes.toLong(c.getValueArray(), c.getValueOffset(), 
c.getValueLength());
   }
 
-  private static long[] getReplicationBarriers(Result result) {
+  public static long[] getReplicationBarriers(Result result) {
 return result.getColumnCells(HConstants.REPLICATION_BARRIER_FAMILY, 
HConstants.SEQNUM_QUALIFIER)
   
.stream().mapToLong(MetaTableAccessor::getReplicationBarrier).sorted().distinct().toArray();
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/644bfe36/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
--
diff --git 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
index 857b385..e2479e0 100644
--- 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
+++ 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
@@ -39,15 +39,6 @@ public final class ReplicationUtils {
   private ReplicationUtils() {
   }
 
-  /**
-   * @param c Configuration to look at
-   * @return True if replication for bulk load data is enabled.
-   */
-  public static boolean isReplicationForBulkLoadDataEnabled(final 
Configuration c) {
-return c.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
-  HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT);
-  }
-
   public static Configuration 
getPeerClusterConfiguration(ReplicationPeerConfig peerConfig,
   Configuration baseConf) throws ReplicationException {
 Configuration otherConf;
@@ -135,4 +126,51 @@ public final class ReplicationUtils {
 isTableCFsEqual(rpc1.getTableCFsMap(), rpc2.getTableCFsMap());
 }
   }
+
+  /**
+   * @param c Configuration to look at
+   * @return True if replication for bulk load data is enabled.
+   */
+  public static boolean isReplicationForBulkLoadDataEnabled(final 
Configuration c) {
+return c.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
+  HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT);
+  }
+
+  /**
+   * Returns whether we should replicate the given table.
+   */
+  public static boolean contains(ReplicationPeerConfig peerConfig, TableName 
tableName) {
+String namespace = tableName.getNamespaceAsString();
+if (peerConfig.replicateAllUserTables()) {
+  // replicate all user tables, but filter by exclude namespaces and 
table-cfs config
+  Set excludeNamespaces = peerConfig.getExcludeNamespaces();
+  if (excludeNamespaces != null && excludeNamespaces.contains(namespace)) {
+return false;
+  }
+  Map excludedTableCFs = 
peerConfig.getTableCFsMap();
+  // trap here, must check existence first since HashMap allows null value.
+  if (excludedTableCFs == null || 
!excludedTableCFs.containsKey(tableName)) {
+return true;
+  }
+  List cfs = excludedTableCFs.get(tableName);
+  // if cfs is null or empty then we can make 

[09/20] hbase git commit: HBASE-20117 Cleanup the unused replication barriers in meta table

[zhangduo]

HBASE-20117 Cleanup the unused replication barriers in meta table


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/71f72f42
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/71f72f42
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/71f72f42

Branch: refs/heads/HBASE-20046-branch-2
Commit: 71f72f42a62a14d56cc38a74c02fdf8e805f136a
Parents: d74e7f8
Author: zhangduo 
Authored: Tue Mar 13 21:36:06 2018 +0800
Committer: zhangduo 
Committed: Sun Apr 8 11:18:38 2018 +0800

--
 .../apache/hadoop/hbase/MetaTableAccessor.java  |   2 +-
 .../hbase/replication/ReplicationUtils.java |  56 +++-
 .../org/apache/hadoop/hbase/master/HMaster.java |  91 +++---
 .../cleaner/ReplicationBarrierCleaner.java  | 162 ++
 .../replication/ReplicationPeerManager.java |  10 +
 .../NamespaceTableCfWALEntryFilter.java |  39 +--
 .../cleaner/TestReplicationBarrierCleaner.java  | 293 +++
 .../TestSerialReplicationChecker.java   |   2 +-
 8 files changed, 565 insertions(+), 90 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/71f72f42/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index 2a88b56..a800c1c 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -2053,7 +2053,7 @@ public class MetaTableAccessor {
 return Bytes.toLong(c.getValueArray(), c.getValueOffset(), 
c.getValueLength());
   }
 
-  private static long[] getReplicationBarriers(Result result) {
+  public static long[] getReplicationBarriers(Result result) {
 return result.getColumnCells(HConstants.REPLICATION_BARRIER_FAMILY, 
HConstants.SEQNUM_QUALIFIER)
   
.stream().mapToLong(MetaTableAccessor::getReplicationBarrier).sorted().distinct().toArray();
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/71f72f42/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
--
diff --git 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
index 857b385..e2479e0 100644
--- 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
+++ 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
@@ -39,15 +39,6 @@ public final class ReplicationUtils {
   private ReplicationUtils() {
   }
 
-  /**
-   * @param c Configuration to look at
-   * @return True if replication for bulk load data is enabled.
-   */
-  public static boolean isReplicationForBulkLoadDataEnabled(final 
Configuration c) {
-return c.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
-  HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT);
-  }
-
   public static Configuration 
getPeerClusterConfiguration(ReplicationPeerConfig peerConfig,
   Configuration baseConf) throws ReplicationException {
 Configuration otherConf;
@@ -135,4 +126,51 @@ public final class ReplicationUtils {
 isTableCFsEqual(rpc1.getTableCFsMap(), rpc2.getTableCFsMap());
 }
   }
+
+  /**
+   * @param c Configuration to look at
+   * @return True if replication for bulk load data is enabled.
+   */
+  public static boolean isReplicationForBulkLoadDataEnabled(final 
Configuration c) {
+return c.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
+  HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT);
+  }
+
+  /**
+   * Returns whether we should replicate the given table.
+   */
+  public static boolean contains(ReplicationPeerConfig peerConfig, TableName 
tableName) {
+String namespace = tableName.getNamespaceAsString();
+if (peerConfig.replicateAllUserTables()) {
+  // replicate all user tables, but filter by exclude namespaces and 
table-cfs config
+  Set excludeNamespaces = peerConfig.getExcludeNamespaces();
+  if (excludeNamespaces != null && excludeNamespaces.contains(namespace)) {
+return false;
+  }
+  Map excludedTableCFs = 
peerConfig.getTableCFsMap();
+  // trap here, must check existence first since HashMap allows null value.
+  if (excludedTableCFs == null || 
!excludedTableCFs.containsKey(tableName)) {
+return true;
+  }
+  List cfs = excludedTableCFs.get(tableName);
+  // if cfs is null or empty then we can make 

hbase git commit: HBASE-20117 Cleanup the unused replication barriers in meta table

[zhangduo]

Repository: hbase
Updated Branches:
  refs/heads/master b16e03c13 -> b7308ee01


HBASE-20117 Cleanup the unused replication barriers in meta table


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b7308ee0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b7308ee0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b7308ee0

Branch: refs/heads/master
Commit: b7308ee01c653051db4060ba28dadb5bcc74b7de
Parents: b16e03c
Author: zhangduo 
Authored: Tue Mar 13 21:36:06 2018 +0800
Committer: zhangduo 
Committed: Wed Mar 14 12:08:15 2018 +0800

--
 .../apache/hadoop/hbase/MetaTableAccessor.java  |   2 +-
 .../hbase/replication/ReplicationUtils.java |  38 +++
 .../org/apache/hadoop/hbase/master/HMaster.java |  91 +++---
 .../cleaner/ReplicationBarrierCleaner.java  | 162 ++
 .../replication/ReplicationPeerManager.java |  10 +
 .../NamespaceTableCfWALEntryFilter.java |  39 +--
 .../cleaner/TestReplicationBarrierCleaner.java  | 293 +++
 .../TestSerialReplicationChecker.java   |   2 +-
 8 files changed, 556 insertions(+), 81 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b7308ee0/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index 5f1a8b7..57a0350 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -2056,7 +2056,7 @@ public class MetaTableAccessor {
 return Bytes.toLong(c.getValueArray(), c.getValueOffset(), 
c.getValueLength());
   }
 
-  private static long[] getReplicationBarriers(Result result) {
+  public static long[] getReplicationBarriers(Result result) {
 return result.getColumnCells(HConstants.REPLICATION_BARRIER_FAMILY, 
HConstants.SEQNUM_QUALIFIER)
   
.stream().mapToLong(MetaTableAccessor::getReplicationBarrier).sorted().distinct().toArray();
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/b7308ee0/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
--
diff --git 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
index 2a6870a..e2479e0 100644
--- 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
+++ 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
@@ -135,4 +135,42 @@ public final class ReplicationUtils {
 return c.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
   HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT);
   }
+
+  /**
+   * Returns whether we should replicate the given table.
+   */
+  public static boolean contains(ReplicationPeerConfig peerConfig, TableName 
tableName) {
+String namespace = tableName.getNamespaceAsString();
+if (peerConfig.replicateAllUserTables()) {
+  // replicate all user tables, but filter by exclude namespaces and 
table-cfs config
+  Set excludeNamespaces = peerConfig.getExcludeNamespaces();
+  if (excludeNamespaces != null && excludeNamespaces.contains(namespace)) {
+return false;
+  }
+  Map excludedTableCFs = 
peerConfig.getTableCFsMap();
+  // trap here, must check existence first since HashMap allows null value.
+  if (excludedTableCFs == null || 
!excludedTableCFs.containsKey(tableName)) {
+return true;
+  }
+  List cfs = excludedTableCFs.get(tableName);
+  // if cfs is null or empty then we can make sure that we do not need to 
replicate this table,
+  // otherwise, we may still need to replicate the table but filter out 
some families.
+  return cfs != null && !cfs.isEmpty();
+} else {
+  // Not replicate all user tables, so filter by namespaces and table-cfs 
config
+  Set namespaces = peerConfig.getNamespaces();
+  Map tableCFs = peerConfig.getTableCFsMap();
+
+  if (namespaces == null && tableCFs == null) {
+return false;
+  }
+
+  // First filter by namespaces config
+  // If table's namespace in peer config, all the tables data are 
applicable for replication
+  if (namespaces != null && namespaces.contains(namespace)) {
+return true;
+  }
+  return tableCFs != null && tableCFs.containsKey(tableName);
+}
+  }
 }