Repository: hbase Updated Branches: refs/heads/master b392de3e3 -> 0e05537de
HBASE-17460 enable_table_replication can not perform cyclic replication of a table (NITIN VERMA) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0e05537d Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0e05537d Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0e05537d Branch: refs/heads/master Commit: 0e05537dec32d31e1d025ddf60276df01ea0e28b Parents: b392de3 Author: tedyu <yuzhih...@gmail.com> Authored: Fri Feb 17 17:56:53 2017 -0800 Committer: tedyu <yuzhih...@gmail.com> Committed: Fri Feb 17 17:56:53 2017 -0800 ---------------------------------------------------------------------- .../apache/hadoop/hbase/HColumnDescriptor.java | 2 +- .../apache/hadoop/hbase/HTableDescriptor.java | 102 ++++++++++++++++++- .../apache/hadoop/hbase/client/HBaseAdmin.java | 31 ++++-- .../client/replication/ReplicationAdmin.java | 8 +- .../TestReplicationAdminWithClusters.java | 1 + 5 files changed, 129 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/0e05537d/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java index 1597a06..6d1ae3f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java @@ -1076,7 +1076,7 @@ public class HColumnDescriptor implements Comparable<HColumnDescriptor> { public int compareTo(HColumnDescriptor o) { int result = Bytes.compareTo(this.name, o.getName()); if (result == 0) { - // punt on comparison for ordering, just calculate difference + // punt on comparison for ordering, just calculate difference. result = this.values.hashCode() - o.values.hashCode(); if (result < 0) result = -1; http://git-wip-us.apache.org/repos/asf/hbase/blob/0e05537d/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java index 60b85fe..a81c82a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java @@ -40,9 +40,9 @@ import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.TableSchema; -import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; /** @@ -1042,6 +1042,106 @@ public class HTableDescriptor implements Comparable<HTableDescriptor> { } /** + * Detects whether replication has been already enabled on any of the column families of this + * table descriptor. + * @return true if any of the column families has replication enabled. + */ + public boolean isReplicationEnabled() { + // Go through each Column-Family descriptor and check if the + // Replication has been enabled already. + // Return 'true' if replication has been enabled on any CF, + // otherwise return 'false'. + // + boolean result = false; + Iterator<HColumnDescriptor> it = this.families.values().iterator(); + + while (it.hasNext()) { + HColumnDescriptor tempHcd = it.next(); + if (tempHcd.getScope() != HConstants.REPLICATION_SCOPE_LOCAL) { + result = true; + break; + } + } + + return result; + } + + /** + * Compare the contents of the descriptor with another one passed as a parameter for replication + * purpose. The REPLICATION_SCOPE field is ignored during comparison. + * @param obj descriptor on source cluster which needs to be replicated. + * @return true if the contents of the two descriptors match (ignoring just REPLICATION_SCOPE). + * @see java.lang.Object#equals(java.lang.Object) + */ + public boolean compareForReplication(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof HTableDescriptor)) { + return false; + } + + boolean result = false; + + // Create a copy of peer HTD as we need to change its replication + // scope to match with the local HTD. + HTableDescriptor peerHtdCopy = new HTableDescriptor(this); + + // Copy the replication scope of local Htd to remote Htd. + HTableDescriptor localHtd = (HTableDescriptor) obj; + + result = (peerHtdCopy.copyReplicationScope(localHtd) == 0); + + // If copy was successful, compare the two tables now. + if (result == true) { + result = (peerHtdCopy.compareTo(localHtd) == 0); + } + + return result; + } + + /** + * Copies the REPLICATION_SCOPE of table descriptor passed as an argument. Before copy, the method + * ensures that the name of table and column-families should match. + * @param localHtd - The HTableDescriptor of table from source cluster. + * @return 0 If the name of table and column families match and REPLICATION_SCOPE copied + * successfully. 1 If there is any mismatch in the names. + */ + public int copyReplicationScope(final HTableDescriptor localHtd) + { + // Copy the REPLICATION_SCOPE only when table names and the names of + // Column-Families are same. + int result = this.name.compareTo(localHtd.name); + + if (result == 0) { + Iterator<HColumnDescriptor> remoteHCDIter = families.values().iterator(); + Iterator<HColumnDescriptor> localHCDIter = localHtd.families.values().iterator(); + + while (remoteHCDIter.hasNext() && localHCDIter.hasNext()) { + HColumnDescriptor remoteHCD = remoteHCDIter.next(); + HColumnDescriptor localHCD = localHCDIter.next(); + + String remoteHCDName = remoteHCD.getNameAsString(); + String localHCDName = localHCD.getNameAsString(); + + if (remoteHCDName.equals(localHCDName)) + { + remoteHCD.setScope(localHCD.getScope()); + } + else { + result = -1; + break; + } + } + } + + return result; + } + + /** * @see java.lang.Object#hashCode() */ @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/0e05537d/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index 65070b9..6fb448b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -29,7 +29,6 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; @@ -4087,7 +4086,9 @@ public class HBaseAdmin implements Admin { * Connect to peer and check the table descriptor on peer: * <ol> * <li>Create the same table on peer when not exist.</li> - * <li>Throw exception if the table exists on peer cluster but descriptors are not same.</li> + * <li>Throw an exception if the table already has replication enabled on any of the column + * families.</li> + * <li>Throw an exception if the table exists on peer cluster but descriptors are not same.</li> * </ol> * @param tableName name of the table to sync to the peer * @param splits table split keys @@ -4105,20 +4106,32 @@ public class HBaseAdmin implements Admin { Configuration peerConf = getPeerClusterConfiguration(peerDesc); try (Connection conn = ConnectionFactory.createConnection(peerConf); Admin repHBaseAdmin = conn.getAdmin()) { - HTableDescriptor htd = getTableDescriptor(tableName); + HTableDescriptor localHtd = getTableDescriptor(tableName); HTableDescriptor peerHtd = null; if (!repHBaseAdmin.tableExists(tableName)) { - repHBaseAdmin.createTable(htd, splits); + repHBaseAdmin.createTable(localHtd, splits); } else { peerHtd = repHBaseAdmin.getTableDescriptor(tableName); if (peerHtd == null) { throw new IllegalArgumentException("Failed to get table descriptor for table " + tableName.getNameAsString() + " from peer cluster " + peerDesc.getPeerId()); - } else if (!peerHtd.equals(htd)) { - throw new IllegalArgumentException("Table " + tableName.getNameAsString() - + " exists in peer cluster " + peerDesc.getPeerId() - + ", but the table descriptors are not same when compared with source cluster." - + " Thus can not enable the table's replication switch."); + } else { + // To support cyclic replication (HBASE-17460), we need to match the + // REPLICATION_SCOPE of table on both the clusters. We should do this + // only when the replication is not already enabled on local HTD (local + // table on this cluster). + // + if (localHtd.isReplicationEnabled()) { + throw new IllegalArgumentException("Table " + tableName.getNameAsString() + + " has replication already enabled for at least one Column Family."); + } else { + if (!peerHtd.compareForReplication(localHtd)) { + throw new IllegalArgumentException("Table " + tableName.getNameAsString() + + " exists in peer cluster " + peerDesc.getPeerId() + + ", but the table descriptors are not same when compared with source cluster." + + " Thus can not enable the table's replication switch."); + } + } } } } http://git-wip-us.apache.org/repos/asf/hbase/blob/0e05537d/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java index 706f81e..4e74d87 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java @@ -18,9 +18,6 @@ */ package org.apache.hadoop.hbase.client.replication; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; - import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; @@ -29,9 +26,9 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.TreeMap; import java.util.Map.Entry; import java.util.Set; +import java.util.TreeMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -62,6 +59,9 @@ import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; + /** * <p> * This class provides the administrative interface to HBase cluster http://git-wip-us.apache.org/repos/asf/hbase/blob/0e05537d/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java index 96fca47..24889ad 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java @@ -77,6 +77,7 @@ public class TestReplicationAdminWithClusters extends TestReplicationBase { @Test(timeout = 300000) public void testEnableReplicationWhenSlaveClusterDoesntHaveTable() throws Exception { + admin1.disableTableReplication(tableName); admin2.disableTable(tableName); admin2.deleteTable(tableName); assertFalse(admin2.tableExists(tableName));
