Author: jdcryans
Date: Fri Aug 9 20:45:30 2013
New Revision: 1512489
URL: http://svn.apache.org/r1512489
Log:
HBASE-8663 a HBase Shell command to list the tables replicated from
current cluster (Demai Ni via JD)
Added:
hbase/branches/0.95/hbase-server/src/main/ruby/shell/commands/list_replicated_tables.rb
Modified:
hbase/branches/0.95/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
hbase/branches/0.95/hbase-server/src/main/ruby/hbase/replication_admin.rb
hbase/branches/0.95/hbase-server/src/main/ruby/shell.rb
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
Modified:
hbase/branches/0.95/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java?rev=1512489&r1=1512488&r2=1512489&view=diff
==============================================================================
---
hbase/branches/0.95/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
(original)
+++
hbase/branches/0.95/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
Fri Aug 9 20:45:30 2013
@@ -30,10 +30,16 @@ import org.apache.hadoop.hbase.replicati
import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.zookeeper.KeeperException;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HColumnDescriptor;
import java.io.Closeable;
import java.io.IOException;
import java.util.Map;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.lang.Integer;
/**
* <p>
@@ -61,6 +67,15 @@ import java.util.Map;
public class ReplicationAdmin implements Closeable {
private static final Log LOG = LogFactory.getLog(ReplicationAdmin.class);
+ public static final String TNAME = "tableName";
+ public static final String CFNAME = "columnFamlyName";
+
+ // only Global for now, can add other type
+ // such as, 1) no global replication, or 2) the table is replicated to this
cluster, etc.
+ public static final String REPLICATIONTYPE = "replicationType";
+ public static final String REPLICATIONGLOBAL = Integer
+ .toString(HConstants.REPLICATION_SCOPE_GLOBAL);
+
private final HConnection connection;
private final ReplicationQueuesClient replicationQueuesClient;
private final ReplicationPeers replicationPeers;
@@ -166,4 +181,38 @@ public class ReplicationAdmin implements
this.connection.close();
}
}
+
+
+ /**
+ * Find all column families that are replicated from this cluster
+ * @return the full list of the replicated column families of this cluster
as:
+ * tableName, family name, replicationType
+ *
+ * Currently replicationType is Global. In the future, more replication
+ * types may be extended here. For example
+ * 1) the replication may only apply to selected peers instead of all peers
+ * 2) the replicationType may indicate the host Cluster servers as Slave
+ * for the table:columnFam.
+ */
+ public List<HashMap<String, String>> listReplicated() throws IOException {
+ List<HashMap<String, String>> replicationColFams = new
ArrayList<HashMap<String, String>>();
+ HTableDescriptor[] tables = this.connection.listTables();
+
+ for (HTableDescriptor table : tables) {
+ HColumnDescriptor[] columns = table.getColumnFamilies();
+ String tableName = table.getNameAsString();
+ for (HColumnDescriptor column : columns) {
+ if (column.getScope() != HConstants.REPLICATION_SCOPE_LOCAL) {
+ // At this moment, the columfam is replicated to all peers
+ HashMap<String, String> replicationEntry = new HashMap<String,
String>();
+ replicationEntry.put(TNAME, tableName);
+ replicationEntry.put(CFNAME, column.getNameAsString());
+ replicationEntry.put(REPLICATIONTYPE, REPLICATIONGLOBAL);
+ replicationColFams.add(replicationEntry);
+ }
+ }
+ }
+
+ return replicationColFams;
+ }
}
Modified:
hbase/branches/0.95/hbase-server/src/main/ruby/hbase/replication_admin.rb
URL:
http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/ruby/hbase/replication_admin.rb?rev=1512489&r1=1512488&r2=1512489&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/ruby/hbase/replication_admin.rb
(original)
+++ hbase/branches/0.95/hbase-server/src/main/ruby/hbase/replication_admin.rb
Fri Aug 9 20:45:30 2013
@@ -42,6 +42,13 @@ module Hbase
@replication_admin.removePeer(id)
end
+
+
#---------------------------------------------------------------------------------------------
+ # Show replcated tables/column families, and their ReplicationType
+ def list_replicated_tables
+ @replication_admin.listReplicated()
+ end
+
#----------------------------------------------------------------------------------------------
# List all peer clusters
def list_peers
Modified: hbase/branches/0.95/hbase-server/src/main/ruby/shell.rb
URL:
http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/ruby/shell.rb?rev=1512489&r1=1512488&r2=1512489&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/ruby/shell.rb (original)
+++ hbase/branches/0.95/hbase-server/src/main/ruby/shell.rb Fri Aug 9 20:45:30
2013
@@ -317,6 +317,7 @@ Shell.load_command_group(
list_peers
enable_peer
disable_peer
+ list_replicated_tables
]
)
Added:
hbase/branches/0.95/hbase-server/src/main/ruby/shell/commands/list_replicated_tables.rb
URL:
http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/ruby/shell/commands/list_replicated_tables.rb?rev=1512489&view=auto
==============================================================================
---
hbase/branches/0.95/hbase-server/src/main/ruby/shell/commands/list_replicated_tables.rb
(added)
+++
hbase/branches/0.95/hbase-server/src/main/ruby/shell/commands/list_replicated_tables.rb
Fri Aug 9 20:45:30 2013
@@ -0,0 +1,52 @@
+#
+# Copyright The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+ module Commands
+ class ListReplicatedTables< Command
+ def help
+ return <<-EOF
+List all the tables and column families replicated from this cluster
+
+ hbase> list_replicated_tables
+ hbase> list_replicated_tables 'abc.*'
+EOF
+ end
+
+ def command(regex = ".*")
+ now = Time.now
+
+ formatter.header([ "TABLE:COLUMNFAMILY", "ReplicationType" ], [ 32 ])
+ list = replication_admin.list_replicated_tables
+ regex = /#{regex}/ unless regex.is_a?(Regexp)
+ list = list.select {|s|
regex.match(s.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::TNAME))}
+ list.each do |e|
+ if
e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::REPLICATIONTYPE)
==
org.apache.hadoop.hbase.client.replication.ReplicationAdmin::REPLICATIONGLOBAL
+ replicateType = "GLOBAL"
+ else
+ replicateType = "unknown"
+ end
+
formatter.row([e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::TNAME)
+ ":" +
e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::CFNAME),
replicateType], true, [32])
+ end
+ formatter.footer(now)
+ end
+ end
+ end
+end
Modified:
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java?rev=1512489&r1=1512488&r2=1512489&view=diff
==============================================================================
---
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
(original)
+++
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
Fri Aug 9 20:45:30 2013
@@ -18,11 +18,19 @@
package org.apache.hadoop.hbase.replication;
+import java.util.HashMap;
+import java.util.List;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
import org.apache.hadoop.hbase.mapreduce.replication.VerifyReplication;
import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
@@ -39,6 +47,7 @@ import org.junit.experimental.categories
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
+import static org.junit.Assert.assertTrue;
@Category(LargeTests.class)
public class TestReplicationSmallTests extends TestReplicationBase {
@@ -477,5 +486,59 @@ public class TestReplicationSmallTests e
WALEdit edit = WALEdit.createCompaction(compactionDescriptor);
Replication.scopeWALEdits(htable1.getTableDescriptor(), new HLogKey(),
edit);
}
+
+ /**
+ * Test for HBASE-8663
+ * Create two new Tables with colfamilies enabled for replication then run
+ * ReplicationAdmin.listReplicated(). Finally verify the table:colfamilies.
Note:
+ * TestReplicationAdmin is a better place for this testing but it would need
mocks.
+ * @throws Exception
+ */
+ @Test(timeout = 300000)
+ public void testVerifyListReplicatedTable() throws Exception {
+ LOG.info("testVerifyListReplicatedTable");
+
+ final String tName = "VerifyListReplicated_";
+ final String colFam = "cf1";
+ final int numOfTables = 3;
+
+ HBaseAdmin hadmin = new HBaseAdmin(conf1);
+
+ // Create Tables
+ for (int i = 0; i < numOfTables; i++) {
+ HTableDescriptor ht = new HTableDescriptor(TableName.valueOf(tName + i));
+ HColumnDescriptor cfd = new HColumnDescriptor(colFam);
+ cfd.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
+ ht.addFamily(cfd);
+ hadmin.createTable(ht);
+ }
+
+ // verify the result
+ List<HashMap<String, String>> replicationColFams = admin.listReplicated();
+ int[] match = new int[numOfTables]; // array of 3 with init value of zero
+
+ for (int i = 0; i < replicationColFams.size(); i++) {
+ HashMap<String, String> replicationEntry = replicationColFams.get(i);
+ String tn = replicationEntry.get(ReplicationAdmin.TNAME);
+ if ((tn.startsWith(tName)) &&
replicationEntry.get(ReplicationAdmin.CFNAME).equals(colFam)) {
+ int m = Integer.parseInt(tn.substring(tn.length() - 1)); // get the
last digit
+ match[m]++; // should only increase once
+ }
+ }
+
+ // check the matching result
+ for (int i = 0; i < match.length; i++) {
+ assertTrue("listReplicated() does not match table " + i, (match[i] ==
1));
+ }
+
+ // drop tables
+ for (int i = 0; i < numOfTables; i++) {
+ String ht = tName + i;
+ hadmin.disableTable(ht);
+ hadmin.deleteTable(ht);
+ }
+
+ hadmin.close();
+ }
}
