Repository: hbase Updated Branches: refs/heads/master f04eeecff -> 3978aa3af
HBASE-13823 Procedure V2: remove unnecessaery operaions on AssignmentManager#recoverTableInDisablingState() and recoverTableInEnablingState() (Stephen Yuan Jiang) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3978aa3a Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3978aa3a Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3978aa3a Branch: refs/heads/master Commit: 3978aa3af0f16e612e0705be0983ea621cbdd588 Parents: f04eeec Author: Stephen Yuan Jiang <syuanjiang...@gmail.com> Authored: Fri Jul 15 12:26:31 2016 -0700 Committer: Stephen Yuan Jiang <syuanjiang...@gmail.com> Committed: Fri Jul 15 12:26:31 2016 -0700 ---------------------------------------------------------------------- .../hadoop/hbase/master/AssignmentManager.java | 61 ----- .../master/handler/DisableTableHandler.java | 239 ----------------- .../master/handler/EnableTableHandler.java | 265 ------------------- .../hadoop/hbase/client/TestEnableTable.java | 254 ++++++++++++++++++ .../master/handler/TestEnableTableHandler.java | 254 ------------------ 5 files changed, 254 insertions(+), 819 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/3978aa3a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index 55a15ee..2ffe466 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -75,8 +75,6 @@ import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper; import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer; -import org.apache.hadoop.hbase.master.handler.DisableTableHandler; -import org.apache.hadoop.hbase.master.handler.EnableTableHandler; import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition; import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; @@ -424,8 +422,6 @@ public class AssignmentManager { // it will reconstruct master state and cleanup any leftovers from previous master process. boolean failover = processDeadServersAndRegionsInTransition(deadServers); - recoverTableInDisablingState(); - recoverTableInEnablingState(); LOG.info("Joined the cluster in " + (System.currentTimeMillis() - startTime) + "ms, failover=" + failover); } @@ -1766,63 +1762,6 @@ public class AssignmentManager { } /** - * Recover the tables that were not fully moved to DISABLED state. These - * tables are in DISABLING state when the master restarted/switched. - * - * @throws KeeperException - * @throws TableNotFoundException - * @throws IOException - */ - private void recoverTableInDisablingState() - throws KeeperException, IOException { - Set<TableName> disablingTables = - tableStateManager.getTablesInStates(TableState.State.DISABLING); - if (disablingTables.size() != 0) { - for (TableName tableName : disablingTables) { - // Recover by calling DisableTableHandler - LOG.info("The table " + tableName - + " is in DISABLING state. Hence recovering by moving the table" - + " to DISABLED state."); - new DisableTableHandler(this.server, tableName, - this, tableLockManager, true).prepare().process(); - } - } - } - - /** - * Recover the tables that are not fully moved to ENABLED state. These tables - * are in ENABLING state when the master restarted/switched - * - * @throws KeeperException - * @throws org.apache.hadoop.hbase.TableNotFoundException - * @throws IOException - */ - private void recoverTableInEnablingState() - throws KeeperException, IOException { - Set<TableName> enablingTables = tableStateManager. - getTablesInStates(TableState.State.ENABLING); - if (enablingTables.size() != 0) { - for (TableName tableName : enablingTables) { - // Recover by calling EnableTableHandler - LOG.info("The table " + tableName - + " is in ENABLING state. Hence recovering by moving the table" - + " to ENABLED state."); - // enableTable in sync way during master startup, - // no need to invoke coprocessor - EnableTableHandler eth = new EnableTableHandler(this.server, tableName, - this, tableLockManager, true); - try { - eth.prepare(); - } catch (TableNotFoundException e) { - LOG.warn("Table " + tableName + " not found in hbase:meta to recover."); - continue; - } - eth.process(); - } - } - } - - /** * Processes list of regions in transition at startup */ void processRegionsInTransition(Collection<RegionState> regionsInTransition) { http://git-wip-us.apache.org/repos/asf/hbase/blob/3978aa3a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java deleted file mode 100644 index bfa9b5f..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java +++ /dev/null @@ -1,239 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.handler; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.ExecutorService; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.Server; -import org.apache.hadoop.hbase.TableNotEnabledException; -import org.apache.hadoop.hbase.TableNotFoundException; -import org.apache.hadoop.hbase.MetaTableAccessor; -import org.apache.hadoop.hbase.client.TableState; -import org.apache.hadoop.hbase.constraint.ConstraintException; -import org.apache.hadoop.hbase.executor.EventHandler; -import org.apache.hadoop.hbase.executor.EventType; -import org.apache.hadoop.hbase.master.AssignmentManager; -import org.apache.hadoop.hbase.master.BulkAssigner; -import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.master.MasterCoprocessorHost; -import org.apache.hadoop.hbase.master.RegionState; -import org.apache.hadoop.hbase.master.RegionStates; -import org.apache.hadoop.hbase.master.TableLockManager; -import org.apache.hadoop.hbase.master.TableLockManager.TableLock; -import org.apache.htrace.Trace; - -/** - * Handler to run disable of a table. - */ -@InterfaceAudience.Private -public class DisableTableHandler extends EventHandler { - private static final Log LOG = LogFactory.getLog(DisableTableHandler.class); - private final TableName tableName; - private final AssignmentManager assignmentManager; - private final TableLockManager tableLockManager; - private final boolean skipTableStateCheck; - private TableLock tableLock; - - public DisableTableHandler(Server server, TableName tableName, - AssignmentManager assignmentManager, TableLockManager tableLockManager, - boolean skipTableStateCheck) { - super(server, EventType.C_M_DISABLE_TABLE); - this.tableName = tableName; - this.assignmentManager = assignmentManager; - this.tableLockManager = tableLockManager; - this.skipTableStateCheck = skipTableStateCheck; - } - - public DisableTableHandler prepare() - throws TableNotFoundException, TableNotEnabledException, IOException { - if(tableName.equals(TableName.META_TABLE_NAME)) { - throw new ConstraintException("Cannot disable catalog table"); - } - //acquire the table write lock, blocking - this.tableLock = this.tableLockManager.writeLock(tableName, - EventType.C_M_DISABLE_TABLE.toString()); - this.tableLock.acquire(); - - boolean success = false; - try { - // Check if table exists - if (!MetaTableAccessor.tableExists(this.server.getConnection(), tableName)) { - throw new TableNotFoundException(tableName); - } - - // There could be multiple client requests trying to disable or enable - // the table at the same time. Ensure only the first request is honored - // After that, no other requests can be accepted until the table reaches - // DISABLED or ENABLED. - //TODO: reevaluate this since we have table locks now - if (!skipTableStateCheck) { - TableState.State state = this.assignmentManager. - getTableStateManager().setTableStateIfInStates( - this.tableName, TableState.State.DISABLING, - TableState.State.ENABLED); - if (state!=null) { - LOG.info("Table " + tableName + " isn't enabled;is "+state.name()+"; skipping disable"); - throw new TableNotEnabledException(this.tableName+" state is "+state.name()); - } - } - success = true; - } finally { - if (!success) { - releaseTableLock(); - } - } - - return this; - } - - @Override - public String toString() { - String name = "UnknownServerName"; - if(server != null && server.getServerName() != null) { - name = server.getServerName().toString(); - } - return getClass().getSimpleName() + "-" + name + "-" + getSeqid() + "-" + - tableName; - } - - @Override - public void process() { - try { - LOG.info("Attempting to disable table " + this.tableName); - MasterCoprocessorHost cpHost = ((HMaster) this.server) - .getMasterCoprocessorHost(); - if (cpHost != null) { - cpHost.preDisableTableAction(this.tableName); - } - handleDisableTable(); - if (cpHost != null) { - cpHost.postCompletedDisableTableAction(this.tableName); - } - } catch (IOException e) { - LOG.error("Error trying to disable table " + this.tableName, e); - } finally { - releaseTableLock(); - } - } - - private void releaseTableLock() { - if (this.tableLock != null) { - try { - this.tableLock.release(); - } catch (IOException ex) { - LOG.warn("Could not release the table lock", ex); - } - } - } - - private void handleDisableTable() throws IOException { - // Set table disabling flag up in zk. - this.assignmentManager.getTableStateManager().setTableState(this.tableName, - TableState.State.DISABLING); - boolean done = false; - while (true) { - // Get list of online regions that are of this table. Regions that are - // already closed will not be included in this list; i.e. the returned - // list is not ALL regions in a table, its all online regions according - // to the in-memory state on this master. - final List<HRegionInfo> regions = this.assignmentManager - .getRegionStates().getRegionsOfTable(tableName); - if (regions.size() == 0) { - done = true; - break; - } - LOG.info("Offlining " + regions.size() + " regions."); - BulkDisabler bd = new BulkDisabler(this.server, regions); - try { - if (bd.bulkAssign()) { - done = true; - break; - } - } catch (InterruptedException e) { - LOG.warn("Disable was interrupted"); - // Preserve the interrupt. - Thread.currentThread().interrupt(); - break; - } - } - // Flip the table to disabled if success. - if (done) this.assignmentManager.getTableStateManager().setTableState(this.tableName, - TableState.State.DISABLED); - LOG.info("Disabled table, " + this.tableName + ", is done=" + done); - } - - /** - * Run bulk disable. - */ - class BulkDisabler extends BulkAssigner { - private final List<HRegionInfo> regions; - - BulkDisabler(final Server server, final List<HRegionInfo> regions) { - super(server); - this.regions = regions; - } - - @Override - protected void populatePool(ExecutorService pool) { - RegionStates regionStates = assignmentManager.getRegionStates(); - for (HRegionInfo region: regions) { - if (regionStates.isRegionInTransition(region) - && !regionStates.isRegionInState(region, RegionState.State.FAILED_CLOSE)) { - continue; - } - final HRegionInfo hri = region; - pool.execute(Trace.wrap("DisableTableHandler.BulkDisabler",new Runnable() { - public void run() { - assignmentManager.unassign(hri); - } - })); - } - } - - @Override - protected boolean waitUntilDone(long timeout) - throws InterruptedException { - long startTime = System.currentTimeMillis(); - long remaining = timeout; - List<HRegionInfo> regions = null; - long lastLogTime = startTime; - while (!server.isStopped() && remaining > 0) { - Thread.sleep(waitingTimeForEvents); - regions = assignmentManager.getRegionStates().getRegionsOfTable(tableName); - long now = System.currentTimeMillis(); - // Don't log more than once every ten seconds. Its obnoxious. And only log table regions - // if we are waiting a while for them to go down... - if (LOG.isDebugEnabled() && ((now - lastLogTime) > 10000)) { - lastLogTime = now; - LOG.debug("Disable waiting until done; " + remaining + " ms remaining; " + regions); - } - if (regions.isEmpty()) break; - remaining = timeout - (now - startTime); - } - return regions != null && regions.isEmpty(); - } - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/3978aa3a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java deleted file mode 100644 index 7edf3f1..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java +++ /dev/null @@ -1,265 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.handler; - -import java.io.IOException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.MetaTableAccessor; -import org.apache.hadoop.hbase.Server; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.TableNotDisabledException; -import org.apache.hadoop.hbase.TableNotFoundException; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.client.TableState; -import org.apache.hadoop.hbase.executor.EventHandler; -import org.apache.hadoop.hbase.executor.EventType; -import org.apache.hadoop.hbase.master.AssignmentManager; -import org.apache.hadoop.hbase.master.BulkAssigner; -import org.apache.hadoop.hbase.master.GeneralBulkAssigner; -import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.master.MasterCoprocessorHost; -import org.apache.hadoop.hbase.master.MasterServices; -import org.apache.hadoop.hbase.master.RegionStates; -import org.apache.hadoop.hbase.master.ServerManager; -import org.apache.hadoop.hbase.master.TableLockManager; -import org.apache.hadoop.hbase.master.TableLockManager.TableLock; -import org.apache.hadoop.hbase.util.Pair; -import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; - -/** - * Handler to run enable of a table. - */ -@InterfaceAudience.Private -public class EnableTableHandler extends EventHandler { - private static final Log LOG = LogFactory.getLog(EnableTableHandler.class); - private final TableName tableName; - private final AssignmentManager assignmentManager; - private final TableLockManager tableLockManager; - private boolean skipTableStateCheck = false; - private TableLock tableLock; - private MasterServices services; - - public EnableTableHandler(Server server, TableName tableName, - AssignmentManager assignmentManager, TableLockManager tableLockManager, - boolean skipTableStateCheck) { - super(server, EventType.C_M_ENABLE_TABLE); - this.tableName = tableName; - this.assignmentManager = assignmentManager; - this.tableLockManager = tableLockManager; - this.skipTableStateCheck = skipTableStateCheck; - } - - public EnableTableHandler(MasterServices services, TableName tableName, - AssignmentManager assignmentManager, - TableLockManager tableLockManager, boolean skipTableStateCheck) { - this((Server)services, tableName, assignmentManager, tableLockManager, - skipTableStateCheck); - this.services = services; - } - - public EnableTableHandler prepare() - throws TableNotFoundException, TableNotDisabledException, IOException { - //acquire the table write lock, blocking - this.tableLock = this.tableLockManager.writeLock(tableName, - EventType.C_M_ENABLE_TABLE.toString()); - this.tableLock.acquire(); - - boolean success = false; - try { - // Check if table exists - if (!MetaTableAccessor.tableExists(this.server.getConnection(), tableName)) { - throw new TableNotFoundException(tableName); - } - - // There could be multiple client requests trying to disable or enable - // the table at the same time. Ensure only the first request is honored - // After that, no other requests can be accepted until the table reaches - // DISABLED or ENABLED. - if (!skipTableStateCheck) { - TableState.State state = this.assignmentManager - .getTableStateManager().setTableStateIfInStates( - this.tableName, TableState.State.ENABLING, - TableState.State.DISABLED); - if (state!=null) { - LOG.info("Table " + tableName + " isn't disabled;is "+state.name()+"; skipping enable"); - throw new TableNotDisabledException(this.tableName+" state is "+state.name()); - } - } - success = true; - } finally { - if (!success) { - releaseTableLock(); - } - } - return this; - } - - @Override - public String toString() { - String name = "UnknownServerName"; - if(server != null && server.getServerName() != null) { - name = server.getServerName().toString(); - } - return getClass().getSimpleName() + "-" + name + "-" + getSeqid() + "-" + - tableName; - } - - @Override - public void process() { - try { - LOG.info("Attempting to enable the table " + this.tableName); - MasterCoprocessorHost cpHost = ((HMaster) this.server) - .getMasterCoprocessorHost(); - if (cpHost != null) { - cpHost.preEnableTableAction(this.tableName); - } - handleEnableTable(); - if (cpHost != null) { - cpHost.postCompletedEnableTableAction(this.tableName); - } - } catch (IOException | InterruptedException e) { - LOG.error("Error trying to enable the table " + this.tableName, e); - } finally { - releaseTableLock(); - } - } - - private void releaseTableLock() { - if (this.tableLock != null) { - try { - this.tableLock.release(); - } catch (IOException ex) { - LOG.warn("Could not release the table lock", ex); - } - } - } - - private void handleEnableTable() throws IOException, - InterruptedException { - // I could check table is disabling and if so, not enable but require - // that user first finish disabling but that might be obnoxious. - - this.assignmentManager.getTableStateManager().setTableState(this.tableName, - TableState.State.ENABLING); - boolean done = false; - ServerManager serverManager = ((HMaster)this.server).getServerManager(); - // Get the regions of this table. We're done when all listed - // tables are onlined. - List<Pair<HRegionInfo, ServerName>> tableRegionsAndLocations; - if (TableName.META_TABLE_NAME.equals(tableName)) { - tableRegionsAndLocations = new MetaTableLocator().getMetaRegionsAndLocations( - server.getZooKeeper()); - } else { - tableRegionsAndLocations = MetaTableAccessor.getTableRegionsAndLocations( - server.getConnection(), tableName, true); - } - - int countOfRegionsInTable = tableRegionsAndLocations.size(); - Map<HRegionInfo, ServerName> regionsToAssign = - regionsToAssignWithServerName(tableRegionsAndLocations); - if (services != null) { - // need to potentially create some regions for the replicas - List<HRegionInfo> unrecordedReplicas = AssignmentManager.replicaRegionsNotRecordedInMeta( - new HashSet<HRegionInfo>(regionsToAssign.keySet()), services); - Map<ServerName, List<HRegionInfo>> srvToUnassignedRegs = - this.assignmentManager.getBalancer().roundRobinAssignment(unrecordedReplicas, - serverManager.getOnlineServersList()); - if (srvToUnassignedRegs != null) { - for (Map.Entry<ServerName, List<HRegionInfo>> entry : srvToUnassignedRegs.entrySet()) { - for (HRegionInfo h : entry.getValue()) { - regionsToAssign.put(h, entry.getKey()); - } - } - } - } - int regionsCount = regionsToAssign.size(); - if (regionsCount == 0) { - done = true; - } - LOG.info("Table '" + this.tableName + "' has " + countOfRegionsInTable - + " regions, of which " + regionsCount + " are offline."); - List<ServerName> onlineServers = serverManager.createDestinationServersList(); - Map<ServerName, List<HRegionInfo>> bulkPlan = - this.assignmentManager.getBalancer().retainAssignment(regionsToAssign, onlineServers); - if (bulkPlan != null) { - LOG.info("Bulk assigning " + regionsCount + " region(s) across " + bulkPlan.size() - + " server(s), retainAssignment=true"); - - BulkAssigner ba = - new GeneralBulkAssigner(this.server, bulkPlan, this.assignmentManager, true); - try { - if (ba.bulkAssign()) { - done = true; - } - } catch (InterruptedException e) { - LOG.warn("Enable operation was interrupted when enabling table '" - + this.tableName + "'"); - // Preserve the interrupt. - Thread.currentThread().interrupt(); - } - } else { - LOG.info("Balancer was unable to find suitable servers for table " + tableName - + ", leaving unassigned"); - done = true; - } - if (done) { - // Flip the table to enabled. - this.assignmentManager.getTableStateManager().setTableState( - this.tableName, TableState.State.ENABLED); - LOG.info("Table '" + this.tableName - + "' was successfully enabled. Status: done=" + done); - } else { - LOG.warn("Table '" + this.tableName - + "' wasn't successfully enabled. Status: done=" + done); - } - } - - /** - * @param regionsInMeta - * @return List of regions neither in transition nor assigned. - * @throws IOException - */ - private Map<HRegionInfo, ServerName> regionsToAssignWithServerName( - final List<Pair<HRegionInfo, ServerName>> regionsInMeta) throws IOException { - Map<HRegionInfo, ServerName> regionsToAssign = - new HashMap<HRegionInfo, ServerName>(regionsInMeta.size()); - RegionStates regionStates = this.assignmentManager.getRegionStates(); - for (Pair<HRegionInfo, ServerName> regionLocation : regionsInMeta) { - HRegionInfo hri = regionLocation.getFirst(); - ServerName sn = regionLocation.getSecond(); - if (regionStates.isRegionOffline(hri)) { - regionsToAssign.put(hri, sn); - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("Skipping assign for the region " + hri + " during enable table " - + hri.getTable() + " because its already in tranition or assigned."); - } - } - } - return regionsToAssign; - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/3978aa3a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestEnableTable.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestEnableTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestEnableTable.java new file mode 100644 index 0000000..0de04f3 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestEnableTable.java @@ -0,0 +1,254 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.util.ArrayList; +import java.util.List; +import java.io.IOException; +import java.util.concurrent.CountDownLatch; + +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@Category({ MasterTests.class, MediumTests.class }) +public class TestEnableTable { + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final Log LOG = LogFactory.getLog(TestEnableTable.class); + private static final byte[] FAMILYNAME = Bytes.toBytes("fam"); + + @Before + public void setUp() throws Exception { + TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, + MasterSyncObserver.class.getName()); + TEST_UTIL.startMiniCluster(1); + } + + @After + public void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test(timeout = 300000) + public void testEnableTableWithNoRegionServers() throws Exception { + final TableName tableName = TableName.valueOf("testEnableTableWithNoRegionServers"); + final MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + final HMaster m = cluster.getMaster(); + final Admin admin = TEST_UTIL.getHBaseAdmin(); + final HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(FAMILYNAME)); + admin.createTable(desc); + admin.disableTable(tableName); + TEST_UTIL.waitTableDisabled(tableName.getName()); + + admin.enableTable(tableName); + TEST_UTIL.waitTableEnabled(tableName); + // disable once more + admin.disableTable(tableName); + + TEST_UTIL.waitUntilNoRegionsInTransition(60000); + // now stop region servers + JVMClusterUtil.RegionServerThread rs = cluster.getRegionServerThreads().get(0); + rs.getRegionServer().stop("stop"); + cluster.waitForRegionServerToStop(rs.getRegionServer().getServerName(), 10000); + + LOG.debug("Now enabling table " + tableName); + + admin.enableTable(tableName); + assertTrue(admin.isTableEnabled(tableName)); + + JVMClusterUtil.RegionServerThread rs2 = cluster.startRegionServer(); + cluster.waitForRegionServerToStart(rs2.getRegionServer().getServerName().getHostname(), + rs2.getRegionServer().getServerName().getPort(), 60000); + + List<HRegionInfo> regions = TEST_UTIL.getHBaseAdmin().getTableRegions(tableName); + assertEquals(1, regions.size()); + for (HRegionInfo region : regions) { + TEST_UTIL.getHBaseAdmin().assign(region.getEncodedNameAsBytes()); + } + LOG.debug("Waiting for table assigned " + tableName); + TEST_UTIL.waitUntilAllRegionsAssigned(tableName); + List<HRegionInfo> onlineRegions = admin.getOnlineRegions( + rs2.getRegionServer().getServerName()); + ArrayList<HRegionInfo> tableRegions = filterTableRegions(tableName, onlineRegions); + assertEquals(1, tableRegions.size()); + } + + private ArrayList<HRegionInfo> filterTableRegions(final TableName tableName, + List<HRegionInfo> onlineRegions) { + return Lists.newArrayList(Iterables.filter(onlineRegions, new Predicate<HRegionInfo>() { + @Override + public boolean apply(HRegionInfo input) { + return input.getTable().equals(tableName); + } + })); + } + + /** + * We were only clearing rows that had a hregioninfo column in hbase:meta. Mangled rows that + * were missing the hregioninfo because of error were being left behind messing up any + * subsequent table made with the same name. HBASE-12980 + * @throws IOException + * @throws InterruptedException + */ + @Test(timeout=60000) + public void testDeleteForSureClearsAllTableRowsFromMeta() + throws IOException, InterruptedException { + final TableName tableName = TableName.valueOf("testDeleteForSureClearsAllTableRowsFromMeta"); + final Admin admin = TEST_UTIL.getHBaseAdmin(); + final HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(FAMILYNAME)); + try { + createTable(TEST_UTIL, desc, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); + } catch (Exception e) { + e.printStackTrace(); + fail("Got an exception while creating " + tableName); + } + // Now I have a nice table, mangle it by removing the HConstants.REGIONINFO_QUALIFIER_STR + // content from a few of the rows. + try (Table metaTable = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME)) { + try (ResultScanner scanner = metaTable.getScanner( + MetaTableAccessor.getScanForTableName(TEST_UTIL.getConnection(), tableName))) { + for (Result result : scanner) { + // Just delete one row. + Delete d = new Delete(result.getRow()); + d.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + LOG.info("Mangled: " + d); + metaTable.delete(d); + break; + } + } + admin.disableTable(tableName); + TEST_UTIL.waitTableDisabled(tableName.getName()); + // Rely on the coprocessor based latch to make the operation synchronous. + try { + deleteTable(TEST_UTIL, tableName); + } catch (Exception e) { + e.printStackTrace(); + fail("Got an exception while deleting " + tableName); + } + int rowCount = 0; + try (ResultScanner scanner = + metaTable.getScanner(MetaTableAccessor.getScanForTableName(TEST_UTIL.getConnection(), tableName))) { + for (Result result : scanner) { + LOG.info("Found when none expected: " + result); + rowCount++; + } + } + assertEquals(0, rowCount); + } + } + + public static class MasterSyncObserver extends BaseMasterObserver { + volatile CountDownLatch tableCreationLatch = null; + volatile CountDownLatch tableDeletionLatch = null; + + @Override + public void postCompletedCreateTableAction( + final ObserverContext<MasterCoprocessorEnvironment> ctx, + final HTableDescriptor desc, + final HRegionInfo[] regions) throws IOException { + // the AccessController test, some times calls only and directly the + // postCompletedCreateTableAction() + if (tableCreationLatch != null) { + tableCreationLatch.countDown(); + } + } + + @Override + public void postCompletedDeleteTableAction( + final ObserverContext<MasterCoprocessorEnvironment> ctx, + final TableName tableName) + throws IOException { + // the AccessController test, some times calls only and directly the postDeleteTableHandler() + if (tableDeletionLatch != null) { + tableDeletionLatch.countDown(); + } + } + } + + public static void createTable(HBaseTestingUtility testUtil, + HTableDescriptor htd, byte [][] splitKeys) + throws Exception { + // NOTE: We need a latch because admin is not sync, + // so the postOp coprocessor method may be called after the admin operation returned. + MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster() + .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName()); + observer.tableCreationLatch = new CountDownLatch(1); + Admin admin = testUtil.getHBaseAdmin(); + if (splitKeys != null) { + admin.createTable(htd, splitKeys); + } else { + admin.createTable(htd); + } + observer.tableCreationLatch.await(); + observer.tableCreationLatch = null; + testUtil.waitUntilAllRegionsAssigned(htd.getTableName()); + } + + public static void deleteTable(HBaseTestingUtility testUtil, TableName tableName) + throws Exception { + // NOTE: We need a latch because admin is not sync, + // so the postOp coprocessor method may be called after the admin operation returned. + MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster() + .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName()); + observer.tableDeletionLatch = new CountDownLatch(1); + Admin admin = testUtil.getHBaseAdmin(); + try { + admin.disableTable(tableName); + } catch (Exception e) { + LOG.debug("Table: " + tableName + " already disabled, so just deleting it."); + } + admin.deleteTable(tableName); + observer.tableDeletionLatch.await(); + observer.tableDeletionLatch = null; + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/3978aa3a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestEnableTableHandler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestEnableTableHandler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestEnableTableHandler.java deleted file mode 100644 index a01ccc4..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestEnableTableHandler.java +++ /dev/null @@ -1,254 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.handler; - -import java.util.ArrayList; -import java.util.List; -import java.io.IOException; -import java.util.concurrent.CountDownLatch; - -import com.google.common.base.Predicate; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.MetaTableAccessor; -import org.apache.hadoop.hbase.MiniHBaseCluster; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver; -import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; -import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; -import org.apache.hadoop.hbase.coprocessor.ObserverContext; -import org.apache.hadoop.hbase.testclassification.MasterTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.JVMClusterUtil; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -@Category({ MasterTests.class, MediumTests.class }) -public class TestEnableTableHandler { - private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private static final Log LOG = LogFactory.getLog(TestEnableTableHandler.class); - private static final byte[] FAMILYNAME = Bytes.toBytes("fam"); - - @Before - public void setUp() throws Exception { - TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, - MasterSyncObserver.class.getName()); - TEST_UTIL.startMiniCluster(1); - } - - @After - public void tearDown() throws Exception { - TEST_UTIL.shutdownMiniCluster(); - } - - @Test(timeout = 300000) - public void testEnableTableWithNoRegionServers() throws Exception { - final TableName tableName = TableName.valueOf("testEnableTableWithNoRegionServers"); - final MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); - final HMaster m = cluster.getMaster(); - final Admin admin = TEST_UTIL.getHBaseAdmin(); - final HTableDescriptor desc = new HTableDescriptor(tableName); - desc.addFamily(new HColumnDescriptor(FAMILYNAME)); - admin.createTable(desc); - admin.disableTable(tableName); - TEST_UTIL.waitTableDisabled(tableName.getName()); - - admin.enableTable(tableName); - TEST_UTIL.waitTableEnabled(tableName); - // disable once more - admin.disableTable(tableName); - - TEST_UTIL.waitUntilNoRegionsInTransition(60000); - // now stop region servers - JVMClusterUtil.RegionServerThread rs = cluster.getRegionServerThreads().get(0); - rs.getRegionServer().stop("stop"); - cluster.waitForRegionServerToStop(rs.getRegionServer().getServerName(), 10000); - - LOG.debug("Now enabling table " + tableName); - - admin.enableTable(tableName); - assertTrue(admin.isTableEnabled(tableName)); - - JVMClusterUtil.RegionServerThread rs2 = cluster.startRegionServer(); - cluster.waitForRegionServerToStart(rs2.getRegionServer().getServerName().getHostname(), - rs2.getRegionServer().getServerName().getPort(), 60000); - - List<HRegionInfo> regions = TEST_UTIL.getHBaseAdmin().getTableRegions(tableName); - assertEquals(1, regions.size()); - for (HRegionInfo region : regions) { - TEST_UTIL.getHBaseAdmin().assign(region.getEncodedNameAsBytes()); - } - LOG.debug("Waiting for table assigned " + tableName); - TEST_UTIL.waitUntilAllRegionsAssigned(tableName); - List<HRegionInfo> onlineRegions = admin.getOnlineRegions( - rs2.getRegionServer().getServerName()); - ArrayList<HRegionInfo> tableRegions = filterTableRegions(tableName, onlineRegions); - assertEquals(1, tableRegions.size()); - } - - private ArrayList<HRegionInfo> filterTableRegions(final TableName tableName, - List<HRegionInfo> onlineRegions) { - return Lists.newArrayList(Iterables.filter(onlineRegions, new Predicate<HRegionInfo>() { - @Override - public boolean apply(HRegionInfo input) { - return input.getTable().equals(tableName); - } - })); - } - - /** - * We were only clearing rows that had a hregioninfo column in hbase:meta. Mangled rows that - * were missing the hregioninfo because of error were being left behind messing up any - * subsequent table made with the same name. HBASE-12980 - * @throws IOException - * @throws InterruptedException - */ - @Test(timeout=60000) - public void testDeleteForSureClearsAllTableRowsFromMeta() - throws IOException, InterruptedException { - final TableName tableName = TableName.valueOf("testDeleteForSureClearsAllTableRowsFromMeta"); - final Admin admin = TEST_UTIL.getHBaseAdmin(); - final HTableDescriptor desc = new HTableDescriptor(tableName); - desc.addFamily(new HColumnDescriptor(FAMILYNAME)); - try { - createTable(TEST_UTIL, desc, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); - } catch (Exception e) { - e.printStackTrace(); - fail("Got an exception while creating " + tableName); - } - // Now I have a nice table, mangle it by removing the HConstants.REGIONINFO_QUALIFIER_STR - // content from a few of the rows. - try (Table metaTable = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME)) { - try (ResultScanner scanner = - metaTable.getScanner(MetaTableAccessor.getScanForTableName(TEST_UTIL.getConnection(), tableName))) { - for (Result result : scanner) { - // Just delete one row. - Delete d = new Delete(result.getRow()); - d.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); - LOG.info("Mangled: " + d); - metaTable.delete(d); - break; - } - } - admin.disableTable(tableName); - TEST_UTIL.waitTableDisabled(tableName.getName()); - // Rely on the coprocessor based latch to make the operation synchronous. - try { - deleteTable(TEST_UTIL, tableName); - } catch (Exception e) { - e.printStackTrace(); - fail("Got an exception while deleting " + tableName); - } - int rowCount = 0; - try (ResultScanner scanner = - metaTable.getScanner(MetaTableAccessor.getScanForTableName(TEST_UTIL.getConnection(), tableName))) { - for (Result result : scanner) { - LOG.info("Found when none expected: " + result); - rowCount++; - } - } - assertEquals(0, rowCount); - } - } - - public static class MasterSyncObserver extends BaseMasterObserver { - volatile CountDownLatch tableCreationLatch = null; - volatile CountDownLatch tableDeletionLatch = null; - - @Override - public void postCompletedCreateTableAction( - final ObserverContext<MasterCoprocessorEnvironment> ctx, - final HTableDescriptor desc, - final HRegionInfo[] regions) throws IOException { - // the AccessController test, some times calls only and directly the - // postCompletedCreateTableAction() - if (tableCreationLatch != null) { - tableCreationLatch.countDown(); - } - } - - @Override - public void postCompletedDeleteTableAction( - final ObserverContext<MasterCoprocessorEnvironment> ctx, - final TableName tableName) - throws IOException { - // the AccessController test, some times calls only and directly the postDeleteTableHandler() - if (tableDeletionLatch != null) { - tableDeletionLatch.countDown(); - } - } - } - - public static void createTable(HBaseTestingUtility testUtil, - HTableDescriptor htd, byte [][] splitKeys) - throws Exception { - // NOTE: We need a latch because admin is not sync, - // so the postOp coprocessor method may be called after the admin operation returned. - MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster() - .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName()); - observer.tableCreationLatch = new CountDownLatch(1); - Admin admin = testUtil.getHBaseAdmin(); - if (splitKeys != null) { - admin.createTable(htd, splitKeys); - } else { - admin.createTable(htd); - } - observer.tableCreationLatch.await(); - observer.tableCreationLatch = null; - testUtil.waitUntilAllRegionsAssigned(htd.getTableName()); - } - - public static void deleteTable(HBaseTestingUtility testUtil, TableName tableName) - throws Exception { - // NOTE: We need a latch because admin is not sync, - // so the postOp coprocessor method may be called after the admin operation returned. - MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster() - .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName()); - observer.tableDeletionLatch = new CountDownLatch(1); - Admin admin = testUtil.getHBaseAdmin(); - try { - admin.disableTable(tableName); - } catch (Exception e) { - LOG.debug("Table: " + tableName + " already disabled, so just deleting it."); - } - admin.deleteTable(tableName); - observer.tableDeletionLatch.await(); - observer.tableDeletionLatch = null; - } -}
