HDFS-9645. DiskBalancer: Add Query RPC. (Contributed by Anu Engineer)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96fe685b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96fe685b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96fe685b Branch: refs/heads/HDFS-1312 Commit: 96fe685b7a4d8db63caabae9fae4987627f08231 Parents: 0501d43 Author: Arpit Agarwal <a...@apache.org> Authored: Wed Jan 20 10:47:30 2016 -0800 Committer: Arpit Agarwal <a...@apache.org> Committed: Thu Jun 23 18:18:48 2016 -0700 ---------------------------------------------------------------------- .../hdfs/protocol/ClientDatanodeProtocol.java | 6 ++ .../ClientDatanodeProtocolTranslatorPB.java | 22 +++++ .../hadoop/hdfs/server/datanode/WorkStatus.java | 85 ++++++++++++++++++++ .../src/main/proto/ClientDatanodeProtocol.proto | 26 ++++++ .../hadoop-hdfs/HDFS-1312_CHANGES.txt | 2 + ...tDatanodeProtocolServerSideTranslatorPB.java | 23 ++++++ .../hadoop/hdfs/server/datanode/DataNode.java | 5 ++ .../diskbalancer/TestDiskBalancerRPC.java | 48 ++++++++++- 8 files changed, 214 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/96fe685b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java index 125a3c1..705c98f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenSelector; import org.apache.hadoop.security.KerberosInfo; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenInfo; +import org.apache.hadoop.hdfs.server.datanode.WorkStatus; /** An client-datanode protocol for block recovery */ @@ -177,4 +178,9 @@ public interface ClientDatanodeProtocol { */ void cancelDiskBalancePlan(String planID) throws IOException; + + /** + * Gets the status of an executing diskbalancer Plan. + */ + WorkStatus queryDiskBalancerPlan() throws IOException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/96fe685b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java index e037fcf..59f2fd2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java @@ -54,7 +54,10 @@ import org.apache.hadoop.hdfs.protocol.proto.ReconfigurationProtocolProtos.Start import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.TriggerBlockReportRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.SubmitDiskBalancerPlanRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.CancelPlanRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.QueryPlanStatusRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.QueryPlanStatusResponseProto; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; +import org.apache.hadoop.hdfs.server.datanode.WorkStatus; import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.ProtocolMetaInterface; @@ -377,4 +380,23 @@ public class ClientDatanodeProtocolTranslatorPB implements throw ProtobufHelper.getRemoteException(e); } } + + /** + * Gets the status of an executing diskbalancer Plan. + */ + @Override + public WorkStatus queryDiskBalancerPlan() throws IOException { + try { + QueryPlanStatusRequestProto request = + QueryPlanStatusRequestProto.newBuilder().build(); + QueryPlanStatusResponseProto response = + rpcProxy.queryDiskBalancerPlan(NULL_CONTROLLER, request); + return new WorkStatus(response.hasResult() ? response.getResult() : 0, + response.hasPlanID() ? response.getPlanID() : null, + response.hasStatus() ? response.getStatus() : null, + response.hasCurrentStatus() ? response.getCurrentStatus() : null); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/96fe685b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/WorkStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/WorkStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/WorkStatus.java new file mode 100644 index 0000000..259a311 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/WorkStatus.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.hdfs.server.datanode; + +import org.apache.hadoop.classification.InterfaceAudience; + +/** + * Helper class that reports how much work has has been done by the node. + */ +@InterfaceAudience.Private +public class WorkStatus { + private int result; + private String planID; + private String status; + private String currentState; + + /** + * Constructs a workStatus Object. + * + * @param result - int + * @param planID - Plan ID + * @param status - Current Status + * @param currentState - Current State + */ + public WorkStatus(int result, String planID, String status, + String currentState) { + this.result = result; + this.planID = planID; + this.status = status; + this.currentState = currentState; + } + + /** + * Returns result. + * + * @return long + */ + public int getResult() { + return result; + } + + /** + * Returns planID. + * + * @return String + */ + public String getPlanID() { + return planID; + } + + /** + * Returns Status. + * + * @return String + */ + public String getStatus() { + return status; + } + + /** + * Gets current Status. + * + * @return - Json String + */ + public String getCurrentState() { + return currentState; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/96fe685b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto index b65766b..28c8681 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto @@ -180,6 +180,26 @@ message CancelPlanRequestProto { message CancelPlanResponseProto { } + +/** + * This message allows a client to query data node to see + * if a disk balancer plan is executing and if so what is + * the status. + */ +message QueryPlanStatusRequestProto { +} + +/** + * This message describes a plan if it is in progress + */ +message QueryPlanStatusResponseProto { + optional uint32 result = 1; + optional string status = 2; + optional string planID = 3; + optional string currentStatus = 4; + +} + /** * Protocol used from client to the Datanode. * See the request and response for details of rpc call. @@ -249,4 +269,10 @@ service ClientDatanodeProtocolService { */ rpc cancelDiskBalancerPlan(CancelPlanRequestProto) returns (CancelPlanResponseProto); + + /** + * Gets the status of an executing Plan + */ + rpc queryDiskBalancerPlan(QueryPlanStatusRequestProto) + returns (QueryPlanStatusResponseProto); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/96fe685b/hadoop-hdfs-project/hadoop-hdfs/HDFS-1312_CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/HDFS-1312_CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/HDFS-1312_CHANGES.txt index 8ceb45b..c6a5554 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/HDFS-1312_CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/HDFS-1312_CHANGES.txt @@ -19,3 +19,5 @@ HDFS-1312 Change Log HDFS-9595. DiskBalancer: Add cancelPlan RPC. (Anu Engineer via Arpit Agarwal) + HDFS-9645. DiskBalancer: Add Query RPC. (Anu Engineer via Arpit Agarwal) + http://git-wip-us.apache.org/repos/asf/hadoop/blob/96fe685b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java index 27fe803..03717c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java @@ -51,9 +51,12 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.Submit import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.SubmitDiskBalancerPlanResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.CancelPlanRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.CancelPlanResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.QueryPlanStatusRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.QueryPlanStatusResponseProto; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; +import org.apache.hadoop.hdfs.server.datanode.WorkStatus; /** * Implementation for protobuf service that forwards requests @@ -281,4 +284,24 @@ public class ClientDatanodeProtocolServerSideTranslatorPB implements } } + /** + * Gets the status of an executing Plan. + */ + @Override + public QueryPlanStatusResponseProto queryDiskBalancerPlan( + RpcController controller, QueryPlanStatusRequestProto request) + throws ServiceException { + try { + WorkStatus result = impl.queryDiskBalancerPlan(); + return QueryPlanStatusResponseProto + .newBuilder() + .setResult(result.getResult()) + .setPlanID(result.getPlanID()) + .setStatus(result.getStatus()) + .setCurrentStatus(result.getCurrentState()) + .build(); + } catch (Exception e) { + throw new ServiceException(e); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/96fe685b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index b2d9994..b5bd7b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -3317,4 +3317,9 @@ public class DataNode extends ReconfigurableBase throw new DiskbalancerException("Not Implemented", 0); } + @Override + public WorkStatus queryDiskBalancerPlan() throws IOException { + checkSuperuserPrivilege(); + throw new DiskbalancerException("Not Implemented", 0); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/96fe685b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java index 35d3f91..a127816 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java @@ -35,6 +35,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import java.io.IOException; import java.net.URI; public class TestDiskBalancerRPC { @@ -43,6 +44,7 @@ public class TestDiskBalancerRPC { private MiniDFSCluster cluster; private Configuration conf; + @Before public void setUp() throws Exception { conf = new HdfsConfiguration(); @@ -113,11 +115,51 @@ public class TestDiskBalancerRPC { // Since submitDiskBalancerPlan is not implemented yet, it throws an // Exception, this will be modified with the actual implementation. - thrown.expect(DiskbalancerException.class); - dataNode.submitDiskBalancerPlan(planHash, planVersion, 10, plan.toJson()); - + try { + dataNode.submitDiskBalancerPlan(planHash, planVersion, 10, plan.toJson()); + } catch (DiskbalancerException ex) { + // Let us ignore this for time being. + } thrown.expect(DiskbalancerException.class); dataNode.cancelDiskBalancePlan(planHash); } + + @Test + public void TestQueryTestRpc() throws Exception { + final int dnIndex = 0; + cluster.restartDataNode(dnIndex); + cluster.waitActive(); + ClusterConnector nameNodeConnector = + ConnectorFactory.getCluster(cluster.getFileSystem(0).getUri(), conf); + + DiskBalancerCluster diskBalancerCluster = new DiskBalancerCluster + (nameNodeConnector); + diskBalancerCluster.readClusterInfo(); + Assert.assertEquals(cluster.getDataNodes().size(), + diskBalancerCluster.getNodes().size()); + diskBalancerCluster.setNodesToProcess(diskBalancerCluster.getNodes()); + DiskBalancerDataNode node = diskBalancerCluster.getNodes().get(0); + GreedyPlanner planner = new GreedyPlanner(10.0f, node); + NodePlan plan = new NodePlan(node.getDataNodeName(), node.getDataNodePort + ()); + planner.balanceVolumeSet(node, node.getVolumeSets().get("DISK"), plan); + + final int planVersion = 0; // So far we support only one version. + DataNode dataNode = cluster.getDataNodes().get(dnIndex); + String planHash = DigestUtils.sha512Hex(plan.toJson()); + + // Since submitDiskBalancerPlan is not implemented yet, it throws an + // Exception, this will be modified with the actual implementation. + try { + dataNode.submitDiskBalancerPlan(planHash, planVersion, 10, plan.toJson()); + } catch (DiskbalancerException ex) { + // Let us ignore this for time being. + } + + // TODO : This will be fixed when we have implementation for this + // function in server side. + thrown.expect(DiskbalancerException.class); + dataNode.queryDiskBalancerPlan(); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org