Author: sradia Date: Sun Dec 4 20:44:36 2011 New Revision: 1210208 URL: http://svn.apache.org/viewvc?rev=1210208&view=rev Log: HADOOP-7862 Move the support for multiple protocols to lower layer so that Writable, PB and Avro can all use it (includes HDFS and MR changes to match) (Sanjay)
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestInterDatanodeProtocol.java Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1210208&r1=1210207&r2=1210208&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Sun Dec 4 20:44:36 2011 @@ -68,6 +68,10 @@ Trunk (unreleased changes) HDFS-2410. Further cleanup of hardcoded configuration keys and values. (suresh) + HADOOP-7862 Hdfs changes to work with HADOOP 7862: + Move the support for multiple protocols to lower layer so that Writable, + PB and Avro can all use it (Sanjay) + OPTIMIZATIONS HDFS-2477. Optimize computing the diff between a block report and the namenode state. (Tomasz Nykiel via hairong) Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1210208&r1=1210207&r2=1210208&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Sun Dec 4 20:44:36 2011 @@ -143,6 +143,7 @@ import org.apache.hadoop.ipc.ProtocolSig import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.ipc.RpcPayloadHeader.RpcKind; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.util.MBeans; import org.apache.hadoop.net.DNS; @@ -531,7 +532,7 @@ public class DataNode extends Configured InterDatanodeProtocolServerSideTranslatorR23 interDatanodeProtocolServerTranslator = new InterDatanodeProtocolServerSideTranslatorR23(this); - ipcServer.addProtocol(InterDatanodeWireProtocol.class, + ipcServer.addProtocol(RpcKind.RPC_WRITABLE, InterDatanodeWireProtocol.class, interDatanodeProtocolServerTranslator); // set service-level authorization security policy Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java?rev=1210208&r1=1210207&r2=1210208&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java Sun Dec 4 20:44:36 2011 @@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.server.pro import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.ipc.RpcPayloadHeader.RpcKind; import org.apache.hadoop.net.NetUtils; /** @@ -210,7 +211,7 @@ public class BackupNode extends NameNode super(conf, nn); JournalProtocolServerSideTranslatorR23 journalProtocolTranslator = new JournalProtocolServerSideTranslatorR23(this); - this.clientRpcServer.addProtocol(JournalWireProtocol.class, + this.clientRpcServer.addProtocol(RpcKind.RPC_WRITABLE, JournalWireProtocol.class, journalProtocolTranslator); nnRpcAddress = nn.nnRpcAddress; } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java?rev=1210208&r1=1210207&r2=1210208&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java Sun Dec 4 20:44:36 2011 @@ -84,6 +84,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.ProtocolSignature; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.ipc.RpcPayloadHeader.RpcKind; import org.apache.hadoop.net.Node; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.Groups; @@ -146,13 +147,16 @@ class NameNodeRpcServer implements Namen dnSocketAddr.getHostName(), dnSocketAddr.getPort(), serviceHandlerCount, false, conf, namesystem.getDelegationTokenSecretManager()); - this.serviceRpcServer.addProtocol(DatanodeProtocol.class, this); - this.serviceRpcServer.addProtocol(NamenodeProtocol.class, this); - this.serviceRpcServer.addProtocol( + this.serviceRpcServer.addProtocol(RpcKind.RPC_WRITABLE, + DatanodeProtocol.class, this); + this.serviceRpcServer.addProtocol(RpcKind.RPC_WRITABLE, + NamenodeProtocol.class, this); + this.serviceRpcServer.addProtocol(RpcKind.RPC_WRITABLE, RefreshAuthorizationPolicyProtocol.class, this); - this.serviceRpcServer.addProtocol( + this.serviceRpcServer.addProtocol(RpcKind.RPC_WRITABLE, RefreshUserMappingsProtocol.class, this); - this.serviceRpcServer.addProtocol(GetUserMappingsProtocol.class, this); + this.serviceRpcServer.addProtocol(RpcKind.RPC_WRITABLE, + GetUserMappingsProtocol.class, this); this.serviceRPCAddress = this.serviceRpcServer.getListenerAddress(); nn.setRpcServiceServerAddress(conf, serviceRPCAddress); @@ -167,12 +171,16 @@ class NameNodeRpcServer implements Namen clientProtocolServerTranslator, socAddr.getHostName(), socAddr.getPort(), handlerCount, false, conf, namesystem.getDelegationTokenSecretManager()); - this.clientRpcServer.addProtocol(DatanodeProtocol.class, this); - this.clientRpcServer.addProtocol(NamenodeProtocol.class, this); - this.clientRpcServer.addProtocol( + this.clientRpcServer.addProtocol(RpcKind.RPC_WRITABLE, + DatanodeProtocol.class, this); + this.clientRpcServer.addProtocol(RpcKind.RPC_WRITABLE, + NamenodeProtocol.class, this); + this.clientRpcServer.addProtocol(RpcKind.RPC_WRITABLE, RefreshAuthorizationPolicyProtocol.class, this); - this.clientRpcServer.addProtocol(RefreshUserMappingsProtocol.class, this); - this.clientRpcServer.addProtocol(GetUserMappingsProtocol.class, this); + this.clientRpcServer.addProtocol(RpcKind.RPC_WRITABLE, + RefreshUserMappingsProtocol.class, this); + this.clientRpcServer.addProtocol(RpcKind.RPC_WRITABLE, + GetUserMappingsProtocol.class, this); // set service-level authorization security policy Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java?rev=1210208&r1=1210207&r2=1210208&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java Sun Dec 4 20:44:36 2011 @@ -62,6 +62,7 @@ import org.apache.hadoop.hdfs.server.pro import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.ipc.RpcPayloadHeader.RpcKind; import org.apache.hadoop.net.NetUtils; import org.mockito.internal.stubbing.answers.ThrowsException; import org.mockito.invocation.InvocationOnMock; @@ -97,7 +98,7 @@ public class TestDFSClientRetries extend } @Override - public Writable call(String protocol, Writable param, long receiveTime) + public Writable call(RpcKind rpcKind, String protocol, Writable param, long receiveTime) throws IOException { if (sleep) { // sleep a bit Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestInterDatanodeProtocol.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestInterDatanodeProtocol.java?rev=1210208&r1=1210207&r2=1210208&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestInterDatanodeProtocol.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestInterDatanodeProtocol.java Sun Dec 4 20:44:36 2011 @@ -28,6 +28,7 @@ import java.net.SocketTimeoutException; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.ipc.RpcPayloadHeader.RpcKind; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.net.NetUtils; @@ -81,7 +82,7 @@ public class TestInterDatanodeProtocol { } @Override - public Writable call(String protocol, Writable param, long receiveTime) + public Writable call(RpcKind rpcKind, String protocol, Writable param, long receiveTime) throws IOException { if (sleep) { // sleep a bit