Author: vinodkv Date: Tue Jun 18 04:04:30 2013 New Revision: 1494019 URL: http://svn.apache.org/r1494019 Log: YARN-834. Fixed annotations for yarn-client module, reorganized packages and clearly differentiated *Async apis. Contributed by Arun C Murthy and Zhijie Shen. svn merge --ignore-ancestry -c 1494017 ../../trunk/
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java Modified: hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java?rev=1494019&r1=1494018&r2=1494019&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java Tue Jun 18 04:04:30 2013 @@ -20,9 +20,12 @@ package org.apache.hadoop.mapred; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; @@ -40,34 +43,85 @@ import org.apache.hadoop.security.token. import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; -import org.apache.hadoop.yarn.client.YarnClientImpl; +import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ProtoUtils; -public class ResourceMgrDelegate extends YarnClientImpl { +import com.google.common.annotations.VisibleForTesting; + +public class ResourceMgrDelegate extends YarnClient { private static final Log LOG = LogFactory.getLog(ResourceMgrDelegate.class); private YarnConfiguration conf; private GetNewApplicationResponse application; private ApplicationId applicationId; + @Private + @VisibleForTesting + protected YarnClient client; + private InetSocketAddress rmAddress; /** - * Delegate responsible for communicating with the Resource Manager's {@link ApplicationClientProtocol}. + * Delegate responsible for communicating with the Resource Manager's + * {@link ApplicationClientProtocol}. * @param conf the configuration object. */ public ResourceMgrDelegate(YarnConfiguration conf) { - super(); + this(conf, null); + } + + /** + * Delegate responsible for communicating with the Resource Manager's + * {@link ApplicationClientProtocol}. + * @param conf the configuration object. + * @param rmAddress the address of the Resource Manager + */ + public ResourceMgrDelegate(YarnConfiguration conf, + InetSocketAddress rmAddress) { + super(ResourceMgrDelegate.class.getName()); this.conf = conf; + this.rmAddress = rmAddress; + if (rmAddress == null) { + client = YarnClient.createYarnClient(); + } else { + client = YarnClient.createYarnClient(rmAddress); + } init(conf); start(); } + @Override + protected void serviceInit(Configuration conf) throws Exception { + if (rmAddress == null) { + this.rmAddress = conf.getSocketAddr(YarnConfiguration.RM_ADDRESS, + YarnConfiguration.DEFAULT_RM_ADDRESS, + YarnConfiguration.DEFAULT_RM_PORT); + } + client.init(conf); + super.serviceInit(conf); + } + + @Override + protected void serviceStart() throws Exception { + client.start(); + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + client.stop(); + super.serviceStop(); + } + public TaskTrackerInfo[] getActiveTrackers() throws IOException, InterruptedException { try { - return TypeConverter.fromYarnNodes(super.getNodeReports()); + return TypeConverter.fromYarnNodes(client.getNodeReports()); } catch (YarnException e) { throw new IOException(e); } @@ -75,7 +129,7 @@ public class ResourceMgrDelegate extends public JobStatus[] getAllJobs() throws IOException, InterruptedException { try { - return TypeConverter.fromYarnApps(super.getApplicationList(), this.conf); + return TypeConverter.fromYarnApps(client.getApplicationList(), this.conf); } catch (YarnException e) { throw new IOException(e); } @@ -91,7 +145,7 @@ public class ResourceMgrDelegate extends public ClusterMetrics getClusterMetrics() throws IOException, InterruptedException { try { - YarnClusterMetrics metrics = super.getYarnClusterMetrics(); + YarnClusterMetrics metrics = client.getYarnClusterMetrics(); ClusterMetrics oldMetrics = new ClusterMetrics(1, 1, 1, 1, 1, 1, metrics.getNumNodeManagers() * 10, @@ -112,7 +166,7 @@ public class ResourceMgrDelegate extends InterruptedException { try { return ProtoUtils.convertFromProtoFormat( - super.getRMDelegationToken(renewer), rmAddress); + client.getRMDelegationToken(renewer), rmAddress); } catch (YarnException e) { throw new IOException(e); } @@ -124,7 +178,7 @@ public class ResourceMgrDelegate extends public JobID getNewJobID() throws IOException, InterruptedException { try { - this.application = super.getNewApplication(); + this.application = client.getNewApplication(); this.applicationId = this.application.getApplicationId(); return TypeConverter.fromYarn(applicationId); } catch (YarnException e) { @@ -136,7 +190,7 @@ public class ResourceMgrDelegate extends InterruptedException { try { org.apache.hadoop.yarn.api.records.QueueInfo queueInfo = - super.getQueueInfo(queueName); + client.getQueueInfo(queueName); return (queueInfo == null) ? null : TypeConverter.fromYarn(queueInfo, conf); } catch (YarnException e) { @@ -147,7 +201,7 @@ public class ResourceMgrDelegate extends public QueueAclsInfo[] getQueueAclsForCurrentUser() throws IOException, InterruptedException { try { - return TypeConverter.fromYarnQueueUserAclsInfo(super + return TypeConverter.fromYarnQueueUserAclsInfo(client .getQueueAclsInfo()); } catch (YarnException e) { throw new IOException(e); @@ -156,7 +210,7 @@ public class ResourceMgrDelegate extends public QueueInfo[] getQueues() throws IOException, InterruptedException { try { - return TypeConverter.fromYarnQueueInfo(super.getAllQueues(), this.conf); + return TypeConverter.fromYarnQueueInfo(client.getAllQueues(), this.conf); } catch (YarnException e) { throw new IOException(e); } @@ -164,7 +218,7 @@ public class ResourceMgrDelegate extends public QueueInfo[] getRootQueues() throws IOException, InterruptedException { try { - return TypeConverter.fromYarnQueueInfo(super.getRootQueueInfos(), + return TypeConverter.fromYarnQueueInfo(client.getRootQueueInfos(), this.conf); } catch (YarnException e) { throw new IOException(e); @@ -174,7 +228,7 @@ public class ResourceMgrDelegate extends public QueueInfo[] getChildQueues(String parent) throws IOException, InterruptedException { try { - return TypeConverter.fromYarnQueueInfo(super.getChildQueueInfos(parent), + return TypeConverter.fromYarnQueueInfo(client.getChildQueueInfos(parent), this.conf); } catch (YarnException e) { throw new IOException(e); @@ -216,4 +270,82 @@ public class ResourceMgrDelegate extends public ApplicationId getApplicationId() { return applicationId; } + + @Override + public GetNewApplicationResponse getNewApplication() throws YarnException, + IOException { + return client.getNewApplication(); + } + + @Override + public ApplicationId + submitApplication(ApplicationSubmissionContext appContext) + throws YarnException, IOException { + return client.submitApplication(appContext); + } + + @Override + public void killApplication(ApplicationId applicationId) + throws YarnException, IOException { + client.killApplication(applicationId); + } + + @Override + public ApplicationReport getApplicationReport(ApplicationId appId) + throws YarnException, IOException { + return client.getApplicationReport(appId); + } + + @Override + public List<ApplicationReport> getApplicationList() throws YarnException, + IOException { + return client.getApplicationList(); + } + + @Override + public YarnClusterMetrics getYarnClusterMetrics() throws YarnException, + IOException { + return client.getYarnClusterMetrics(); + } + + @Override + public List<NodeReport> getNodeReports() throws YarnException, IOException { + return client.getNodeReports(); + } + + @Override + public org.apache.hadoop.yarn.api.records.Token getRMDelegationToken( + Text renewer) throws YarnException, IOException { + return client.getRMDelegationToken(renewer); + } + + @Override + public org.apache.hadoop.yarn.api.records.QueueInfo getQueueInfo( + String queueName) throws YarnException, IOException { + return client.getQueueInfo(queueName); + } + + @Override + public List<org.apache.hadoop.yarn.api.records.QueueInfo> getAllQueues() + throws YarnException, IOException { + return client.getAllQueues(); + } + + @Override + public List<org.apache.hadoop.yarn.api.records.QueueInfo> getRootQueueInfos() + throws YarnException, IOException { + return client.getRootQueueInfos(); + } + + @Override + public List<org.apache.hadoop.yarn.api.records.QueueInfo> getChildQueueInfos( + String parent) throws YarnException, IOException { + return client.getChildQueueInfos(parent); + } + + @Override + public List<QueueUserACLInfo> getQueueAclsInfo() throws YarnException, + IOException { + return client.getQueueAclsInfo(); + } } Modified: hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java?rev=1494019&r1=1494018&r2=1494019&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java Tue Jun 18 04:04:30 2013 @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.Records; @@ -67,8 +68,9 @@ public class TestResourceMgrDelegate { ResourceMgrDelegate delegate = new ResourceMgrDelegate( new YarnConfiguration()) { @Override - protected void serviceStart() { - this.rmClient = applicationsManager; + protected void serviceStart() throws Exception { + Assert.assertTrue(this.client instanceof YarnClientImpl); + ((YarnClientImpl) this.client).setRMClient(applicationsManager); } }; delegate.getRootQueues(); @@ -110,8 +112,9 @@ public class TestResourceMgrDelegate { ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate( new YarnConfiguration()) { @Override - protected void serviceStart() { - this.rmClient = applicationsManager; + protected void serviceStart() throws Exception { + Assert.assertTrue(this.client instanceof YarnClientImpl); + ((YarnClientImpl) this.client).setRMClient(applicationsManager); } }; JobStatus[] allJobs = resourceMgrDelegate.getAllJobs(); Modified: hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java?rev=1494019&r1=1494018&r2=1494019&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java Tue Jun 18 04:04:30 2013 @@ -82,6 +82,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; +import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -200,8 +201,9 @@ public class TestYARNRunner extends Test final ApplicationClientProtocol clientRMProtocol = mock(ApplicationClientProtocol.class); ResourceMgrDelegate delegate = new ResourceMgrDelegate(conf) { @Override - protected void serviceStart() { - this.rmClient = clientRMProtocol; + protected void serviceStart() throws Exception { + assertTrue(this.client instanceof YarnClientImpl); + ((YarnClientImpl) this.client).setRMClient(clientRMProtocol); } }; /* make sure kill calls finish application master */ Modified: hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java?rev=1494019&r1=1494018&r2=1494019&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java Tue Jun 18 04:04:30 2013 @@ -37,6 +37,7 @@ import org.apache.hadoop.security.token. import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; +import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -110,8 +111,9 @@ public class TestYarnClientProtocolProvi ResourceMgrDelegate rmgrDelegate = new ResourceMgrDelegate( new YarnConfiguration(conf)) { @Override - protected void serviceStart() { - this.rmClient = cRMProtocol; + protected void serviceStart() throws Exception { + assertTrue(this.client instanceof YarnClientImpl); + ((YarnClientImpl) this.client).setRMClient(cRMProtocol); } }; yrunner.setResourceMgrDelegate(rmgrDelegate);