[
https://issues.apache.org/jira/browse/YARN-10229?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
wei qi updated YARN-10229:
--------------------------
Comment: was deleted
(was: h3. patch for this issue
h3.
{code:java}
Index:
hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
---
hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
(revision 7caf768a8c9a639b6139b2cae8656c89e3d8c58d)
+++
hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
(date 1586435005249)
@@ -20,6 +20,9 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.ByteString;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.UpdateContainerTokenEvent;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerTokenUpdatedEvent;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEvent;
@@ -168,6 +171,8 @@
import java.net.InetSocketAddress;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -176,6 +181,7 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
@@ -941,13 +947,17 @@
// type AM and if the AMRMProxy service is enabled
if (amrmProxyEnabled && containerTokenIdentifier.getContainerType()
.equals(ContainerType.APPLICATION_MASTER)) {
- this.getAMRMProxyService().processApplicationStartRequest(request);
+ AtomicBoolean
isClientConnectToYRouter=checkIsClientConnectYRouter(request);
+ if(isClientConnectToYRouter.get()){
+
+
this.getAMRMProxyService().processApplicationStartRequest(request);
+ }
}
performContainerPreStartChecks(nmTokenIdentifier, request,
containerTokenIdentifier);
startContainerInternal(containerTokenIdentifier, request);
succeededContainers.add(containerId);
- } catch (YarnException e) {
+ } catch (YarnException | InterruptedException e) {
failedContainers.put(containerId,
SerializedException.newInstance(e));
} catch (InvalidToken ie) {
failedContainers
@@ -963,6 +973,21 @@
}
}
+ private AtomicBoolean checkIsClientConnectYRouter(StartContainerRequest
request) throws IOException, InterruptedException {
+ AtomicBoolean isClientConnectYRouter = new AtomicBoolean(false);
+
UserGroupInformation.getLoginUser().doAs((PrivilegedExceptionAction<Object>) ()
-> {
+ FileSystem fs = FileSystem.get(getConfig());
+ Path jobXmlPath = new
Path(request.getContainerLaunchContext().getLocalResources().get("job.xml").getResource().getFile());
+ FSDataInputStream in = fs.open(jobXmlPath);
+ String xmlContent = IOUtils.toString(in, StandardCharsets.UTF_8);
+ if (xmlContent.contains("localhost:8049")) {
+ isClientConnectYRouter.set(true);
+ }
+ return null;
+ });
+ return isClientConnectYRouter;
+ }
+
private void performContainerPreStartChecks(
NMTokenIdentifier nmTokenIdentifier, StartContainerRequest request,
ContainerTokenIdentifier containerTokenIdentifier)
{code}
)
> [Federation] Client should be able to submit application to RM directly using
> normal client conf
> ------------------------------------------------------------------------------------------------
>
> Key: YARN-10229
> URL: https://issues.apache.org/jira/browse/YARN-10229
> Project: Hadoop YARN
> Issue Type: Wish
> Components: amrmproxy, federation
> Affects Versions: 3.1.1
> Reporter: JohnsonGuo
> Priority: Major
> Attachments: ContainerManagerImpl.java
>
>
> Scenario： When enable the yarn federation feature with multi yarn clusters,
> one can submit their job to yarn-router by *modified* their client
> configuration with yarn router address.
> But if one still wants to submit their jobs via the original client (before
> enable federation) to RM directly, it will encounter the AMRMToken exception.
> That means once enable federation ,if some one want to submit job, they have
> to modify the client conf.
>
> one possible solution for this Scenario is:
> In NodeManger, when the client ApplicationMaster request comes:
> * get the client job.xml from HDFS "".
> * parse the "yarn.resourcemanager.scheduler.address" parameter in job.xml
> * if the value of the parameter is "localhost:8049"(AMRM address),then do
> the AMRMToken valid process
> * if the value of the parameter is "rm:port"(rm address),then skip the
> AMRMToken valid process
>
>
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
