[jira] [Updated] (YARN-11215) MRAppmaster fails due to file permission bugs
[ https://issues.apache.org/jira/browse/YARN-11215?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-11215: - Description: 2022-06-21 12:30:11,175 INFO [main] org.apache.hadoop.service.AbstractService: Service JobHistoryEventHandler failed in state INITED org.apache.hadoop.yarn.exceptions.YarnRuntimeException: org.apache.hadoop.security.AccessControlException: Permission denied: user=user2, access=EXECUTE, inode="/tmp/hadoop-yarn/staging/history":user1:supergroup:drwx-- at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:506) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkTraverse(FSPermissionChecker.java:422) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:333) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermissionWithContext(FSPermissionChecker.java:370) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:240) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkTraverse(FSPermissionChecker.java:713) at org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkTraverse(FSDirectory.java:1892) at org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkTraverse(FSDirectory.java:1910) at org.apache.hadoop.hdfs.server.namenode.FSDirectory.resolvePath(FSDirectory.java:727) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.getFileInfo(FSDirStatAndListingOp.java:112) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getFileInfo(FSNamesystem.java:3350) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getFileInfo(NameNodeRpcServer.java:1208) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getFileInfo(ClientNamenodeProtocolServerSideTranslatorPB.java:1042) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:604) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:572) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:556) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1093) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1043) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:971) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1878) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2976) at org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler.serviceInit(JobHistoryEventHandler.java:232) at org.apache.hadoop.service.AbstractService.init(AbstractService.java:164) at org.apache.hadoop.service.CompositeService.serviceInit(CompositeService.java:109) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceInit(MRAppMaster.java:494) at org.apache.hadoop.service.AbstractService.init(AbstractService.java:164) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster$6.run(MRAppMaster.java:1760) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1878) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.initAndStartAppMaster(MRAppMaster.java:1757) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.main(MRAppMaster.java:1691) was: 2022-06-21 12:30:11,175 INFO [main] org.apache.hadoop.service.AbstractService: Service JobHistoryEventHandler failed in state INITED org.apache.hadoop.yarn.exceptions.YarnRuntimeException: org.apache.hadoop.security.AccessControlException: Permission denied: user=user2, access=EXECUTE, inode="/tmp/hadoop-yarn/staging/history":user1:supergroup:drwx-- at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:506) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkTraverse(FSPermissionChecker.java:422) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:333) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermissionWithContext(FSPermissionChecker.java:370) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:240)
[jira] [Created] (YARN-11215) MRAppmaster fails due to file permission bugs
lujie created YARN-11215: Summary: MRAppmaster fails due to file permission bugs Key: YARN-11215 URL: https://issues.apache.org/jira/browse/YARN-11215 Project: Hadoop YARN Issue Type: Bug Reporter: lujie 2022-06-21 12:30:11,175 INFO [main] org.apache.hadoop.service.AbstractService: Service JobHistoryEventHandler failed in state INITED org.apache.hadoop.yarn.exceptions.YarnRuntimeException: org.apache.hadoop.security.AccessControlException: Permission denied: user=user2, access=EXECUTE, inode="/tmp/hadoop-yarn/staging/history":user1:supergroup:drwx-- at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:506) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkTraverse(FSPermissionChecker.java:422) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:333) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermissionWithContext(FSPermissionChecker.java:370) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:240) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkTraverse(FSPermissionChecker.java:713) at org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkTraverse(FSDirectory.java:1892) at org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkTraverse(FSDirectory.java:1910) at org.apache.hadoop.hdfs.server.namenode.FSDirectory.resolvePath(FSDirectory.java:727) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.getFileInfo(FSDirStatAndListingOp.java:112) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getFileInfo(FSNamesystem.java:3350) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getFileInfo(NameNodeRpcServer.java:1208) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getFileInfo(ClientNamenodeProtocolServerSideTranslatorPB.java:1042) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:604) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:572) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:556) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1093) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1043) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:971) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1878) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2976) at org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler.serviceInit(JobHistoryEventHandler.java:232) at org.apache.hadoop.service.AbstractService.init(AbstractService.java:164) at org.apache.hadoop.service.CompositeService.serviceInit(CompositeService.java:109) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceInit(MRAppMaster.java:494) at org.apache.hadoop.service.AbstractService.init(AbstractService.java:164) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster$6.run(MRAppMaster.java:1760) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1878) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.initAndStartAppMaster(MRAppMaster.java:1757) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.main(MRAppMaster.java:1691) 2022-06-23 05:56:04,936 ERROR org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager: Error while trying to scan the directory hdfs://dffdddc36db0:8020/tmp/hadoop-yarn/staging/history/done_intermediate/user2 org.apache.hadoop.security.AccessControlException: Permission denied: user=mapred, access=READ_EXECUTE, inode="/tmp/hadoop-yarn/staging/history/done_intermediate/user2":user2:supergroup:drwxrwx--- at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:506) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:352) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermissionWithContext(FSPermissionChecker.java:370) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecke
[jira] [Created] (YARN-11214) Fails to run the job due to file permission bugs
lujie created YARN-11214: Summary: Fails to run the job due to file permission bugs Key: YARN-11214 URL: https://issues.apache.org/jira/browse/YARN-11214 Project: Hadoop YARN Issue Type: Bug Reporter: lujie when user1 run the job and staging dir /tmp/hadoop-yarn is created by user1, its permissson is 700. hence user2 failes to run jobs. Configuring for multihomed network 2022-06-21 11:33:31,066 INFO client.DefaultNoHARMFailoverProxyProvider: Connecting to ResourceManager at resourcemanager/172.22.0.5:8032 2022-06-21 11:33:31,296 INFO client.AHSProxy: Connecting to Application History server at historyserver/172.22.0.4:10200 Exception in thread "main" org.apache.hadoop.security.AccessControlException: Permission denied: user=user2, access=EXECUTE, inode="/tmp/hadoop-yarn":user1:supergroup:drwx-- at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:506) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkTraverse(FSPermissionChecker.java:422) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:333) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermissionWithContext(FSPermissionChecker.java:370) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:240) at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkTraverse(FSPermissionChecker.java:713) at org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkTraverse(FSDirectory.java:1892) at org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkTraverse(FSDirectory.java:1910) at org.apache.hadoop.hdfs.server.namenode.FSDirectory.resolvePath(FSDirectory.java:727) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.getFileInfo(FSDirStatAndListingOp.java:112) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getFileInfo(FSNamesystem.java:3350) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getFileInfo(NameNodeRpcServer.java:1208) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getFileInfo(ClientNamenodeProtocolServerSideTranslatorPB.java:1042) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:604) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:572) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:556) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1093) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1043) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:971) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1878) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2976) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) at org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:1741) at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1753) at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1750) at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1765) at org.apache.hadoop.mapreduce.JobSubmissionFiles.getStagingDir(JobSubmissionFiles.java:136) at org.apache.hadoop.mapreduce.JobSubmissionFiles.getStagingDir(JobSubmissionFiles.java:113) at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:148) at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1571) at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1568) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subjec
[jira] [Updated] (YARN-11151) sensitive infor may leak due to crash
[
https://issues.apache.org/jira/browse/YARN-11151?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-11151:
-
Priority: Critical (was: Major)
> sensitive infor may leak due to crash
> -
>
> Key: YARN-11151
> URL: https://issues.apache.org/jira/browse/YARN-11151
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Critical
>
> we init LevelDBCacheTimelineStore, RollingLevelDBTimelineStore,
> EntityGroupFSTimelineStore and LeveldbTimelineStore,
> LeveldbTimelineStateStore like:
>
> {code:java}
> try {
> localFS = FileSystem.getLocal(conf);
> if (!localFS.exists(dbPath)) {
> if (!localFS.mkdirs(dbPath)) {
> throw new IOException("Couldn't create directory for leveldb " +
> "timeline store " + dbPath);
> }
> localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
> }
> } finally {
> IOUtils.cleanupWithLogger(LOG, localFS);
> } {code}
>
> if node crash before setPermisson, then the permisison will be 755 forever
>
> code should be like :
> {code:java}
> try {
> localFS = FileSystem.getLocal(conf);
> if (!localFS.exists(dbPath)) {
> if (!localFS.mkdirs(dbPath)) {
> throw new IOException("Couldn't create directory for leveldb " +
> "timeline store " + dbPath);
> }
>
> }
>
> if(!localFS.getStatus(dbPath).getPermmision().equlas(LeveldbUtils.LEVELDB_DIR_UMASK))){
>localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
> }
> finally {
> IOUtils.cleanupWithLogger(LOG, localFS);
> } {code}
>
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-11151) sensitive infor may leak due to crash
[
https://issues.apache.org/jira/browse/YARN-11151?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-11151:
-
Description:
we init LevelDBCacheTimelineStore, RollingLevelDBTimelineStore,
EntityGroupFSTimelineStore and LeveldbTimelineStore, LeveldbTimelineStateStore
like:
{code:java}
try {
localFS = FileSystem.getLocal(conf);
if (!localFS.exists(dbPath)) {
if (!localFS.mkdirs(dbPath)) {
throw new IOException("Couldn't create directory for leveldb " +
"timeline store " + dbPath);
}
localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
}
} finally {
IOUtils.cleanupWithLogger(LOG, localFS);
} {code}
if node crash before setPermisson, then the permisison will be 755 forever
code should be like :
{code:java}
try {
localFS = FileSystem.getLocal(conf);
if (!localFS.exists(dbPath)) {
if (!localFS.mkdirs(dbPath)) {
throw new IOException("Couldn't create directory for leveldb " +
"timeline store " + dbPath);
}
}
if(!localFS.getStatus(dbPath).getPermmision().equlas(LeveldbUtils.LEVELDB_DIR_UMASK))){
localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
}
finally {
IOUtils.cleanupWithLogger(LOG, localFS);
} {code}
was:
we init LevelDBCacheTimelineStore, RollingLevelDBTimelineStore,
EntityGroupFSTimelineStore and LeveldbTimelineStore like:
{code:java}
try {
localFS = FileSystem.getLocal(conf);
if (!localFS.exists(dbPath)) {
if (!localFS.mkdirs(dbPath)) {
throw new IOException("Couldn't create directory for leveldb " +
"timeline store " + dbPath);
}
localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
}
} finally {
IOUtils.cleanupWithLogger(LOG, localFS);
} {code}
if node crash before setPermisson, then the permisison will be 755 forever
code should be like :
{code:java}
try {
localFS = FileSystem.getLocal(conf);
if (!localFS.exists(dbPath)) {
if (!localFS.mkdirs(dbPath)) {
throw new IOException("Couldn't create directory for leveldb " +
"timeline store " + dbPath);
}
}
if(!localFS.getStatus(dbPath).getPermmision().equlas(LeveldbUtils.LEVELDB_DIR_UMASK))){
localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
}
finally {
IOUtils.cleanupWithLogger(LOG, localFS);
} {code}
> sensitive infor may leak due to crash
> -
>
> Key: YARN-11151
> URL: https://issues.apache.org/jira/browse/YARN-11151
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Major
>
> we init LevelDBCacheTimelineStore, RollingLevelDBTimelineStore,
> EntityGroupFSTimelineStore and LeveldbTimelineStore,
> LeveldbTimelineStateStore like:
>
> {code:java}
> try {
> localFS = FileSystem.getLocal(conf);
> if (!localFS.exists(dbPath)) {
> if (!localFS.mkdirs(dbPath)) {
> throw new IOException("Couldn't create directory for leveldb " +
> "timeline store " + dbPath);
> }
> localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
> }
> } finally {
> IOUtils.cleanupWithLogger(LOG, localFS);
> } {code}
>
> if node crash before setPermisson, then the permisison will be 755 forever
>
> code should be like :
> {code:java}
> try {
> localFS = FileSystem.getLocal(conf);
> if (!localFS.exists(dbPath)) {
> if (!localFS.mkdirs(dbPath)) {
> throw new IOException("Couldn't create directory for leveldb " +
> "timeline store " + dbPath);
> }
>
> }
>
> if(!localFS.getStatus(dbPath).getPermmision().equlas(LeveldbUtils.LEVELDB_DIR_UMASK))){
>localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
> }
> finally {
> IOUtils.cleanupWithLogger(LOG, localFS);
> } {code}
>
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-11151) sensitive infor may leak due to crash
[
https://issues.apache.org/jira/browse/YARN-11151?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-11151:
-
Description:
we init LevelDBCacheTimelineStore, RollingLevelDBTimelineStore,
EntityGroupFSTimelineStore and LeveldbTimelineStore like:
{code:java}
try {
localFS = FileSystem.getLocal(conf);
if (!localFS.exists(dbPath)) {
if (!localFS.mkdirs(dbPath)) {
throw new IOException("Couldn't create directory for leveldb " +
"timeline store " + dbPath);
}
localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
}
} finally {
IOUtils.cleanupWithLogger(LOG, localFS);
} {code}
if node crash before setPermisson, then the permisison will be 755 forever
code should be like :
{code:java}
try {
localFS = FileSystem.getLocal(conf);
if (!localFS.exists(dbPath)) {
if (!localFS.mkdirs(dbPath)) {
throw new IOException("Couldn't create directory for leveldb " +
"timeline store " + dbPath);
}
}
if(!localFS.getStatus(dbPath).getPermmision().equlas(LeveldbUtils.LEVELDB_DIR_UMASK))){
localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
}
finally {
IOUtils.cleanupWithLogger(LOG, localFS);
} {code}
was:
we init LevelDBCacheTimelineStore, RollingLevelDBTimelineStore and
LeveldbTimelineStore like:
{code:java}
try {
localFS = FileSystem.getLocal(conf);
if (!localFS.exists(dbPath)) {
if (!localFS.mkdirs(dbPath)) {
throw new IOException("Couldn't create directory for leveldb " +
"timeline store " + dbPath);
}
localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
}
} finally {
IOUtils.cleanupWithLogger(LOG, localFS);
} {code}
if node crash before setPermisson, then the permisison will be 755 forever
code should be like :
{code:java}
try {
localFS = FileSystem.getLocal(conf);
if (!localFS.exists(dbPath)) {
if (!localFS.mkdirs(dbPath)) {
throw new IOException("Couldn't create directory for leveldb " +
"timeline store " + dbPath);
}
}
if(!localFS.getStatus(dbPath).getPermmision().equlas(LeveldbUtils.LEVELDB_DIR_UMASK))){
localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
}
finally {
IOUtils.cleanupWithLogger(LOG, localFS);
} {code}
> sensitive infor may leak due to crash
> -
>
> Key: YARN-11151
> URL: https://issues.apache.org/jira/browse/YARN-11151
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Major
>
> we init LevelDBCacheTimelineStore, RollingLevelDBTimelineStore,
> EntityGroupFSTimelineStore and LeveldbTimelineStore like:
>
> {code:java}
> try {
> localFS = FileSystem.getLocal(conf);
> if (!localFS.exists(dbPath)) {
> if (!localFS.mkdirs(dbPath)) {
> throw new IOException("Couldn't create directory for leveldb " +
> "timeline store " + dbPath);
> }
> localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
> }
> } finally {
> IOUtils.cleanupWithLogger(LOG, localFS);
> } {code}
>
> if node crash before setPermisson, then the permisison will be 755 forever
>
> code should be like :
> {code:java}
> try {
> localFS = FileSystem.getLocal(conf);
> if (!localFS.exists(dbPath)) {
> if (!localFS.mkdirs(dbPath)) {
> throw new IOException("Couldn't create directory for leveldb " +
> "timeline store " + dbPath);
> }
>
> }
>
> if(!localFS.getStatus(dbPath).getPermmision().equlas(LeveldbUtils.LEVELDB_DIR_UMASK))){
>localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
> }
> finally {
> IOUtils.cleanupWithLogger(LOG, localFS);
> } {code}
>
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-11151) sensitive infor may leak due to crash
[
https://issues.apache.org/jira/browse/YARN-11151?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-11151:
-
Description:
we init LevelDBCacheTimelineStore, RollingLevelDBTimelineStore and
LeveldbTimelineStore like:
{code:java}
try {
localFS = FileSystem.getLocal(conf);
if (!localFS.exists(dbPath)) {
if (!localFS.mkdirs(dbPath)) {
throw new IOException("Couldn't create directory for leveldb " +
"timeline store " + dbPath);
}
localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
}
} finally {
IOUtils.cleanupWithLogger(LOG, localFS);
} {code}
if node crash before setPermisson, then the permisison will be 755 forever
code should be like :
{code:java}
try {
localFS = FileSystem.getLocal(conf);
if (!localFS.exists(dbPath)) {
if (!localFS.mkdirs(dbPath)) {
throw new IOException("Couldn't create directory for leveldb " +
"timeline store " + dbPath);
}
}
if(!localFS.getStatus(dbPath).getPermmision().equlas(LeveldbUtils.LEVELDB_DIR_UMASK))){
localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
}
finally {
IOUtils.cleanupWithLogger(LOG, localFS);
} {code}
was:
we init LevelDBCacheTimelineStore and LeveldbTimelineStore like:
{code:java}
try {
localFS = FileSystem.getLocal(conf);
if (!localFS.exists(dbPath)) {
if (!localFS.mkdirs(dbPath)) {
throw new IOException("Couldn't create directory for leveldb " +
"timeline store " + dbPath);
}
localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
}
} finally {
IOUtils.cleanupWithLogger(LOG, localFS);
} {code}
if node crash before setPermisson, then the permisison will be 755 forever
code should be like :
{code:java}
try {
localFS = FileSystem.getLocal(conf);
if (!localFS.exists(dbPath)) {
if (!localFS.mkdirs(dbPath)) {
throw new IOException("Couldn't create directory for leveldb " +
"timeline store " + dbPath);
}
}
if(!localFS.getStatus(dbPath).getPermmision().equlas(LeveldbUtils.LEVELDB_DIR_UMASK))){
localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
}
finally {
IOUtils.cleanupWithLogger(LOG, localFS);
} {code}
> sensitive infor may leak due to crash
> -
>
> Key: YARN-11151
> URL: https://issues.apache.org/jira/browse/YARN-11151
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Major
>
> we init LevelDBCacheTimelineStore, RollingLevelDBTimelineStore and
> LeveldbTimelineStore like:
>
> {code:java}
> try {
> localFS = FileSystem.getLocal(conf);
> if (!localFS.exists(dbPath)) {
> if (!localFS.mkdirs(dbPath)) {
> throw new IOException("Couldn't create directory for leveldb " +
> "timeline store " + dbPath);
> }
> localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
> }
> } finally {
> IOUtils.cleanupWithLogger(LOG, localFS);
> } {code}
>
> if node crash before setPermisson, then the permisison will be 755 forever
>
> code should be like :
> {code:java}
> try {
> localFS = FileSystem.getLocal(conf);
> if (!localFS.exists(dbPath)) {
> if (!localFS.mkdirs(dbPath)) {
> throw new IOException("Couldn't create directory for leveldb " +
> "timeline store " + dbPath);
> }
>
> }
>
> if(!localFS.getStatus(dbPath).getPermmision().equlas(LeveldbUtils.LEVELDB_DIR_UMASK))){
>localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
> }
> finally {
> IOUtils.cleanupWithLogger(LOG, localFS);
> } {code}
>
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-11151) sensitive infor may leak due to crash
[
https://issues.apache.org/jira/browse/YARN-11151?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-11151:
-
Description:
we init LevelDBCacheTimelineStore and LeveldbTimelineStore like:
{code:java}
try {
localFS = FileSystem.getLocal(conf);
if (!localFS.exists(dbPath)) {
if (!localFS.mkdirs(dbPath)) {
throw new IOException("Couldn't create directory for leveldb " +
"timeline store " + dbPath);
}
localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
}
} finally {
IOUtils.cleanupWithLogger(LOG, localFS);
} {code}
if node crash before setPermisson, then the permisison will be 755 forever
code should be like :
{code:java}
try {
localFS = FileSystem.getLocal(conf);
if (!localFS.exists(dbPath)) {
if (!localFS.mkdirs(dbPath)) {
throw new IOException("Couldn't create directory for leveldb " +
"timeline store " + dbPath);
}
}
if(!localFS.getStatus(dbPath).getPermmision().equlas(LeveldbUtils.LEVELDB_DIR_UMASK))){
localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
}
finally {
IOUtils.cleanupWithLogger(LOG, localFS);
} {code}
was:
we init LevelDBCacheTimelineStore and LeveldbTimelineStore like:
{code:java}
try {
localFS = FileSystem.getLocal(conf);
if (!localFS.exists(dbPath)) {
if (!localFS.mkdirs(dbPath)) {
throw new IOException("Couldn't create directory for leveldb " +
"timeline store " + dbPath);
}
localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
}
} finally {
IOUtils.cleanupWithLogger(LOG, localFS);
} {code}
if node crash before setPermisson, then the permisison will be 755 forever
code should be like :
{code:java}
try {
localFS = FileSystem.getLocal(conf);
if (!localFS.exists(dbPath)) {
if (!localFS.mkdirs(dbPath)) {
throw new IOException("Couldn't create directory for leveldb " +
"timeline store " + dbPath);
}
}
if
(localFS.getStatus(dbPath).getPermmision().equlas(LeveldbUtils.LEVELDB_DIR_UMASK))){
localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);} }
finally {
IOUtils.cleanupWithLogger(LOG, localFS);
} {code}
> sensitive infor may leak due to crash
> -
>
> Key: YARN-11151
> URL: https://issues.apache.org/jira/browse/YARN-11151
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Major
>
> we init LevelDBCacheTimelineStore and LeveldbTimelineStore like:
>
> {code:java}
> try {
> localFS = FileSystem.getLocal(conf);
> if (!localFS.exists(dbPath)) {
> if (!localFS.mkdirs(dbPath)) {
> throw new IOException("Couldn't create directory for leveldb " +
> "timeline store " + dbPath);
> }
> localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
> }
> } finally {
> IOUtils.cleanupWithLogger(LOG, localFS);
> } {code}
>
> if node crash before setPermisson, then the permisison will be 755 forever
>
> code should be like :
> {code:java}
> try {
> localFS = FileSystem.getLocal(conf);
> if (!localFS.exists(dbPath)) {
> if (!localFS.mkdirs(dbPath)) {
> throw new IOException("Couldn't create directory for leveldb " +
> "timeline store " + dbPath);
> }
>
> }
>
> if(!localFS.getStatus(dbPath).getPermmision().equlas(LeveldbUtils.LEVELDB_DIR_UMASK))){
>localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
> }
> finally {
> IOUtils.cleanupWithLogger(LOG, localFS);
> } {code}
>
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Created] (YARN-11151) sensitive infor may leak due to crash
lujie created YARN-11151:
Summary: sensitive infor may leak due to crash
Key: YARN-11151
URL: https://issues.apache.org/jira/browse/YARN-11151
Project: Hadoop YARN
Issue Type: Bug
Reporter: lujie
we init LevelDBCacheTimelineStore and LeveldbTimelineStore like:
{code:java}
try {
localFS = FileSystem.getLocal(conf);
if (!localFS.exists(dbPath)) {
if (!localFS.mkdirs(dbPath)) {
throw new IOException("Couldn't create directory for leveldb " +
"timeline store " + dbPath);
}
localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);
}
} finally {
IOUtils.cleanupWithLogger(LOG, localFS);
} {code}
if node crash before setPermisson, then the permisison will be 755 forever
code should be like :
{code:java}
try {
localFS = FileSystem.getLocal(conf);
if (!localFS.exists(dbPath)) {
if (!localFS.mkdirs(dbPath)) {
throw new IOException("Couldn't create directory for leveldb " +
"timeline store " + dbPath);
}
}
if
(localFS.getStatus(dbPath).getPermmision().equlas(LeveldbUtils.LEVELDB_DIR_UMASK))){
localFS.setPermission(dbPath, LeveldbUtils.LEVELDB_DIR_UMASK);} }
finally {
IOUtils.cleanupWithLogger(LOG, localFS);
} {code}
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-11150) sensitive inform may leak due to crash
[
https://issues.apache.org/jira/browse/YARN-11150?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-11150:
-
Priority: Critical (was: Major)
> sensitive inform may leak due to crash
> --
>
> Key: YARN-11150
> URL: https://issues.apache.org/jira/browse/YARN-11150
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Critical
>
> current, we implement
>
> {code:java}
> // private FSDataOutputStream createLogFileStream(FileSystem fileSystem,
> Path logPathToCreate)
> throws IOException {
> FSDataOutputStream streamToCreate;
> if (!isAppendSupported) {
> logPathToCreate =
> new Path(logPathToCreate.getParent(),
> (logPathToCreate.getName() + "_" + Time.monotonicNow()));
> }
> if (!fileSystem.exists(logPathToCreate)) {
> streamToCreate = fileSystem.create(logPathToCreate, false);
> fileSystem.setPermission(logPathToCreate,
> new FsPermission(FILE_LOG_PERMISSIONS));
> } else {
> streamToCreate = fileSystem.append(logPathToCreate);
> }
> return streamToCreate;
> } {code}
> but if node crash before setPermission, then the permission of log file will
> be 755 forever.
> code should be like
> {code:java}
> // private FSDataOutputStream createLogFileStream(FileSystem fileSystem,
> Path logPathToCreate)
> throws IOException {
> FSDataOutputStream streamToCreate;
> if (!isAppendSupported) {
> logPathToCreate =
> new Path(logPathToCreate.getParent(),
> (logPathToCreate.getName() + "_" + Time.monotonicNow()));
> }
> if (!fileSystem.exists(logPathToCreate)) {
> streamToCreate = fileSystem.create(logPathToCreate, false);
> fileSystem.setPermission(logPathToCreate,
> new FsPermission(FILE_LOG_PERMISSIONS));
> } else {
> streamToCreate = fileSystem.append(logPathToCreate);
> }
> if
> (!fileSystem.getStatus(logPathToCreate).getPermission().equals(FILE_LOG_PERMISSIONS))
> {
> fs.setPermission(dir, FILE_LOG_PERMISSIONS);
> }
> } {code}
>
>
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-11150) sensitive inform may leak due to crash
[
https://issues.apache.org/jira/browse/YARN-11150?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-11150:
-
Description:
current, we implement
{code:java}
// private FSDataOutputStream createLogFileStream(FileSystem fileSystem,
Path logPathToCreate)
throws IOException {
FSDataOutputStream streamToCreate;
if (!isAppendSupported) {
logPathToCreate =
new Path(logPathToCreate.getParent(),
(logPathToCreate.getName() + "_" + Time.monotonicNow()));
}
if (!fileSystem.exists(logPathToCreate)) {
streamToCreate = fileSystem.create(logPathToCreate, false);
fileSystem.setPermission(logPathToCreate,
new FsPermission(FILE_LOG_PERMISSIONS));
} else {
streamToCreate = fileSystem.append(logPathToCreate);
}
return streamToCreate;
} {code}
but if node crash before setPermission, then the permission of log file will be
755 forever.
code should be like
{code:java}
// private FSDataOutputStream createLogFileStream(FileSystem fileSystem,
Path logPathToCreate)
throws IOException {
FSDataOutputStream streamToCreate;
if (!isAppendSupported) {
logPathToCreate =
new Path(logPathToCreate.getParent(),
(logPathToCreate.getName() + "_" + Time.monotonicNow()));
}
if (!fileSystem.exists(logPathToCreate)) {
streamToCreate = fileSystem.create(logPathToCreate, false);
fileSystem.setPermission(logPathToCreate,
new FsPermission(FILE_LOG_PERMISSIONS));
} else {
streamToCreate = fileSystem.append(logPathToCreate);
}
if
(!fileSystem.getStatus(logPathToCreate).getPermission().equals(FILE_LOG_PERMISSIONS))
{
fs.setPermission(dir, FILE_LOG_PERMISSIONS);
}
} {code}
was:
current, we implement
{code:java}
// private FSDataOutputStream createLogFileStream(FileSystem fileSystem,
Path logPathToCreate)
throws IOException {
FSDataOutputStream streamToCreate;
if (!isAppendSupported) {
logPathToCreate =
new Path(logPathToCreate.getParent(),
(logPathToCreate.getName() + "_" + Time.monotonicNow()));
}
if (!fileSystem.exists(logPathToCreate)) {
streamToCreate = fileSystem.create(logPathToCreate, false);
fileSystem.setPermission(logPathToCreate,
new FsPermission(FILE_LOG_PERMISSIONS));
} else {
streamToCreate = fileSystem.append(logPathToCreate);
}
return streamToCreate;
} {code}
but if node crash before setPermission, then the permission of log file will be
755 forever.
code should be like
{code:java}
// private FSDataOutputStream createLogFileStream(FileSystem fileSystem,
Path logPathToCreate)
throws IOException {
FSDataOutputStream streamToCreate;
if (!isAppendSupported) {
logPathToCreate =
new Path(logPathToCreate.getParent(),
(logPathToCreate.getName() + "_" + Time.monotonicNow()));
}
if (!fileSystem.exists(logPathToCreate)) {
streamToCreate = fileSystem.create(logPathToCreate, false);
fileSystem.setPermission(logPathToCreate,
new FsPermission(FILE_LOG_PERMISSIONS));
} else {
streamToCreate = fileSystem.append(logPathToCreate);
}
if
(!fileSystem.getStatus(logPathToCreate).getPermission().equals(FILE_LOG_PERMISSIONS))
{
fs.setPermission(dir, FILE_LOG_PERMISSIONS);
}
} {code}
> sensitive inform may leak due to crash
> --
>
> Key: YARN-11150
> URL: https://issues.apache.org/jira/browse/YARN-11150
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Major
>
> current, we implement
>
> {code:java}
> // private FSDataOutputStream createLogFileStream(FileSystem fileSystem,
> Path logPathToCreate)
> throws IOException {
> FSDataOutputStream streamToCreate;
> if (!isAppendSupported) {
> logPathToCreate =
> new Path(logPathToCreate.getParent(),
> (logPathToCreate.getName() + "_" + Time.monotonicNow()));
> }
> if (!fileSystem.exists(logPathToCreate)) {
> streamToCreate = fileSystem.create(logPathToCreate, false);
> fileSystem.setPermission(logPathToCreate,
> new FsPermission(FILE_LOG_PERMISSIONS));
> } else {
> streamToCreate = fileSystem.append(logPathToCreate);
> }
> return streamToCreate;
> } {code}
> but if node crash before setPermission, then the permission of log file will
> be 755 forever.
> code should be like
> {code:java}
> // private FSDataOutputStream createLogFileStream(FileSystem fileSystem,
> Path logPa
[jira] [Updated] (YARN-11150) sensitive inform may leak due to crash
[
https://issues.apache.org/jira/browse/YARN-11150?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-11150:
-
Description:
current, we implement
{code:java}
// private FSDataOutputStream createLogFileStream(FileSystem fileSystem,
Path logPathToCreate)
throws IOException {
FSDataOutputStream streamToCreate;
if (!isAppendSupported) {
logPathToCreate =
new Path(logPathToCreate.getParent(),
(logPathToCreate.getName() + "_" + Time.monotonicNow()));
}
if (!fileSystem.exists(logPathToCreate)) {
streamToCreate = fileSystem.create(logPathToCreate, false);
fileSystem.setPermission(logPathToCreate,
new FsPermission(FILE_LOG_PERMISSIONS));
} else {
streamToCreate = fileSystem.append(logPathToCreate);
}
return streamToCreate;
} {code}
but if node crash before setPermission, then the permission of log file will be
755 forever.
code should be like
{code:java}
// private FSDataOutputStream createLogFileStream(FileSystem fileSystem,
Path logPathToCreate)
throws IOException {
FSDataOutputStream streamToCreate;
if (!isAppendSupported) {
logPathToCreate =
new Path(logPathToCreate.getParent(),
(logPathToCreate.getName() + "_" + Time.monotonicNow()));
}
if (!fileSystem.exists(logPathToCreate)) {
streamToCreate = fileSystem.create(logPathToCreate, false);
fileSystem.setPermission(logPathToCreate,
new FsPermission(FILE_LOG_PERMISSIONS));
} else {
streamToCreate = fileSystem.append(logPathToCreate);
}
if
(!fileSystem.getStatus(logPathToCreate).getPermission().equals(FILE_LOG_PERMISSIONS))
{
fs.setPermission(dir, FILE_LOG_PERMISSIONS);
}
} {code}
> sensitive inform may leak due to crash
> --
>
> Key: YARN-11150
> URL: https://issues.apache.org/jira/browse/YARN-11150
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Major
>
> current, we implement
>
> {code:java}
> // private FSDataOutputStream createLogFileStream(FileSystem fileSystem,
> Path logPathToCreate)
> throws IOException {
> FSDataOutputStream streamToCreate;
> if (!isAppendSupported) {
> logPathToCreate =
> new Path(logPathToCreate.getParent(),
> (logPathToCreate.getName() + "_" + Time.monotonicNow()));
> }
> if (!fileSystem.exists(logPathToCreate)) {
> streamToCreate = fileSystem.create(logPathToCreate, false);
> fileSystem.setPermission(logPathToCreate,
> new FsPermission(FILE_LOG_PERMISSIONS));
> } else {
> streamToCreate = fileSystem.append(logPathToCreate);
> }
> return streamToCreate;
> } {code}
> but if node crash before setPermission, then the permission of log file will
> be 755 forever.
>
>
> code should be like
>
>
> {code:java}
> // private FSDataOutputStream createLogFileStream(FileSystem fileSystem,
> Path logPathToCreate)
> throws IOException {
> FSDataOutputStream streamToCreate;
> if (!isAppendSupported) {
> logPathToCreate =
> new Path(logPathToCreate.getParent(),
> (logPathToCreate.getName() + "_" + Time.monotonicNow()));
> }
> if (!fileSystem.exists(logPathToCreate)) {
> streamToCreate = fileSystem.create(logPathToCreate, false);
> fileSystem.setPermission(logPathToCreate,
> new FsPermission(FILE_LOG_PERMISSIONS));
> } else {
> streamToCreate = fileSystem.append(logPathToCreate);
> }
> if
> (!fileSystem.getStatus(logPathToCreate).getPermission().equals(FILE_LOG_PERMISSIONS))
> {
> fs.setPermission(dir, FILE_LOG_PERMISSIONS);
> }
> } {code}
>
>
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Created] (YARN-11150) sensitive inform may leak due to crash
lujie created YARN-11150: Summary: sensitive inform may leak due to crash Key: YARN-11150 URL: https://issues.apache.org/jira/browse/YARN-11150 Project: Hadoop YARN Issue Type: Bug Reporter: lujie -- This message was sent by Atlassian Jira (v8.20.7#820007) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Comment Edited] (YARN-10980) fix CVE-2020-8908
[ https://issues.apache.org/jira/browse/YARN-10980?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17429936#comment-17429936 ] lujie edited comment on YARN-10980 at 10/18/21, 11:20 AM: -- due to forget to fetech upstream, i open this issue. this bug is fixed by https://issues.apache.org/jira/browse/HADOOP-17653. was (Author: xiaoheipangzi): it is fixed by https://issues.apache.org/jira/browse/HADOOP-17963 > fix CVE-2020-8908 > - > > Key: YARN-10980 > URL: https://issues.apache.org/jira/browse/YARN-10980 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Major > Labels: pull-request-available > Time Spent: 10m > Remaining Estimate: 0h > > see [https://www.cvedetails.com/cve/CVE-2020-8908/] > > A temp directory creation vulnerability exists in all versions of Guava, > allowing an attacker with access to the machine to potentially access data in > a temporary directory created by the Guava API > com.google.common.io.Files.createTempDir(). By default, on unix-like systems, > the created directory is world-readable (readable by an attacker with access > to the system). The method in question has been marked @Deprecated in > versions 30.0 and later and should not be used. For Android developers, we > recommend choosing a temporary directory API provided by Android, such as > context.getCacheDir(). For other Java developers, we recommend migrating to > the Java 7 API java.nio.file.Files.createTempDirectory() which explicitly > configures permissions of 700, or configuring the Java runtime's > java.io.tmpdir system property to point to a location whose permissions are > appropriately configured. -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-10980) fix CVE-2020-8908
[ https://issues.apache.org/jira/browse/YARN-10980?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17429936#comment-17429936 ] lujie commented on YARN-10980: -- it is fixed by https://issues.apache.org/jira/browse/HADOOP-17963 > fix CVE-2020-8908 > - > > Key: YARN-10980 > URL: https://issues.apache.org/jira/browse/YARN-10980 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Major > > see [https://www.cvedetails.com/cve/CVE-2020-8908/] > > A temp directory creation vulnerability exists in all versions of Guava, > allowing an attacker with access to the machine to potentially access data in > a temporary directory created by the Guava API > com.google.common.io.Files.createTempDir(). By default, on unix-like systems, > the created directory is world-readable (readable by an attacker with access > to the system). The method in question has been marked @Deprecated in > versions 30.0 and later and should not be used. For Android developers, we > recommend choosing a temporary directory API provided by Android, such as > context.getCacheDir(). For other Java developers, we recommend migrating to > the Java 7 API java.nio.file.Files.createTempDirectory() which explicitly > configures permissions of 700, or configuring the Java runtime's > java.io.tmpdir system property to point to a location whose permissions are > appropriately configured. -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Resolved] (YARN-10980) fix CVE-2020-8908
[ https://issues.apache.org/jira/browse/YARN-10980?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie resolved YARN-10980. -- Resolution: Fixed > fix CVE-2020-8908 > - > > Key: YARN-10980 > URL: https://issues.apache.org/jira/browse/YARN-10980 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Major > > see [https://www.cvedetails.com/cve/CVE-2020-8908/] > > A temp directory creation vulnerability exists in all versions of Guava, > allowing an attacker with access to the machine to potentially access data in > a temporary directory created by the Guava API > com.google.common.io.Files.createTempDir(). By default, on unix-like systems, > the created directory is world-readable (readable by an attacker with access > to the system). The method in question has been marked @Deprecated in > versions 30.0 and later and should not be used. For Android developers, we > recommend choosing a temporary directory API provided by Android, such as > context.getCacheDir(). For other Java developers, we recommend migrating to > the Java 7 API java.nio.file.Files.createTempDirectory() which explicitly > configures permissions of 700, or configuring the Java runtime's > java.io.tmpdir system property to point to a location whose permissions are > appropriately configured. -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10980) fix CVE-2020-8908
[ https://issues.apache.org/jira/browse/YARN-10980?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-10980: - Description: see [https://www.cvedetails.com/cve/CVE-2020-8908/] A temp directory creation vulnerability exists in all versions of Guava, allowing an attacker with access to the machine to potentially access data in a temporary directory created by the Guava API com.google.common.io.Files.createTempDir(). By default, on unix-like systems, the created directory is world-readable (readable by an attacker with access to the system). The method in question has been marked @Deprecated in versions 30.0 and later and should not be used. For Android developers, we recommend choosing a temporary directory API provided by Android, such as context.getCacheDir(). For other Java developers, we recommend migrating to the Java 7 API java.nio.file.Files.createTempDirectory() which explicitly configures permissions of 700, or configuring the Java runtime's java.io.tmpdir system property to point to a location whose permissions are appropriately configured. > fix CVE-2020-8908 > - > > Key: YARN-10980 > URL: https://issues.apache.org/jira/browse/YARN-10980 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Major > > see [https://www.cvedetails.com/cve/CVE-2020-8908/] > > A temp directory creation vulnerability exists in all versions of Guava, > allowing an attacker with access to the machine to potentially access data in > a temporary directory created by the Guava API > com.google.common.io.Files.createTempDir(). By default, on unix-like systems, > the created directory is world-readable (readable by an attacker with access > to the system). The method in question has been marked @Deprecated in > versions 30.0 and later and should not be used. For Android developers, we > recommend choosing a temporary directory API provided by Android, such as > context.getCacheDir(). For other Java developers, we recommend migrating to > the Java 7 API java.nio.file.Files.createTempDirectory() which explicitly > configures permissions of 700, or configuring the Java runtime's > java.io.tmpdir system property to point to a location whose permissions are > appropriately configured. -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Created] (YARN-10980) fix CVE-2020-8908
lujie created YARN-10980: Summary: fix CVE-2020-8908 Key: YARN-10980 URL: https://issues.apache.org/jira/browse/YARN-10980 Project: Hadoop YARN Issue Type: Bug Reporter: lujie -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Created] (YARN-10976) Resource Leak due to Files.walk
lujie created YARN-10976:
Summary: Resource Leak due to Files.walk
Key: YARN-10976
URL: https://issues.apache.org/jira/browse/YARN-10976
Project: Hadoop YARN
Issue Type: Improvement
Reporter: lujie
Stream creates by File.walk should be closed, like jdk said:
* The returned stream encapsulates one or more \{@link DirectoryStream}s.
* If timely disposal of file system resources is required, the
* {@code try}-with-resources construct should be used to ensure that the
* stream's \{@link Stream#close close} method is invoked after the stream
* operations are completed. Operating on a closed stream will result in an
* {@link java.lang.IllegalStateException}.
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10819) Security Policy of log file is inconsistent
[ https://issues.apache.org/jira/browse/YARN-10819?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-10819: - Description: I have enable security for our cluster, and create three users : yarn, user1, user2. I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job as *user2*, the mode of files are !image-2021-06-11-14-32-38-351.png! But i do not want others, like *user1*, see *user2's* logs, I have fixed on bug about preventing user1 seeing the job infomation of user2 , i.e. https://issues.apache.org/jira/browse/YARN-10555 I think we should have a way to resolve this inconsistent, or at least warn us in configuration. was: I have enable security for our cluster, and create three users : yarn, user1, user2. I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job as *user2*, the mode of files are !image-2021-06-11-14-32-38-351.png! But i do not want others, like *user1*, see *user2's* logs, I have fixed on bug about preventing user1 seeing the job infomation of user2 , i.e. https://issues.apache.org/jira/browse/YARN-10555 I think we should have a way to resolve this inconsistent. > Security Policy of log file is inconsistent > --- > > Key: YARN-10819 > URL: https://issues.apache.org/jira/browse/YARN-10819 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Major > Attachments: image-2021-06-11-14-32-38-351.png > > > I have enable security for our cluster, and create three users : yarn, > user1, user2. > I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job > as *user2*, the mode of files are > !image-2021-06-11-14-32-38-351.png! > But i do not want others, like *user1*, see *user2's* logs, > > I have fixed on bug about preventing user1 seeing the job infomation of > user2 , i.e. https://issues.apache.org/jira/browse/YARN-10555 > > I think we should have a way to resolve this inconsistent, or at least warn > us in configuration. > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10819) Security Policy of log file is inconsistent
[ https://issues.apache.org/jira/browse/YARN-10819?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-10819: - Description: I have enable security for our cluster, and create three users : yarn, user1, user2. I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job as *user2*, the mode of files are !image-2021-06-11-14-32-38-351.png! But i do not want others, like *user1*, see my logs, I have fixed on bug about preventing user1 seeing the infomation of user2 , i.e. https://issues.apache.org/jira/browse/YARN-10555 I think we should have a way to resolve this inconsistent. was: I have enable security for our cluster, and create three users : yarn, user1, user2. I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job as *user2*, the mode of files are !image-2021-06-11-14-32-38-351.png! But i do not want others, like *user1*, see my logs, I have fixed on bug about preventing user1 seeing the infomation of user2. I think we should have we way to resolve this . > Security Policy of log file is inconsistent > --- > > Key: YARN-10819 > URL: https://issues.apache.org/jira/browse/YARN-10819 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Major > Attachments: image-2021-06-11-14-32-38-351.png > > > I have enable security for our cluster, and create three users : yarn, > user1, user2. > I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job > as *user2*, the mode of files are > !image-2021-06-11-14-32-38-351.png! > But i do not want others, like *user1*, see my logs, > > I have fixed on bug about preventing user1 seeing the infomation of user2 , > i.e. https://issues.apache.org/jira/browse/YARN-10555 > > I think we should have a way to resolve this inconsistent. > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10819) Security Policy of log file is inconsistent
[ https://issues.apache.org/jira/browse/YARN-10819?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-10819: - Description: I have enable security for our cluster, and create three users : yarn, user1, user2. I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job as *user2*, the mode of files are !image-2021-06-11-14-32-38-351.png! But i do not want others, like *user1*, see *user2's* logs, I have fixed on bug about preventing user1 seeing the job infomation of user2 , i.e. https://issues.apache.org/jira/browse/YARN-10555 I think we should have a way to resolve this inconsistent. was: I have enable security for our cluster, and create three users : yarn, user1, user2. I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job as *user2*, the mode of files are !image-2021-06-11-14-32-38-351.png! But i do not want others, like *user1*, see my logs, I have fixed on bug about preventing user1 seeing the job infomation of user2 , i.e. https://issues.apache.org/jira/browse/YARN-10555 I think we should have a way to resolve this inconsistent. > Security Policy of log file is inconsistent > --- > > Key: YARN-10819 > URL: https://issues.apache.org/jira/browse/YARN-10819 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Major > Attachments: image-2021-06-11-14-32-38-351.png > > > I have enable security for our cluster, and create three users : yarn, > user1, user2. > I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job > as *user2*, the mode of files are > !image-2021-06-11-14-32-38-351.png! > But i do not want others, like *user1*, see *user2's* logs, > > I have fixed on bug about preventing user1 seeing the job infomation of > user2 , i.e. https://issues.apache.org/jira/browse/YARN-10555 > > I think we should have a way to resolve this inconsistent. > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10819) Security Policy of log file is inconsistent
[ https://issues.apache.org/jira/browse/YARN-10819?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-10819: - Description: I have enable security for our cluster, and create three users : yarn, user1, user2. I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job as *user2*, the mode of files are !image-2021-06-11-14-32-38-351.png! But i do not want others, like *user1*, see my logs, I have fixed on bug about preventing user1 seeing the job infomation of user2 , i.e. https://issues.apache.org/jira/browse/YARN-10555 I think we should have a way to resolve this inconsistent. was: I have enable security for our cluster, and create three users : yarn, user1, user2. I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job as *user2*, the mode of files are !image-2021-06-11-14-32-38-351.png! But i do not want others, like *user1*, see my logs, I have fixed on bug about preventing user1 seeing the infomation of user2 , i.e. https://issues.apache.org/jira/browse/YARN-10555 I think we should have a way to resolve this inconsistent. > Security Policy of log file is inconsistent > --- > > Key: YARN-10819 > URL: https://issues.apache.org/jira/browse/YARN-10819 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Major > Attachments: image-2021-06-11-14-32-38-351.png > > > I have enable security for our cluster, and create three users : yarn, > user1, user2. > I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job > as *user2*, the mode of files are > !image-2021-06-11-14-32-38-351.png! > But i do not want others, like *user1*, see my logs, > > I have fixed on bug about preventing user1 seeing the job infomation of > user2 , i.e. https://issues.apache.org/jira/browse/YARN-10555 > > I think we should have a way to resolve this inconsistent. > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10819) Security Policy of log file is inconsistent
[ https://issues.apache.org/jira/browse/YARN-10819?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-10819: - Description: I have enable security for our cluster, and create three users : yarn, user1, user2. I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job as *user2*, the mode of files are !image-2021-06-11-14-32-38-351.png! But i do not want others, like *user1*, see my logs, I have fixed on bug about preventing user1 seeing the infomation of user2. I think we should have we way to resolve this . was: I have enable security for our cluster, and create three users : yarn, user1, user2. I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job as *user2*, the mode of files are !image-2021-06-11-14-32-38-351.png! But i do not want others, like *user1*, see my logs, I have fixed on bug about preventing user1 seeing the infomation of user2. I think we should have we way to prevent it. > Security Policy of log file is inconsistent > --- > > Key: YARN-10819 > URL: https://issues.apache.org/jira/browse/YARN-10819 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Major > Attachments: image-2021-06-11-14-32-38-351.png > > > I have enable security for our cluster, and create three users : yarn, > user1, user2. > I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job > as *user2*, the mode of files are > !image-2021-06-11-14-32-38-351.png! > But i do not want others, like *user1*, see my logs, > > I have fixed on bug about preventing user1 seeing the infomation of user2. > > I think we should have we way to resolve this . > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10819) Security Policy of log file is inconsistent
[ https://issues.apache.org/jira/browse/YARN-10819?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-10819: - Description: I have enable security for our cluster, and create three users : yarn, user1, user2. I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job as *user2*, the mode of files are !image-2021-06-11-14-32-38-351.png! But i do not want others, like *user1*, see my logs, I have fixed on bug about preventing user1 seeing the infomation of user2. I think we should have we way to prevent it. was: I have enable security for our cluster, and create three users : yarn, user1, user2. I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job as *user2*, the mode of files are !image-2021-06-11-14-32-38-351.png! But i do not want others, like *user1*, see my logs, I have fixed on bug about preventing user1 seeing the infomation of user2. I think > Security Policy of log file is inconsistent > --- > > Key: YARN-10819 > URL: https://issues.apache.org/jira/browse/YARN-10819 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Major > Attachments: image-2021-06-11-14-32-38-351.png > > > I have enable security for our cluster, and create three users : yarn, > user1, user2. > I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job > as *user2*, the mode of files are > !image-2021-06-11-14-32-38-351.png! > But i do not want others, like *user1*, see my logs, > > I have fixed on bug about preventing user1 seeing the infomation of user2. > > I think we should have we way to prevent it. > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10819) Security Policy of log file is inconsistent
[ https://issues.apache.org/jira/browse/YARN-10819?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-10819: - Summary: Security Policy of log file is inconsistent (was: Security inconsistency for log file) > Security Policy of log file is inconsistent > --- > > Key: YARN-10819 > URL: https://issues.apache.org/jira/browse/YARN-10819 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Major > Attachments: image-2021-06-11-14-32-38-351.png > > > I have enable security for our cluster, and create three users : yarn, > user1, user2. > I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job > as *user2*, the mode of files are > !image-2021-06-11-14-32-38-351.png! > But i do not want others, like *user1*, see my logs, > > I have fixed on bug about preventing user1 seeing the infomation of user2. > > I think > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10819) Security inconsistency for log file
[ https://issues.apache.org/jira/browse/YARN-10819?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-10819: - Description: I have enable security for our cluster, and create three users : yarn, user1, user2. I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job as *user2*, the mode of files are !image-2021-06-11-14-32-38-351.png! But i do not want others, like *user1*, see my logs, I have fixed on bug about preventing user1 seeing the infomation of user2. I think > Security inconsistency for log file > --- > > Key: YARN-10819 > URL: https://issues.apache.org/jira/browse/YARN-10819 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Major > Attachments: image-2021-06-11-14-32-38-351.png > > > I have enable security for our cluster, and create three users : yarn, > user1, user2. > I configured the mode of "yarn.nodemanager.log-dirs" as 750 and submit a job > as *user2*, the mode of files are > !image-2021-06-11-14-32-38-351.png! > But i do not want others, like *user1*, see my logs, > > I have fixed on bug about preventing user1 seeing the infomation of user2. > > I think > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10819) Security inconsistency for log file
[ https://issues.apache.org/jira/browse/YARN-10819?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-10819: - Attachment: image-2021-06-11-14-32-38-351.png > Security inconsistency for log file > --- > > Key: YARN-10819 > URL: https://issues.apache.org/jira/browse/YARN-10819 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Major > Attachments: image-2021-06-11-14-32-38-351.png > > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Created] (YARN-10819) Security inconsistency for log file
lujie created YARN-10819: Summary: Security inconsistency for log file Key: YARN-10819 URL: https://issues.apache.org/jira/browse/YARN-10819 Project: Hadoop YARN Issue Type: Bug Reporter: lujie -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Comment Edited] (YARN-10555) missing access check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17264001#comment-17264001
]
lujie edited comment on YARN-10555 at 4/23/21, 7:43 PM:
again ping->
was (Author: xiaoheipangzi):
ping->
> missing access check before getAppAttempts
> ---
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Labels: pull-request-available, security
> Attachments: YARN-10555_1.patch
>
> Time Spent: 2h
> Remaining Estimate: 0h
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> application_1609318368700_0002 belong to user2
> user1@hadoop11$ curl --negotiate -u :
> http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
> {
> "appAttempts": {
> "appAttempt": [
> {
> "id": 1,
> "startTime": 1609318411566,
> "containerId": "container_1609318368700_0002_01_01",
> "nodeHttpAddress": "hadoop12:8044",
> "nodeId": "hadoop12:36831",
> "logsLink":
> "http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
> "blacklistedNodes": "",
> "nodesBlacklistedBySystem": ""
> }
> ]
> }
> }
> {code}
> Other apis, like getApps and getApp, has access check like "hasAccess(app,
> hsr)", they would hide the logs link if the appid do not belong to query
> user, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
> We need add hasAccess(app, hsr) for getAppAttempts.
>
> Besides, at
> [https://github.com/apache/hadoop/blob/580a6a75a3e3d3b7918edeffd6e93fc211166884/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java#L145]
> it seems that we have a access check in its caller, so now i pass "true" to
> AppAttemptInfo in the patch.
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-10555) missing access check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17273463#comment-17273463
]
lujie commented on YARN-10555:
--
Shoud we merge this pull request if there is no more questions?
> missing access check before getAppAttempts
> ---
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Labels: pull-request-available, security
> Attachments: YARN-10555_1.patch
>
> Time Spent: 1h 50m
> Remaining Estimate: 0h
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> application_1609318368700_0002 belong to user2
> user1@hadoop11$ curl --negotiate -u :
> http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
> {
> "appAttempts": {
> "appAttempt": [
> {
> "id": 1,
> "startTime": 1609318411566,
> "containerId": "container_1609318368700_0002_01_01",
> "nodeHttpAddress": "hadoop12:8044",
> "nodeId": "hadoop12:36831",
> "logsLink":
> "http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
> "blacklistedNodes": "",
> "nodesBlacklistedBySystem": ""
> }
> ]
> }
> }
> {code}
> Other apis, like getApps and getApp, has access check like "hasAccess(app,
> hsr)", they would hide the logs link if the appid do not belong to query
> user, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
> We need add hasAccess(app, hsr) for getAppAttempts.
>
> Besides, at
> [https://github.com/apache/hadoop/blob/580a6a75a3e3d3b7918edeffd6e93fc211166884/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java#L145]
> it seems that we have a access check in its caller, so now i pass "true" to
> AppAttemptInfo in the patch.
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-10555) missing access check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17264001#comment-17264001
]
lujie commented on YARN-10555:
--
ping->
> missing access check before getAppAttempts
> ---
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Labels: pull-request-available, security
> Attachments: YARN-10555_1.patch
>
> Time Spent: 40m
> Remaining Estimate: 0h
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> application_1609318368700_0002 belong to user2
> user1@hadoop11$ curl --negotiate -u :
> http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
> {
> "appAttempts": {
> "appAttempt": [
> {
> "id": 1,
> "startTime": 1609318411566,
> "containerId": "container_1609318368700_0002_01_01",
> "nodeHttpAddress": "hadoop12:8044",
> "nodeId": "hadoop12:36831",
> "logsLink":
> "http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
> "blacklistedNodes": "",
> "nodesBlacklistedBySystem": ""
> }
> ]
> }
> }
> {code}
> Other apis, like getApps and getApp, has access check like "hasAccess(app,
> hsr)", they would hide the logs link if the appid do not belong to query
> user, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
> We need add hasAccess(app, hsr) for getAppAttempts.
>
> Besides, at
> [https://github.com/apache/hadoop/blob/580a6a75a3e3d3b7918edeffd6e93fc211166884/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java#L145]
> it seems that we have a access check in its caller, so now i pass "true" to
> AppAttemptInfo in the patch.
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10555) missing access check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Description:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Other apis, like getApps and getApp, has access check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
Besides, at
[https://github.com/apache/hadoop/blob/580a6a75a3e3d3b7918edeffd6e93fc211166884/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java#L145]
it seems that we have a access check in its caller, so now i pass "true" to
AppAttemptInfo in the patch.
was:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Other apis, like getApps and getApp, has access check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
Besides, at
[https://github.com/apache/hadoop/blob/580a6a75a3e3d3b7918edeffd6e93fc211166884/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java#L145]
it seems that we have a access check in its caller, so now i pass "true" to
AppAttemptInfo.
> missing access check before getAppAttempts
> ---
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Labels: pull-request-available, security
> Attachments: YARN-10555_1.patch
>
> Time Spent: 10m
> Remaining Estimate: 0h
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> application_1609318368700_0002 belong to user2
> user1@h
[jira] [Updated] (YARN-10555) missing access check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Description:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Other apis, like getApps and getApp, has access check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
Besides, at
[https://github.com/apache/hadoop/blob/580a6a75a3e3d3b7918edeffd6e93fc211166884/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java#L145]
it seems that we have a access check in its caller, so now i pass "true" to
AppAttemptInfo.
was:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Other apis, like getApps and getApp, has access check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
Besides, at
[https://github.com/apache/hadoop/blob/580a6a75a3e3d3b7918edeffd6e93fc211166884/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java#L145]
it seems that we have a access check, so now i pass "true" to AppAttemptInfo.
> missing access check before getAppAttempts
> ---
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Labels: pull-request-available, security
> Attachments: YARN-10555_1.patch
>
> Time Spent: 10m
> Remaining Estimate: 0h
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> application_1609318368700_0002 belong to user2
> user1@hadoop11$ curl --negotiate -
[jira] [Updated] (YARN-10555) missing access check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Description:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Other apis, like getApps and getApp, has access check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
Besides, at
[https://github.com/apache/hadoop/blob/580a6a75a3e3d3b7918edeffd6e93fc211166884/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java#L145]
it seems that we have a access check, so now i pass "true" to AppAttemptInfo.
was:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Other apis, like getApps and getApp, has access check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
Besides, at
[https://github.com/apache/hadoop/blob/580a6a75a3e3d3b7918edeffd6e93fc211166884/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java#L145]
it seems that we also miss a access check, but i do not ttigger it, so now i
pass "true" to AppAttemptInfo.
> missing access check before getAppAttempts
> ---
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Labels: pull-request-available, security
> Attachments: YARN-10555_1.patch
>
> Time Spent: 10m
> Remaining Estimate: 0h
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> application_1609318368700_0002 belong to user2
> user1@hadoop11$ curl
[jira] [Updated] (YARN-10555) missing access check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Description:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Other apis, like getApps and getApp, has access check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
Besides, at
[https://github.com/apache/hadoop/blob/580a6a75a3e3d3b7918edeffd6e93fc211166884/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java#L145]
it seems that we also miss a access check, but i do not ttigger it, so now i
pass "true" to AppAttemptInfo.
was:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Other apis, like getApps and getApp, has access check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
Besides, at
[https://github.com/apache/hadoop/blob/580a6a75a3e3d3b7918edeffd6e93fc211166884/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java#L145]
it seems that we also miss a access check, but i do not ttigger it, so now i
pass "true" to AppAttemptInfo.
> missing access check before getAppAttempts
> ---
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Labels: pull-request-available, security
> Attachments: YARN-10555_1.patch
>
> Time Spent: 10m
> Remaining Estimate: 0h
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> application_1609318368700_0002 belong to u
[jira] [Updated] (YARN-10555) missing access check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Description:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Other apis, like getApps and getApp, has access check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
Besides, at
[https://github.com/apache/hadoop/blob/580a6a75a3e3d3b7918edeffd6e93fc211166884/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java#L145]
it seems that we also miss a access check, but i do not ttigger it, so now i
pass "true" to AppAttemptInfo.
was:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Other apis, like getApps and getApp, has access check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
Besides, at
[https://github.com/apache/hadoop/blob/580a6a75a3e3d3b7918edeffd6e93fc211166884/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java#L145]
it seems that we also miss a access check, but i do not ttigger it, so now i
pass "true" to AppAttemptInfo
> missing access check before getAppAttempts
> ---
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Labels: pull-request-available, security
> Attachments: YARN-10555_1.patch
>
> Time Spent: 10m
> Remaining Estimate: 0h
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> application_1609318368700_0002 belong to user
[jira] [Updated] (YARN-10555) missing access check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Description:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Other apis, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
Besides, at
[https://github.com/apache/hadoop/blob/580a6a75a3e3d3b7918edeffd6e93fc211166884/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java#L145]
it seems that we also miss a access check, but i do not ttigger it, so now i
pass "true" to AppAttemptInfo
was:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
Besides, at
[https://github.com/apache/hadoop/blob/580a6a75a3e3d3b7918edeffd6e93fc211166884/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java#L145]
it seems that we also miss a access check, but i do not ttigger it, so now i
pass "true" to AppAttemptInfo
> missing access check before getAppAttempts
> ---
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Labels: pull-request-available, security
> Attachments: YARN-10555_1.patch
>
> Time Spent: 10m
> Remaining Estimate: 0h
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> application_1609318368700_0002 belong to us
[jira] [Updated] (YARN-10555) missing access check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Description:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Other apis, like getApps and getApp, has access check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
Besides, at
[https://github.com/apache/hadoop/blob/580a6a75a3e3d3b7918edeffd6e93fc211166884/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java#L145]
it seems that we also miss a access check, but i do not ttigger it, so now i
pass "true" to AppAttemptInfo
was:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Other apis, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
Besides, at
[https://github.com/apache/hadoop/blob/580a6a75a3e3d3b7918edeffd6e93fc211166884/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java#L145]
it seems that we also miss a access check, but i do not ttigger it, so now i
pass "true" to AppAttemptInfo
> missing access check before getAppAttempts
> ---
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Labels: pull-request-available, security
> Attachments: YARN-10555_1.patch
>
> Time Spent: 10m
> Remaining Estimate: 0h
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> application_1609318368700_0002 belong to user
[jira] [Updated] (YARN-10555) missing access check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Description:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
Besides, at
[https://github.com/apache/hadoop/blob/580a6a75a3e3d3b7918edeffd6e93fc211166884/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java#L145]
it seems that we also miss a access check, but i do not ttigger it, so now i
pass "true" to AppAttemptInfo
was:
It seems that we miss a access check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus anyone can get some sensitive information that she/he should not, like
logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
> missing access check before getAppAttempts
> ---
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Labels: pull-request-available, security
> Attachments: YARN-10555_1.patch
>
> Time Spent: 10m
> Remaining Estimate: 0h
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> application_1609318368700_0002 belong to user2
> user1@hadoop11$ curl --negotiate -u :
> http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
> {
> "appAttempts": {
> "appAttempt": [
> {
> "id": 1,
> "startTime": 1609318411566,
> "containerId": "container_1609318368700_0002_01_01",
> "nodeHttpAddress": "hadoop12:8044
[jira] [Updated] (YARN-10555) missing access check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Description:
It seems that we miss a access check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus anyone can get some sensitive information that she/he should not, like
logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
was:
It seems that we miss a access check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
> missing access check before getAppAttempts
> ---
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Labels: pull-request-available, security
> Attachments: YARN-10555_1.patch
>
> Time Spent: 10m
> Remaining Estimate: 0h
>
> It seems that we miss a access check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus anyone can get some sensitive information that she/he should not, like
> logs link.
> {code:java}
> application_1609318368700_0002 belong to user2
> user1@hadoop11$ curl --negotiate -u :
> http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
> {
> "appAttempts": {
> "appAttempt": [
> {
> "id": 1,
> "startTime": 1609318411566,
> "containerId": "container_1609318368700_0002_01_01",
> "nodeHttpAddress": "hadoop12:8044",
> "nodeId": "hadoop12:36831",
> "logsLink":
> "http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
> "blacklistedNodes": "",
> "nodesBlacklistedBySystem": ""
> }
> ]
> }
> }
> {code}
> Others api, like getApps and getApp, has security check like "hasAccess(app,
> hsr)", th
[jira] [Updated] (YARN-10555) missing access check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Description:
It seems that we miss a access check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
was:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
> missing access check before getAppAttempts
> ---
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Labels: pull-request-available, security
> Attachments: YARN-10555_1.patch
>
> Time Spent: 10m
> Remaining Estimate: 0h
>
> It seems that we miss a access check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> application_1609318368700_0002 belong to user2
> user1@hadoop11$ curl --negotiate -u :
> http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
> {
> "appAttempts": {
> "appAttempt": [
> {
> "id": 1,
> "startTime": 1609318411566,
> "containerId": "container_1609318368700_0002_01_01",
> "nodeHttpAddress": "hadoop12:8044",
> "nodeId": "hadoop12:36831",
> "logsLink":
> "http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
> "blacklistedNodes": "",
> "nodesBlacklistedBySystem": ""
> }
> ]
> }
> }
> {code}
> Others api, like getApps and getApp, has security check like "hasAccess(app,
> hsr)", they would hide the logs link if the appid do not
[jira] [Updated] (YARN-10555) missing access check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Summary: missing access check before getAppAttempts (was: missing
security check before getAppAttempts)
> missing access check before getAppAttempts
> ---
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Labels: security
> Attachments: YARN-10555_1.patch
>
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> application_1609318368700_0002 belong to user2
> user1@hadoop11$ curl --negotiate -u :
> http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
> {
> "appAttempts": {
> "appAttempt": [
> {
> "id": 1,
> "startTime": 1609318411566,
> "containerId": "container_1609318368700_0002_01_01",
> "nodeHttpAddress": "hadoop12:8044",
> "nodeId": "hadoop12:36831",
> "logsLink":
> "http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
> "blacklistedNodes": "",
> "nodesBlacklistedBySystem": ""
> }
> ]
> }
> }
> {code}
> Others api, like getApps and getApp, has security check like "hasAccess(app,
> hsr)", they would hide the logs link if the appid do not belong to query
> user, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
> We need add hasAccess(app, hsr) for getAppAttempts.
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10555) missing security check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Description:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to query user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
was:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to one user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
> missing security check before getAppAttempts
> -
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Labels: security
> Attachments: YARN-10555_1.patch
>
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> application_1609318368700_0002 belong to user2
> user1@hadoop11$ curl --negotiate -u :
> http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
> {
> "appAttempts": {
> "appAttempt": [
> {
> "id": 1,
> "startTime": 1609318411566,
> "containerId": "container_1609318368700_0002_01_01",
> "nodeHttpAddress": "hadoop12:8044",
> "nodeId": "hadoop12:36831",
> "logsLink":
> "http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
> "blacklistedNodes": "",
> "nodesBlacklistedBySystem": ""
> }
> ]
> }
> }
> {code}
> Others api, like getApps and getApp, has security check like "hasAccess(app,
> hsr)", they would hide the logs link if the appid do not belong to query
> user, see
> [https://github.com/apache/hadoop/blob/
[jira] [Assigned] (YARN-10555) missing security check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie reassigned YARN-10555:
Assignee: lujie
> missing security check before getAppAttempts
> -
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Labels: security
> Attachments: YARN-10555_1.patch
>
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> application_1609318368700_0002 belong to user2
> user1@hadoop11$ curl --negotiate -u :
> http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
> {
> "appAttempts": {
> "appAttempt": [
> {
> "id": 1,
> "startTime": 1609318411566,
> "containerId": "container_1609318368700_0002_01_01",
> "nodeHttpAddress": "hadoop12:8044",
> "nodeId": "hadoop12:36831",
> "logsLink":
> "http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
> "blacklistedNodes": "",
> "nodesBlacklistedBySystem": ""
> }
> ]
> }
> }
> {code}
> Others api, like getApps and getApp, has security check like "hasAccess(app,
> hsr)", they would hide the logs link if the appid do not belong to one user,
> see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
> We need add hasAccess(app, hsr) for getAppAttempts.
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10555) missing security check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Description:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would hide the logs link if the appid do not belong to one user,
see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
was:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would not leak the logs link, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
> missing security check before getAppAttempts
> -
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Priority: Critical
> Labels: security
> Attachments: YARN-10555_1.patch
>
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> application_1609318368700_0002 belong to user2
> user1@hadoop11$ curl --negotiate -u :
> http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
> {
> "appAttempts": {
> "appAttempt": [
> {
> "id": 1,
> "startTime": 1609318411566,
> "containerId": "container_1609318368700_0002_01_01",
> "nodeHttpAddress": "hadoop12:8044",
> "nodeId": "hadoop12:36831",
> "logsLink":
> "http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
> "blacklistedNodes": "",
> "nodesBlacklistedBySystem": ""
> }
> ]
> }
> }
> {code}
> Others api, like getApps and getApp, has security check like "hasAccess(app,
> hsr)", they would hide the logs link if the appid do not belong to one user,
> see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-y
[jira] [Comment Edited] (YARN-10555) missing security check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17256467#comment-17256467
]
lujie edited comment on YARN-10555 at 12/30/20, 12:00 PM:
--
output after patch
{code:java}
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609326143645,
"containerId": "",
"nodeHttpAddress": "",
"nodeId": "",
"logsLink": "",
"blacklistedNodes": ""
}
]
}
}
{code}
was (Author: xiaoheipangzi):
after patched, output can be like:
{
"appAttempts": {
"appAttempt": [
{ "id": 1, "startTime": 1609326143645, "containerId": "", "nodeHttpAddress":
"", "nodeId": "", "logsLink": "", "blacklistedNodes": "" }
]
}
}
> missing security check before getAppAttempts
> -
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Priority: Critical
> Labels: security
> Attachments: YARN-10555_1.patch
>
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> application_1609318368700_0002 belong to user2
> user1@hadoop11$ curl --negotiate -u :
> http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
> {
> "appAttempts": {
> "appAttempt": [
> {
> "id": 1,
> "startTime": 1609318411566,
> "containerId": "container_1609318368700_0002_01_01",
> "nodeHttpAddress": "hadoop12:8044",
> "nodeId": "hadoop12:36831",
> "logsLink":
> "http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
> "blacklistedNodes": "",
> "nodesBlacklistedBySystem": ""
> }
> ]
> }
> }
> {code}
> Others api, like getApps and getApp, has security check like "hasAccess(app,
> hsr)", they would not leak the logs link, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
> We need add hasAccess(app, hsr) for getAppAttempts.
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10555) missing security check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Description:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
# application_1609318368700_0002 belong to user2
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would not leak the logs link, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
was:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would not leak the logs link, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
> missing security check before getAppAttempts
> -
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Priority: Critical
> Labels: security
> Attachments: YARN-10555_1.patch
>
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> user1@hadoop11$ curl --negotiate -u :
> http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
> # application_1609318368700_0002 belong to user2
> {
> "appAttempts": {
> "appAttempt": [
> {
> "id": 1,
> "startTime": 1609318411566,
> "containerId": "container_1609318368700_0002_01_01",
> "nodeHttpAddress": "hadoop12:8044",
> "nodeId": "hadoop12:36831",
> "logsLink":
> "http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
> "blacklistedNodes": "",
> "nodesBlacklistedBySystem": ""
> }
> ]
> }
> }
> {code}
> Others api, like getApps and getApp, has security check like "hasAccess(app,
> hsr)", they would not leak the logs link, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/
[jira] [Updated] (YARN-10555) missing security check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Description:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
application_1609318368700_0002 belong to user2
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would not leak the logs link, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
was:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
# application_1609318368700_0002 belong to user2
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would not leak the logs link, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
> missing security check before getAppAttempts
> -
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Priority: Critical
> Labels: security
> Attachments: YARN-10555_1.patch
>
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> application_1609318368700_0002 belong to user2
> user1@hadoop11$ curl --negotiate -u :
> http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
> {
> "appAttempts": {
> "appAttempt": [
> {
> "id": 1,
> "startTime": 1609318411566,
> "containerId": "container_1609318368700_0002_01_01",
> "nodeHttpAddress": "hadoop12:8044",
> "nodeId": "hadoop12:36831",
> "logsLink":
> "http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
> "blacklistedNodes": "",
> "nodesBlacklistedBySystem": ""
> }
> ]
> }
> }
> {code}
> Others api, like getApps and getApp, has security check like "hasAccess(app,
> hsr)", they would not leak the logs link, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/o
[jira] [Updated] (YARN-10555) missing security check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Description:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
user1@hadoop11$ curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would not leak the logs link, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
was:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would not leak the logs link, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
> missing security check before getAppAttempts
> -
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Priority: Critical
> Labels: security
> Attachments: YARN-10555_1.patch
>
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> user1@hadoop11$ curl --negotiate -u :
> http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
> {
> "appAttempts": {
> "appAttempt": [
> {
> "id": 1,
> "startTime": 1609318411566,
> "containerId": "container_1609318368700_0002_01_01",
> "nodeHttpAddress": "hadoop12:8044",
> "nodeId": "hadoop12:36831",
> "logsLink":
> "http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
> "blacklistedNodes": "",
> "nodesBlacklistedBySystem": ""
> }
> ]
> }
> }
> {code}
> Others api, like getApps and getApp, has security check like "hasAccess(app,
> hsr)", they would not leak the logs link, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
> We need add hasAccess(app, hsr) for getAppAttempts.
>
--
This message was
[jira] [Updated] (YARN-10555) missing security check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Description:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would not leak the logs link, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
was:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would not leak the logs link, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
> missing security check before getAppAttempts
> -
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Priority: Critical
> Labels: security
> Attachments: YARN-10555_1.patch
>
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> curl --negotiate -u :
> http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
> {
> "appAttempts": {
> "appAttempt": [
> {
> "id": 1,
> "startTime": 1609318411566,
> "containerId": "container_1609318368700_0002_01_01",
> "nodeHttpAddress": "hadoop12:8044",
> "nodeId": "hadoop12:36831",
> "logsLink":
> "http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
> "blacklistedNodes": "",
> "nodesBlacklistedBySystem": ""
> }
> ]
> }
> }
> {code}
> Others api, like getApps and getApp, has security check like "hasAccess(app,
> hsr)", they would not leak the logs link, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
> We need add hasAccess(app, hsr) for getAppAttempts.
>
--
This message was sent by Atlassian Jira
(v8.3.4
[jira] [Updated] (YARN-10555) missing security check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Description:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would not leak the logs link, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
was:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would not leak the logs link, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
> missing security check before getAppAttempts
> -
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Priority: Critical
> Labels: security
> Attachments: YARN-10555_1.patch
>
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> curl --negotiate -u :
> http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
> {
> "appAttempts": {
> "appAttempt": [
> {
> "id": 1,
> "startTime": 1609318411566,
> "containerId": "container_1609318368700_0002_01_01",
> "nodeHttpAddress": "hadoop12:8044",
> "nodeId": "hadoop12:36831",
> "logsLink":
> "http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
> "blacklistedNodes": "",
> "nodesBlacklistedBySystem": ""
> }
> ]
> }
> }
> {code}
> Others api, like getApps and getApp, has security check like "hasAccess(app,
> hsr)", they would not leak the logs link, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
>
> We need add hasAccess(app, hsr) for getAppAttempts.
>
--
This message was sent by Atlassian Jira
(v8
[jira] [Updated] (YARN-10555) missing security check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Description:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
curl --negotiate -u :
http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would not leak the logs link, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
was:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would not leak the logs link, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
> missing security check before getAppAttempts
> -
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Priority: Critical
> Labels: security
> Attachments: YARN-10555_1.patch
>
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> curl --negotiate -u :
> http://hadoop11:8088/ws/v1/cluster/apps/application_1609318368700_0002/appattempts/|jq
> {
> "appAttempts": {
> "appAttempt": [
> {
> "id": 1,
> "startTime": 1609318411566,
> "containerId": "container_1609318368700_0002_01_01",
> "nodeHttpAddress": "hadoop12:8044",
> "nodeId": "hadoop12:36831",
> "logsLink":
> "http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
> "blacklistedNodes": "",
> "nodesBlacklistedBySystem": ""
> }
> ]
> }
> }
> {code}
> Others api, like getApps and getApp, has security check like "hasAccess(app,
> hsr)", they would not leak the logs link, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
>
> We need add hasAccess(app, hsr) for getAppAttempts.
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: yarn
[jira] [Updated] (YARN-10555) missing security check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10555:
-
Description:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
{code:java}
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609318411566,
"containerId": "container_1609318368700_0002_01_01",
"nodeHttpAddress": "hadoop12:8044",
"nodeId": "hadoop12:36831",
"logsLink":
"http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
"blacklistedNodes": "",
"nodesBlacklistedBySystem": ""
}
]
}
}
{code}
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would not leak the logs link, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
was:
It seems that we miss a security check before getAppAttempts, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
thus we can get the some sensitive information, like logs link.
Others api, like getApps and getApp, has security check like "hasAccess(app,
hsr)", they would not leak the logs link, see
[https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
We need add hasAccess(app, hsr) for getAppAttempts.
> missing security check before getAppAttempts
> -
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
> Components: webapp
>Reporter: lujie
>Priority: Critical
> Labels: security
> Attachments: YARN-10555_1.patch
>
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> {code:java}
> {
> "appAttempts": {
> "appAttempt": [
> {
> "id": 1,
> "startTime": 1609318411566,
> "containerId": "container_1609318368700_0002_01_01",
> "nodeHttpAddress": "hadoop12:8044",
> "nodeId": "hadoop12:36831",
> "logsLink":
> "http://hadoop12:8044/node/containerlogs/container_1609318368700_0002_01_01/user2";,
> "blacklistedNodes": "",
> "nodesBlacklistedBySystem": ""
> }
> ]
> }
> }
> {code}
> Others api, like getApps and getApp, has security check like "hasAccess(app,
> hsr)", they would not leak the logs link, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
>
> We need add hasAccess(app, hsr) for getAppAttempts.
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10555) missing security check before getAppAttempts
[ https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-10555: - Component/s: webapp > missing security check before getAppAttempts > - > > Key: YARN-10555 > URL: https://issues.apache.org/jira/browse/YARN-10555 > Project: Hadoop YARN > Issue Type: Bug > Components: webapp >Reporter: lujie >Priority: Critical > Labels: security > Attachments: YARN-10555_1.patch > > > It seems that we miss a security check before getAppAttempts, see > [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127] > thus we can get the some sensitive information, like logs link. > Others api, like getApps and getApp, has security check like "hasAccess(app, > hsr)", they would not leak the logs link, see > [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098] > > We need add hasAccess(app, hsr) for getAppAttempts. > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10555) missing security check before getAppAttempts
[ https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-10555: - Labels: security (was: ) > missing security check before getAppAttempts > - > > Key: YARN-10555 > URL: https://issues.apache.org/jira/browse/YARN-10555 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Critical > Labels: security > Attachments: YARN-10555_1.patch > > > It seems that we miss a security check before getAppAttempts, see > [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127] > thus we can get the some sensitive information, like logs link. > Others api, like getApps and getApp, has security check like "hasAccess(app, > hsr)", they would not leak the logs link, see > [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098] > > We need add hasAccess(app, hsr) for getAppAttempts. > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10555) missing security check before getAppAttempts
[ https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-10555: - Priority: Critical (was: Major) > missing security check before getAppAttempts > - > > Key: YARN-10555 > URL: https://issues.apache.org/jira/browse/YARN-10555 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Critical > Attachments: YARN-10555_1.patch > > > It seems that we miss a security check before getAppAttempts, see > [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127] > thus we can get the some sensitive information, like logs link. > Others api, like getApps and getApp, has security check like "hasAccess(app, > hsr)", they would not leak the logs link, see > [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098] > > We need add hasAccess(app, hsr) for getAppAttempts. > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Comment Edited] (YARN-10555) missing security check before getAppAttempts
[
https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17256467#comment-17256467
]
lujie edited comment on YARN-10555 at 12/30/20, 11:23 AM:
--
after patched, output can be like:
{
"appAttempts": {
"appAttempt": [
{ "id": 1, "startTime": 1609326143645, "containerId": "", "nodeHttpAddress":
"", "nodeId": "", "logsLink": "", "blacklistedNodes": "" }
]
}
}
was (Author: xiaoheipangzi):
output can be like:
{
"appAttempts": {
"appAttempt": [
{
"id": 1,
"startTime": 1609326143645,
"containerId": "",
"nodeHttpAddress": "",
"nodeId": "",
"logsLink": "",
"blacklistedNodes": ""
}
]
}
}
> missing security check before getAppAttempts
> -
>
> Key: YARN-10555
> URL: https://issues.apache.org/jira/browse/YARN-10555
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Major
> Attachments: YARN-10555_1.patch
>
>
> It seems that we miss a security check before getAppAttempts, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127]
> thus we can get the some sensitive information, like logs link.
> Others api, like getApps and getApp, has security check like "hasAccess(app,
> hsr)", they would not leak the logs link, see
> [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098]
>
> We need add hasAccess(app, hsr) for getAppAttempts.
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10555) missing security check before getAppAttempts
[ https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-10555: - Attachment: (was: YARN-10555_1.patch) > missing security check before getAppAttempts > - > > Key: YARN-10555 > URL: https://issues.apache.org/jira/browse/YARN-10555 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Major > > It seems that we miss a security check before getAppAttempts, see > [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127] > thus we can get the some sensitive information, like logs link. > Others api, like getApps and getApp, has security check like "hasAccess(app, > hsr)", they would not leak the logs link, see > [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098] > > We need add hasAccess(app, hsr) for getAppAttempts. > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10555) missing security check before getAppAttempts
[ https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-10555: - Attachment: YARN-10555_1.patch > missing security check before getAppAttempts > - > > Key: YARN-10555 > URL: https://issues.apache.org/jira/browse/YARN-10555 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Major > Attachments: YARN-10555_1.patch > > > It seems that we miss a security check before getAppAttempts, see > [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127] > thus we can get the some sensitive information, like logs link. > Others api, like getApps and getApp, has security check like "hasAccess(app, > hsr)", they would not leak the logs link, see > [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098] > > We need add hasAccess(app, hsr) for getAppAttempts. > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10555) missing security check before getAppAttempts
[ https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-10555: - Description: It seems that we miss a security check before getAppAttempts, see [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127] thus we can get the some sensitive information, like logs link. Others api, like getApps and getApp, has security check like "hasAccess(app, hsr)", they would not leak the logs link, see [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098] We need add hasAccess(app, hsr) for getAppAttempts. was: It seems that we miss a security check before getAppAttempts, see [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127] thus we can get the some sensitive information, like logs link. Others api, like getApps and getApp, has security check like "hasAccess(app, hsr)", they would not leak the logs link, see [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098] We need add hasAccess(app, hsr) for getAppAttempts. @[~ayushtkn] > missing security check before getAppAttempts > - > > Key: YARN-10555 > URL: https://issues.apache.org/jira/browse/YARN-10555 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Major > > It seems that we miss a security check before getAppAttempts, see > [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127] > thus we can get the some sensitive information, like logs link. > Others api, like getApps and getApp, has security check like "hasAccess(app, > hsr)", they would not leak the logs link, see > [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098] > > We need add hasAccess(app, hsr) for getAppAttempts. > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10555) missing security check before getAppAttempts
[ https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-10555: - Description: It seems that we miss a security check before getAppAttempts, see [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127] thus we can get the some sensitive information, like logs link. Others api, like getApps and getApp, has security check like "hasAccess(app, hsr)", they would not leak the logs link, see [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098] We need add hasAccess(app, hsr) for getAppAttempts. @[~ayushtkn] was: It seems that we miss a security check before getAppAttempts, see [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127] thus we can get the some sensitive information, like logs link. Others api, like getApps and getApp, has security check like "hasAccess(app, hsr)", they would not leak the logs link, see https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098 We need add hasAccess(app, hsr) for getAppAttempts.@ > missing security check before getAppAttempts > - > > Key: YARN-10555 > URL: https://issues.apache.org/jira/browse/YARN-10555 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Major > > It seems that we miss a security check before getAppAttempts, see > [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127] > thus we can get the some sensitive information, like logs link. > Others api, like getApps and getApp, has security check like "hasAccess(app, > hsr)", they would not leak the logs link, see > [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098] > > We need add hasAccess(app, hsr) for getAppAttempts. @[~ayushtkn] > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10555) missing security check before getAppAttempts
[ https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-10555: - Description: It seems that we miss a security check before getAppAttempts, see [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127] thus we can get the some sensitive information, like logs link. Others api, like getApps and getApp, has security check like "hasAccess(app, hsr)", they would not leak the logs link, see https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098 We need add hasAccess(app, hsr) for getAppAttempts.@ was:It seems that we miss a > missing security check before getAppAttempts > - > > Key: YARN-10555 > URL: https://issues.apache.org/jira/browse/YARN-10555 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Major > > It seems that we miss a security check before getAppAttempts, see > [https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1127] > thus we can get the some sensitive information, like logs link. > Others api, like getApps and getApp, has security check like "hasAccess(app, > hsr)", they would not leak the logs link, see > https://github.com/apache/hadoop/blob/513f1995adc9b73f9c7f4c7beb89725b51b313ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java#L1098 > > We need add hasAccess(app, hsr) for getAppAttempts.@ > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10555) missing security check before getAppAttempts
[ https://issues.apache.org/jira/browse/YARN-10555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] lujie updated YARN-10555: - Description: It seems that we miss a > missing security check before getAppAttempts > - > > Key: YARN-10555 > URL: https://issues.apache.org/jira/browse/YARN-10555 > Project: Hadoop YARN > Issue Type: Bug >Reporter: lujie >Priority: Major > > It seems that we miss a -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Created] (YARN-10555) missing security check before getAppAttempts
lujie created YARN-10555: Summary: missing security check before getAppAttempts Key: YARN-10555 URL: https://issues.apache.org/jira/browse/YARN-10555 Project: Hadoop YARN Issue Type: Bug Reporter: lujie -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Resolved] (YARN-10551) non-admin user can change the log level
[
https://issues.apache.org/jira/browse/YARN-10551?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie resolved YARN-10551.
--
Resolution: Not A Problem
misconfiguration! see
https://issues.apache.org/jira/secure/attachment/12832635/HADOOP-13707.001.patch
> non-admin user can change the log level
> ---
>
> Key: YARN-10551
> URL: https://issues.apache.org/jira/browse/YARN-10551
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Major
>
> reproduce:
> 1. login as user1 and do
> {code:java}
> yarn daemonlog -setlevel hadoop11:8088
> org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl DEBUG
> {code}
> 2. login as user2 and run wordcount
> 3. check the log of RM
> {code:java}
> 2020-12-27 10:54:15,917 DEBUG
> org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl: Processing
> event for application_1609065586411_0003 of type START
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-10551) non-admin user can change the log level
[
https://issues.apache.org/jira/browse/YARN-10551?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-10551:
-
Summary: non-admin user can change the log level (was: non-admin user can
chane the log level)
> non-admin user can change the log level
> ---
>
> Key: YARN-10551
> URL: https://issues.apache.org/jira/browse/YARN-10551
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Major
>
> reproduce:
> 1. login as user1 and do
> {code:java}
> yarn daemonlog -setlevel hadoop11:8088
> org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl DEBUG
> {code}
> 2. login as user2 and run wordcount
> 3. check the log of RM
> {code:java}
> 2020-12-27 10:54:15,917 DEBUG
> org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl: Processing
> event for application_1609065586411_0003 of type START
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Created] (YARN-10551) non-admin user can chane the log level
lujie created YARN-10551:
Summary: non-admin user can chane the log level
Key: YARN-10551
URL: https://issues.apache.org/jira/browse/YARN-10551
Project: Hadoop YARN
Issue Type: Bug
Reporter: lujie
reproduce:
1. login as user1 and do
{code:java}
yarn daemonlog -setlevel hadoop11:8088
org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl DEBUG
{code}
2. login as user2 and run wordcount
3. check the log of RM
{code:java}
2020-12-27 10:54:15,917 DEBUG
org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl: Processing event
for application_1609065586411_0003 of type START
{code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-9594) Unknown event arrived at ContainerScheduler: EventType: RECOVERY_COMPLETED
[
https://issues.apache.org/jira/browse/YARN-9594?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16860583#comment-16860583
]
lujie commented on YARN-9594:
-
ping->
> Unknown event arrived at ContainerScheduler: EventType: RECOVERY_COMPLETED
> --
>
> Key: YARN-9594
> URL: https://issues.apache.org/jira/browse/YARN-9594
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Assignee: lujie
>Priority: Major
> Attachments: YARN-9594_1.patch
>
>
> It seems that we miss a break in switch-case
> {code:java}
> case RECOVERY_COMPLETED:
> startPendingContainers(maxOppQueueLength <= 0);
> metrics.setQueuedContainers(queuedOpportunisticContainers.size(),
> queuedGuaranteedContainers.size());
> //break;missed
> default:
> LOG.error("Unknown event arrived at ContainerScheduler: "
> + event.toString());
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-9594) Unknown event arrived at ContainerScheduler: EventType: RECOVERY_COMPLETED
[
https://issues.apache.org/jira/browse/YARN-9594?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-9594:
Description:
It seems that we miss a break in switch-case
{code:java}
case RECOVERY_COMPLETED:
startPendingContainers(maxOppQueueLength <= 0);
metrics.setQueuedContainers(queuedOpportunisticContainers.size(),
queuedGuaranteedContainers.size());
//break;missed
default:
LOG.error("Unknown event arrived at ContainerScheduler: "
+ event.toString());
{code}
was:
It seems that we miss a break in switch-case
{code:java}
case RECOVERY_COMPLETED:
startPendingContainers(maxOppQueueLength <= 0);
metrics.setQueuedContainers(queuedOpportunisticContainers.size(),
queuedGuaranteedContainers.size());
//break;
default:
LOG.error("Unknown event arrived at ContainerScheduler: "
+ event.toString());
{code}
> Unknown event arrived at ContainerScheduler: EventType: RECOVERY_COMPLETED
> --
>
> Key: YARN-9594
> URL: https://issues.apache.org/jira/browse/YARN-9594
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Assignee: lujie
>Priority: Major
> Attachments: YARN-9594_1.patch
>
>
> It seems that we miss a break in switch-case
> {code:java}
> case RECOVERY_COMPLETED:
> startPendingContainers(maxOppQueueLength <= 0);
> metrics.setQueuedContainers(queuedOpportunisticContainers.size(),
> queuedGuaranteedContainers.size());
> //break;missed
> default:
> LOG.error("Unknown event arrived at ContainerScheduler: "
> + event.toString());
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-9594) Unknown event arrived at ContainerScheduler: EventType: RECOVERY_COMPLETED
[
https://issues.apache.org/jira/browse/YARN-9594?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-9594:
Description:
It seems that we miss a break in switch-case
{code:java}
case RECOVERY_COMPLETED:
startPendingContainers(maxOppQueueLength <= 0);
metrics.setQueuedContainers(queuedOpportunisticContainers.size(),
queuedGuaranteedContainers.size());
//break;
default:
LOG.error("Unknown event arrived at ContainerScheduler: "
+ event.toString());
{code}
was:
Seem that we forget a break
{code:java}
case RECOVERY_COMPLETED:
startPendingContainers(maxOppQueueLength <= 0);
metrics.setQueuedContainers(queuedOpportunisticContainers.size(),
queuedGuaranteedContainers.size());
//break;
default:
LOG.error("Unknown event arrived at ContainerScheduler: "
+ event.toString());
{code}
> Unknown event arrived at ContainerScheduler: EventType: RECOVERY_COMPLETED
> --
>
> Key: YARN-9594
> URL: https://issues.apache.org/jira/browse/YARN-9594
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Assignee: lujie
>Priority: Major
> Attachments: YARN-9594_1.patch
>
>
> It seems that we miss a break in switch-case
> {code:java}
> case RECOVERY_COMPLETED:
> startPendingContainers(maxOppQueueLength <= 0);
> metrics.setQueuedContainers(queuedOpportunisticContainers.size(),
> queuedGuaranteedContainers.size());
> //break;
> default:
> LOG.error("Unknown event arrived at ContainerScheduler: "
> + event.toString());
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Assigned] (YARN-9594) Unknown event arrived at ContainerScheduler: EventType: RECOVERY_COMPLETED
[
https://issues.apache.org/jira/browse/YARN-9594?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie reassigned YARN-9594:
---
Assignee: lujie
> Unknown event arrived at ContainerScheduler: EventType: RECOVERY_COMPLETED
> --
>
> Key: YARN-9594
> URL: https://issues.apache.org/jira/browse/YARN-9594
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Assignee: lujie
>Priority: Major
>
> Seem that we forget a break at
> {code:java}
> case RECOVERY_COMPLETED:
> startPendingContainers(maxOppQueueLength <= 0);
> metrics.setQueuedContainers(queuedOpportunisticContainers.size(),
> queuedGuaranteedContainers.size());
> //break;
> default:
> LOG.error("Unknown event arrived at ContainerScheduler: "
> + event.toString());
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-9594) Unknown event arrived at ContainerScheduler: EventType: RECOVERY_COMPLETED
[
https://issues.apache.org/jira/browse/YARN-9594?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-9594:
Description:
Seem that we forget a break at
{code:java}
case RECOVERY_COMPLETED:
startPendingContainers(maxOppQueueLength <= 0);
metrics.setQueuedContainers(queuedOpportunisticContainers.size(),
queuedGuaranteedContainers.size());
//break;
default:
LOG.error("Unknown event arrived at ContainerScheduler: "
+ event.toString());
{code}
was:
{code:java}
case RECOVERY_COMPLETED:
startPendingContainers(maxOppQueueLength <= 0);
metrics.setQueuedContainers(queuedOpportunisticContainers.size(),
queuedGuaranteedContainers.size());
default:
LOG.error("Unknown event arrived at ContainerScheduler: "
+ event.toString());
{code}
> Unknown event arrived at ContainerScheduler: EventType: RECOVERY_COMPLETED
> --
>
> Key: YARN-9594
> URL: https://issues.apache.org/jira/browse/YARN-9594
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Major
>
> Seem that we forget a break at
> {code:java}
> case RECOVERY_COMPLETED:
> startPendingContainers(maxOppQueueLength <= 0);
> metrics.setQueuedContainers(queuedOpportunisticContainers.size(),
> queuedGuaranteedContainers.size());
> //break;
> default:
> LOG.error("Unknown event arrived at ContainerScheduler: "
> + event.toString());
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-9594) Unknown event arrived at ContainerScheduler: EventType: RECOVERY_COMPLETED
[
https://issues.apache.org/jira/browse/YARN-9594?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-9594:
Description:
Seem that we forget a break
{code:java}
case RECOVERY_COMPLETED:
startPendingContainers(maxOppQueueLength <= 0);
metrics.setQueuedContainers(queuedOpportunisticContainers.size(),
queuedGuaranteedContainers.size());
//break;
default:
LOG.error("Unknown event arrived at ContainerScheduler: "
+ event.toString());
{code}
was:
Seem that we forget a break at
{code:java}
case RECOVERY_COMPLETED:
startPendingContainers(maxOppQueueLength <= 0);
metrics.setQueuedContainers(queuedOpportunisticContainers.size(),
queuedGuaranteedContainers.size());
//break;
default:
LOG.error("Unknown event arrived at ContainerScheduler: "
+ event.toString());
{code}
> Unknown event arrived at ContainerScheduler: EventType: RECOVERY_COMPLETED
> --
>
> Key: YARN-9594
> URL: https://issues.apache.org/jira/browse/YARN-9594
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Assignee: lujie
>Priority: Major
>
> Seem that we forget a break
> {code:java}
> case RECOVERY_COMPLETED:
> startPendingContainers(maxOppQueueLength <= 0);
> metrics.setQueuedContainers(queuedOpportunisticContainers.size(),
> queuedGuaranteedContainers.size());
> //break;
> default:
> LOG.error("Unknown event arrived at ContainerScheduler: "
> + event.toString());
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Created] (YARN-9594) Unknown event arrived at ContainerScheduler: EventType: RECOVERY_COMPLETED
lujie created YARN-9594:
---
Summary: Unknown event arrived at ContainerScheduler: EventType:
RECOVERY_COMPLETED
Key: YARN-9594
URL: https://issues.apache.org/jira/browse/YARN-9594
Project: Hadoop YARN
Issue Type: Bug
Reporter: lujie
{code:java}
case RECOVERY_COMPLETED:
startPendingContainers(maxOppQueueLength <= 0);
metrics.setQueuedContainers(queuedOpportunisticContainers.size(),
queuedGuaranteedContainers.size());
default:
LOG.error("Unknown event arrived at ContainerScheduler: "
+ event.toString());
{code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-9589) NPE in HttpURLConnection.disconnect
[
https://issues.apache.org/jira/browse/YARN-9589?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-9589:
Description:
{code:java}
2019-05-30 10:17:59,869 ERROR [IPC Server handler 2 on default port 51923]
org.apache.hadoop.mapred.TaskAttemptListenerImpl: Task:
attempt_1559182584278_0001_r_00_0 - exited : java.lang.NullPointerException
at
sun.net.www.protocol.http.HttpURLConnection.disconnect(HttpURLConnection.java:2821)
at
org.apache.hadoop.mapreduce.task.reduce.Fetcher.closeConnection(Fetcher.java:255)
at org.apache.hadoop.mapreduce.task.reduce.Fetcher.interrupt(Fetcher.java:216)
at org.apache.hadoop.mapreduce.task.reduce.Fetcher.shutDown(Fetcher.java:224)
at org.apache.hadoop.mapreduce.task.reduce.Shuffle.run(Shuffle.java:147)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:377)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:178)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1891)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:172)
{code}
was:
{code:java}
2019-05-30 10:17:59,869 ERROR [IPC Server handler 2 on default port 51923]
org.apache.hadoop.mapred.TaskAttemptListenerImpl: Task:
attempt_1559182584278_0001_r_00_0 - exited : java.lang.NullPointerException
at
sun.net.www.protocol.http.HttpURLConnection.disconnect(HttpURLConnection.java:2821)
at
org.apache.hadoop.mapreduce.task.reduce.Fetcher.closeConnection(Fetcher.java:255)
at org.apache.hadoop.mapreduce.task.reduce.Fetcher.interrupt(Fetcher.java:216)
at org.apache.hadoop.mapreduce.task.reduce.Fetcher.shutDown(Fetcher.java:224)
at org.apache.hadoop.mapreduce.task.reduce.Shuffle.run(Shuffle.java:147)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:377)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:178)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1891)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:172)
{code}
> NPE in HttpURLConnection.disconnect
> ---
>
> Key: YARN-9589
> URL: https://issues.apache.org/jira/browse/YARN-9589
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Major
> Attachments: YARN-9589.zip
>
>
>
> {code:java}
> 2019-05-30 10:17:59,869 ERROR [IPC Server handler 2 on default port 51923]
> org.apache.hadoop.mapred.TaskAttemptListenerImpl: Task:
> attempt_1559182584278_0001_r_00_0 - exited :
> java.lang.NullPointerException
> at
> sun.net.www.protocol.http.HttpURLConnection.disconnect(HttpURLConnection.java:2821)
> at
> org.apache.hadoop.mapreduce.task.reduce.Fetcher.closeConnection(Fetcher.java:255)
> at org.apache.hadoop.mapreduce.task.reduce.Fetcher.interrupt(Fetcher.java:216)
> at org.apache.hadoop.mapreduce.task.reduce.Fetcher.shutDown(Fetcher.java:224)
> at org.apache.hadoop.mapreduce.task.reduce.Shuffle.run(Shuffle.java:147)
> at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:377)
> at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:178)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1891)
> at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:172)
> {code}
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-9589) NPE in HttpURLConnection.disconnect
[
https://issues.apache.org/jira/browse/YARN-9589?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16851677#comment-16851677
]
lujie commented on YARN-9589:
-
Attach the log files
> NPE in HttpURLConnection.disconnect
> ---
>
> Key: YARN-9589
> URL: https://issues.apache.org/jira/browse/YARN-9589
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Major
> Attachments: YARN-9589.zip
>
>
>
>
> {code:java}
> 2019-05-30 10:17:59,869 ERROR [IPC Server handler 2 on default port 51923]
> org.apache.hadoop.mapred.TaskAttemptListenerImpl: Task:
> attempt_1559182584278_0001_r_00_0 - exited :
> java.lang.NullPointerException
> at
> sun.net.www.protocol.http.HttpURLConnection.disconnect(HttpURLConnection.java:2821)
> at
> org.apache.hadoop.mapreduce.task.reduce.Fetcher.closeConnection(Fetcher.java:255)
> at org.apache.hadoop.mapreduce.task.reduce.Fetcher.interrupt(Fetcher.java:216)
> at org.apache.hadoop.mapreduce.task.reduce.Fetcher.shutDown(Fetcher.java:224)
> at org.apache.hadoop.mapreduce.task.reduce.Shuffle.run(Shuffle.java:147)
> at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:377)
> at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:178)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1891)
> at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:172)
> {code}
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-9589) NPE in HttpURLConnection.disconnect
[
https://issues.apache.org/jira/browse/YARN-9589?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-9589:
Attachment: YARN-9589.zip
> NPE in HttpURLConnection.disconnect
> ---
>
> Key: YARN-9589
> URL: https://issues.apache.org/jira/browse/YARN-9589
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Major
> Attachments: YARN-9589.zip
>
>
>
>
> {code:java}
> 2019-05-30 10:17:59,869 ERROR [IPC Server handler 2 on default port 51923]
> org.apache.hadoop.mapred.TaskAttemptListenerImpl: Task:
> attempt_1559182584278_0001_r_00_0 - exited :
> java.lang.NullPointerException
> at
> sun.net.www.protocol.http.HttpURLConnection.disconnect(HttpURLConnection.java:2821)
> at
> org.apache.hadoop.mapreduce.task.reduce.Fetcher.closeConnection(Fetcher.java:255)
> at org.apache.hadoop.mapreduce.task.reduce.Fetcher.interrupt(Fetcher.java:216)
> at org.apache.hadoop.mapreduce.task.reduce.Fetcher.shutDown(Fetcher.java:224)
> at org.apache.hadoop.mapreduce.task.reduce.Shuffle.run(Shuffle.java:147)
> at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:377)
> at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:178)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1891)
> at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:172)
> {code}
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Created] (YARN-9589) NPE in HttpURLConnection.disconnect
lujie created YARN-9589:
---
Summary: NPE in HttpURLConnection.disconnect
Key: YARN-9589
URL: https://issues.apache.org/jira/browse/YARN-9589
Project: Hadoop YARN
Issue Type: Bug
Reporter: lujie
{code:java}
2019-05-30 10:17:59,869 ERROR [IPC Server handler 2 on default port 51923]
org.apache.hadoop.mapred.TaskAttemptListenerImpl: Task:
attempt_1559182584278_0001_r_00_0 - exited : java.lang.NullPointerException
at
sun.net.www.protocol.http.HttpURLConnection.disconnect(HttpURLConnection.java:2821)
at
org.apache.hadoop.mapreduce.task.reduce.Fetcher.closeConnection(Fetcher.java:255)
at org.apache.hadoop.mapreduce.task.reduce.Fetcher.interrupt(Fetcher.java:216)
at org.apache.hadoop.mapreduce.task.reduce.Fetcher.shutDown(Fetcher.java:224)
at org.apache.hadoop.mapreduce.task.reduce.Shuffle.run(Shuffle.java:147)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:377)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:178)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1891)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:172)
{code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-9588) Resource leak due to InvalidToken: appattempt_XXX not found in AMRMTokenSecretManager while RM reboot
[
https://issues.apache.org/jira/browse/YARN-9588?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16851669#comment-16851669
]
lujie commented on YARN-9588:
-
while we reproduce this bug, We also meet NPE in RM recovery, attach the log
> Resource leak due to InvalidToken: appattempt_XXX not found in
> AMRMTokenSecretManager while RM reboot
> -
>
> Key: YARN-9588
> URL: https://issues.apache.org/jira/browse/YARN-9588
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Critical
> Attachments: bug.zip, bugwithNPE.zip
>
>
> HI:
> RM reboot during AM unregistered,then one error happens:
> {code:java}
> 2019-05-29 18:55:11,112 ERROR [Thread-76]
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator: Exception while
> unregistering
> org.apache.hadoop.security.token.SecretManager$InvalidToken:
> appattempt_1559127208490_0001_01 not found in AMRMTokenSecretManager.
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
> at
> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
> at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:422)
> at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53)
> at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateIOException(RPCUtil.java:80)
> at
> org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:119)
> at
> org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.finishApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:94)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:497)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)
> at com.sun.proxy.$Proxy84.finishApplicationMaster(Unknown Source)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.doUnregistration(RMCommunicator.java:227)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.unregister(RMCommunicator.java:189)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.serviceStop(RMCommunicator.java:267)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator.serviceStop(RMContainerAllocator.java:327)
> at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
> at org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:54)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster$ContainerAllocatorRouter.serviceStop(MRAppMaster.java:985)
> at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
> at org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:54)
> at
> org.apache.hadoop.service.ServiceOperations.stopQuietly(ServiceOperations.java:102)
> at org.apache.hadoop.service.CompositeService.stop(CompositeService.java:158)
> at
> org.apache.hadoop.service.CompositeService.serviceStop(CompositeService.java:132)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceStop(MRAppMaster.java:1868)
> at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
> at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.stop(MRAppMaster.java:1309)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster.shutDownJob(MRAppMaster.java:668)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster$JobFinishEventHandler$1.run(MRAppMaster.java:747)
> Caused by:
> org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken):
> appattempt_1559127208490_0001_01 not found in AMRMTokenSecretManager.
> at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1547)
> at org.apache.hadoop.ipc.Client.call(Client.java:1493)
> at org.apache.hadoop.ipc.Client.call(Client.java:1392)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:233)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118)
> at com.sun.proxy.$Proxy83.finishApplicationMaster(Unknown Source)
> at
> org.apache.hadoop.yar
[jira] [Updated] (YARN-9588) Resource leak due to InvalidToken: appattempt_XXX not found in AMRMTokenSecretManager while RM reboot
[
https://issues.apache.org/jira/browse/YARN-9588?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-9588:
Attachment: bugwithNPE.zip
> Resource leak due to InvalidToken: appattempt_XXX not found in
> AMRMTokenSecretManager while RM reboot
> -
>
> Key: YARN-9588
> URL: https://issues.apache.org/jira/browse/YARN-9588
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Critical
> Attachments: bug.zip, bugwithNPE.zip
>
>
> HI:
> RM reboot during AM unregistered,then one error happens:
> {code:java}
> 2019-05-29 18:55:11,112 ERROR [Thread-76]
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator: Exception while
> unregistering
> org.apache.hadoop.security.token.SecretManager$InvalidToken:
> appattempt_1559127208490_0001_01 not found in AMRMTokenSecretManager.
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
> at
> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
> at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:422)
> at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53)
> at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateIOException(RPCUtil.java:80)
> at
> org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:119)
> at
> org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.finishApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:94)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:497)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)
> at com.sun.proxy.$Proxy84.finishApplicationMaster(Unknown Source)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.doUnregistration(RMCommunicator.java:227)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.unregister(RMCommunicator.java:189)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.serviceStop(RMCommunicator.java:267)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator.serviceStop(RMContainerAllocator.java:327)
> at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
> at org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:54)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster$ContainerAllocatorRouter.serviceStop(MRAppMaster.java:985)
> at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
> at org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:54)
> at
> org.apache.hadoop.service.ServiceOperations.stopQuietly(ServiceOperations.java:102)
> at org.apache.hadoop.service.CompositeService.stop(CompositeService.java:158)
> at
> org.apache.hadoop.service.CompositeService.serviceStop(CompositeService.java:132)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceStop(MRAppMaster.java:1868)
> at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
> at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.stop(MRAppMaster.java:1309)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster.shutDownJob(MRAppMaster.java:668)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster$JobFinishEventHandler$1.run(MRAppMaster.java:747)
> Caused by:
> org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken):
> appattempt_1559127208490_0001_01 not found in AMRMTokenSecretManager.
> at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1547)
> at org.apache.hadoop.ipc.Client.call(Client.java:1493)
> at org.apache.hadoop.ipc.Client.call(Client.java:1392)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:233)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118)
> at com.sun.proxy.$Proxy83.finishApplicationMaster(Unknown Source)
> at
> org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.finishApplicationMaster(ApplicationMasterProt
[jira] [Updated] (YARN-9588) Resource leak due to InvalidToken: appattempt_XXX not found in AMRMTokenSecretManager while RM reboot
[
https://issues.apache.org/jira/browse/YARN-9588?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-9588:
Summary: Resource leak due to InvalidToken: appattempt_XXX not found in
AMRMTokenSecretManager while RM reboot (was: InvalidToken: appattempt_XXX not
found in AMRMTokenSecretManager while RM reboot)
> Resource leak due to InvalidToken: appattempt_XXX not found in
> AMRMTokenSecretManager while RM reboot
> -
>
> Key: YARN-9588
> URL: https://issues.apache.org/jira/browse/YARN-9588
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Critical
> Attachments: bug.zip
>
>
> HI:
> RM reboot during AM unregistered,then one error happens:
> {code:java}
> 2019-05-29 18:55:11,112 ERROR [Thread-76]
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator: Exception while
> unregistering
> org.apache.hadoop.security.token.SecretManager$InvalidToken:
> appattempt_1559127208490_0001_01 not found in AMRMTokenSecretManager.
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
> at
> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
> at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:422)
> at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53)
> at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateIOException(RPCUtil.java:80)
> at
> org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:119)
> at
> org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.finishApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:94)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:497)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)
> at com.sun.proxy.$Proxy84.finishApplicationMaster(Unknown Source)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.doUnregistration(RMCommunicator.java:227)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.unregister(RMCommunicator.java:189)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.serviceStop(RMCommunicator.java:267)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator.serviceStop(RMContainerAllocator.java:327)
> at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
> at org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:54)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster$ContainerAllocatorRouter.serviceStop(MRAppMaster.java:985)
> at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
> at org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:54)
> at
> org.apache.hadoop.service.ServiceOperations.stopQuietly(ServiceOperations.java:102)
> at org.apache.hadoop.service.CompositeService.stop(CompositeService.java:158)
> at
> org.apache.hadoop.service.CompositeService.serviceStop(CompositeService.java:132)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceStop(MRAppMaster.java:1868)
> at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
> at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.stop(MRAppMaster.java:1309)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster.shutDownJob(MRAppMaster.java:668)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster$JobFinishEventHandler$1.run(MRAppMaster.java:747)
> Caused by:
> org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken):
> appattempt_1559127208490_0001_01 not found in AMRMTokenSecretManager.
> at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1547)
> at org.apache.hadoop.ipc.Client.call(Client.java:1493)
> at org.apache.hadoop.ipc.Client.call(Client.java:1392)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:233)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118)
> at com.sun.proxy.$Proxy83.finishApplicat
[jira] [Commented] (YARN-9588) InvalidToken: appattempt_XXX not found in AMRMTokenSecretManager while RM reboot
[
https://issues.apache.org/jira/browse/YARN-9588?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16851598#comment-16851598
]
lujie commented on YARN-9588:
-
Attach the log for analysis
> InvalidToken: appattempt_XXX not found in AMRMTokenSecretManager while RM
> reboot
>
>
> Key: YARN-9588
> URL: https://issues.apache.org/jira/browse/YARN-9588
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Critical
> Attachments: bug.zip
>
>
> HI:
> RM reboot during AM unregistered,then one error happens:
> {code:java}
> 2019-05-29 18:55:11,112 ERROR [Thread-76]
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator: Exception while
> unregistering
> org.apache.hadoop.security.token.SecretManager$InvalidToken:
> appattempt_1559127208490_0001_01 not found in AMRMTokenSecretManager.
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
> at
> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
> at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:422)
> at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53)
> at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateIOException(RPCUtil.java:80)
> at
> org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:119)
> at
> org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.finishApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:94)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:497)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)
> at com.sun.proxy.$Proxy84.finishApplicationMaster(Unknown Source)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.doUnregistration(RMCommunicator.java:227)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.unregister(RMCommunicator.java:189)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.serviceStop(RMCommunicator.java:267)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator.serviceStop(RMContainerAllocator.java:327)
> at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
> at org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:54)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster$ContainerAllocatorRouter.serviceStop(MRAppMaster.java:985)
> at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
> at org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:54)
> at
> org.apache.hadoop.service.ServiceOperations.stopQuietly(ServiceOperations.java:102)
> at org.apache.hadoop.service.CompositeService.stop(CompositeService.java:158)
> at
> org.apache.hadoop.service.CompositeService.serviceStop(CompositeService.java:132)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceStop(MRAppMaster.java:1868)
> at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
> at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.stop(MRAppMaster.java:1309)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster.shutDownJob(MRAppMaster.java:668)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster$JobFinishEventHandler$1.run(MRAppMaster.java:747)
> Caused by:
> org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken):
> appattempt_1559127208490_0001_01 not found in AMRMTokenSecretManager.
> at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1547)
> at org.apache.hadoop.ipc.Client.call(Client.java:1493)
> at org.apache.hadoop.ipc.Client.call(Client.java:1392)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:233)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118)
> at com.sun.proxy.$Proxy83.finishApplicationMaster(Unknown Source)
> at
> org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.finishApplicationMaster(ApplicationMasterProtoco
[jira] [Updated] (YARN-9588) InvalidToken: appattempt_XXX not found in AMRMTokenSecretManager while RM reboot
[
https://issues.apache.org/jira/browse/YARN-9588?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-9588:
Attachment: bug.zip
> InvalidToken: appattempt_XXX not found in AMRMTokenSecretManager while RM
> reboot
>
>
> Key: YARN-9588
> URL: https://issues.apache.org/jira/browse/YARN-9588
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Priority: Critical
> Attachments: bug.zip
>
>
> HI:
> RM reboot during AM unregistered,then one error happens:
> {code:java}
> 2019-05-29 18:55:11,112 ERROR [Thread-76]
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator: Exception while
> unregistering
> org.apache.hadoop.security.token.SecretManager$InvalidToken:
> appattempt_1559127208490_0001_01 not found in AMRMTokenSecretManager.
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
> at
> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
> at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:422)
> at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53)
> at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateIOException(RPCUtil.java:80)
> at
> org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:119)
> at
> org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.finishApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:94)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:497)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)
> at com.sun.proxy.$Proxy84.finishApplicationMaster(Unknown Source)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.doUnregistration(RMCommunicator.java:227)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.unregister(RMCommunicator.java:189)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.serviceStop(RMCommunicator.java:267)
> at
> org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator.serviceStop(RMContainerAllocator.java:327)
> at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
> at org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:54)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster$ContainerAllocatorRouter.serviceStop(MRAppMaster.java:985)
> at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
> at org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:54)
> at
> org.apache.hadoop.service.ServiceOperations.stopQuietly(ServiceOperations.java:102)
> at org.apache.hadoop.service.CompositeService.stop(CompositeService.java:158)
> at
> org.apache.hadoop.service.CompositeService.serviceStop(CompositeService.java:132)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceStop(MRAppMaster.java:1868)
> at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
> at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.stop(MRAppMaster.java:1309)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster.shutDownJob(MRAppMaster.java:668)
> at
> org.apache.hadoop.mapreduce.v2.app.MRAppMaster$JobFinishEventHandler$1.run(MRAppMaster.java:747)
> Caused by:
> org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken):
> appattempt_1559127208490_0001_01 not found in AMRMTokenSecretManager.
> at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1547)
> at org.apache.hadoop.ipc.Client.call(Client.java:1493)
> at org.apache.hadoop.ipc.Client.call(Client.java:1392)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:233)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118)
> at com.sun.proxy.$Proxy83.finishApplicationMaster(Unknown Source)
> at
> org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.finishApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:92)
> ... 27 more
> {code}
> This will make
[jira] [Updated] (YARN-9588) InvalidToken: appattempt_XXX not found in AMRMTokenSecretManager while RM reboot
[
https://issues.apache.org/jira/browse/YARN-9588?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-9588:
Description:
HI:
RM reboot during AM unregistered,then one error happens:
{code:java}
2019-05-29 18:55:11,112 ERROR [Thread-76]
org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator: Exception while
unregistering
org.apache.hadoop.security.token.SecretManager$InvalidToken:
appattempt_1559127208490_0001_01 not found in AMRMTokenSecretManager.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:422)
at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53)
at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateIOException(RPCUtil.java:80)
at org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:119)
at
org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.finishApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:94)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)
at com.sun.proxy.$Proxy84.finishApplicationMaster(Unknown Source)
at
org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.doUnregistration(RMCommunicator.java:227)
at
org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.unregister(RMCommunicator.java:189)
at
org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.serviceStop(RMCommunicator.java:267)
at
org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator.serviceStop(RMContainerAllocator.java:327)
at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
at org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:54)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster$ContainerAllocatorRouter.serviceStop(MRAppMaster.java:985)
at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
at org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:54)
at
org.apache.hadoop.service.ServiceOperations.stopQuietly(ServiceOperations.java:102)
at org.apache.hadoop.service.CompositeService.stop(CompositeService.java:158)
at
org.apache.hadoop.service.CompositeService.serviceStop(CompositeService.java:132)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceStop(MRAppMaster.java:1868)
at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.stop(MRAppMaster.java:1309)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster.shutDownJob(MRAppMaster.java:668)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster$JobFinishEventHandler$1.run(MRAppMaster.java:747)
Caused by:
org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken):
appattempt_1559127208490_0001_01 not found in AMRMTokenSecretManager.
at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1547)
at org.apache.hadoop.ipc.Client.call(Client.java:1493)
at org.apache.hadoop.ipc.Client.call(Client.java:1392)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:233)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118)
at com.sun.proxy.$Proxy83.finishApplicationMaster(Unknown Source)
at
org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.finishApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:92)
... 27 more
{code}
This will make the isLastAMRetry = false; current AM won't clean up the
staging dir:
{code:java}
2019-05-29 18:55:11,116 INFO [Thread-76]
org.apache.hadoop.mapreduce.v2.app.MRAppMaster: Skipping cleaning up the
staging dir. assuming AM will be retried.{code}
But NO AM will be retried, because the application is succeeded
{code:java}
2019-05-29 18:55:15,458 INFO
org.apache.hadoop.yarn.server.resourcemanager.RMAppManager: Successfully
recovered 1 out of 1 applications
2019-05-29 18:55:15,459 INFO
org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager:
Recovering CA Certificate and Priv
[jira] [Updated] (YARN-9588) InvalidToken: appattempt_XXX not found in AMRMTokenSecretManager while RM reboot
[
https://issues.apache.org/jira/browse/YARN-9588?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-9588:
Description:
HI:
while application is success, but before AM unregistered, RM reboot, then one
error happens:
{code:java}
2019-05-29 18:55:11,112 ERROR [Thread-76]
org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator: Exception while
unregistering
org.apache.hadoop.security.token.SecretManager$InvalidToken:
appattempt_1559127208490_0001_01 not found in AMRMTokenSecretManager.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:422)
at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53)
at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateIOException(RPCUtil.java:80)
at org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:119)
at
org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.finishApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:94)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)
at com.sun.proxy.$Proxy84.finishApplicationMaster(Unknown Source)
at
org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.doUnregistration(RMCommunicator.java:227)
at
org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.unregister(RMCommunicator.java:189)
at
org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.serviceStop(RMCommunicator.java:267)
at
org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator.serviceStop(RMContainerAllocator.java:327)
at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
at org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:54)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster$ContainerAllocatorRouter.serviceStop(MRAppMaster.java:985)
at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
at org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:54)
at
org.apache.hadoop.service.ServiceOperations.stopQuietly(ServiceOperations.java:102)
at org.apache.hadoop.service.CompositeService.stop(CompositeService.java:158)
at
org.apache.hadoop.service.CompositeService.serviceStop(CompositeService.java:132)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceStop(MRAppMaster.java:1868)
at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.stop(MRAppMaster.java:1309)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster.shutDownJob(MRAppMaster.java:668)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster$JobFinishEventHandler$1.run(MRAppMaster.java:747)
Caused by:
org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken):
appattempt_1559127208490_0001_01 not found in AMRMTokenSecretManager.
at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1547)
at org.apache.hadoop.ipc.Client.call(Client.java:1493)
at org.apache.hadoop.ipc.Client.call(Client.java:1392)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:233)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118)
at com.sun.proxy.$Proxy83.finishApplicationMaster(Unknown Source)
at
org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.finishApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:92)
... 27 more
{code}
This will make the isLastAMRetry = false; current AM won't clean up the
staging dir:
{code:java}
2019-05-29 18:55:11,116 INFO [Thread-76]
org.apache.hadoop.mapreduce.v2.app.MRAppMaster: Skipping cleaning up the
staging dir. assuming AM will be retried.{code}
But NO AM will be retried, because the application is succeeded
{code:java}
2019-05-29 18:55:15,458 INFO
org.apache.hadoop.yarn.server.resourcemanager.RMAppManager: Successfully
recovered 1 out of 1 applications
2019-05-29 18:55:15,459 INFO
org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager:
[jira] [Updated] (YARN-9588) InvalidToken: appattempt_XXX not found in AMRMTokenSecretManager while RM reboot
[
https://issues.apache.org/jira/browse/YARN-9588?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-9588:
Description:
HI:
while application is success, but before AM unregistered, RM reboot, then one
error happens:
{code:java}
2019-05-29 18:55:11,112 ERROR [Thread-76]
org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator: Exception while
unregistering
org.apache.hadoop.security.token.SecretManager$InvalidToken:
appattempt_1559127208490_0001_01 not found in AMRMTokenSecretManager.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:422)
at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53)
at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateIOException(RPCUtil.java:80)
at org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:119)
at
org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.finishApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:94)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)
at com.sun.proxy.$Proxy84.finishApplicationMaster(Unknown Source)
at
org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.doUnregistration(RMCommunicator.java:227)
at
org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.unregister(RMCommunicator.java:189)
at
org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.serviceStop(RMCommunicator.java:267)
at
org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator.serviceStop(RMContainerAllocator.java:327)
at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
at org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:54)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster$ContainerAllocatorRouter.serviceStop(MRAppMaster.java:985)
at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
at org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:54)
at
org.apache.hadoop.service.ServiceOperations.stopQuietly(ServiceOperations.java:102)
at org.apache.hadoop.service.CompositeService.stop(CompositeService.java:158)
at
org.apache.hadoop.service.CompositeService.serviceStop(CompositeService.java:132)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceStop(MRAppMaster.java:1868)
at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.stop(MRAppMaster.java:1309)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster.shutDownJob(MRAppMaster.java:668)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster$JobFinishEventHandler$1.run(MRAppMaster.java:747)
Caused by:
org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken):
appattempt_1559127208490_0001_01 not found in AMRMTokenSecretManager.
at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1547)
at org.apache.hadoop.ipc.Client.call(Client.java:1493)
at org.apache.hadoop.ipc.Client.call(Client.java:1392)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:233)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118)
at com.sun.proxy.$Proxy83.finishApplicationMaster(Unknown Source)
at
org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.finishApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:92)
... 27 more
{code}
This will make the isLastAMRetry = false; current AM won't clean up the
staging dir:
{code:java}
org.apache.hadoop.mapreduce.v2.app.MRAppMaster: Skipping cleaning up the
staging dir. assuming AM will be retried.{code}
But NO AM will be retried, because the application is succeeded
{code:java}
2019-05-29 18:55:15,458 INFO
org.apache.hadoop.yarn.server.resourcemanager.RMAppManager: Successfully
recovered 1 out of 1 applications
2019-05-29 18:55:15,459 INFO
org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager:
Recovering CA Certificate and Private Ke
[jira] [Created] (YARN-9588) InvalidToken: appattempt_XXX not found in AMRMTokenSecretManager while RM reboot
lujie created YARN-9588:
---
Summary: InvalidToken: appattempt_XXX not found in
AMRMTokenSecretManager while RM reboot
Key: YARN-9588
URL: https://issues.apache.org/jira/browse/YARN-9588
Project: Hadoop YARN
Issue Type: Bug
Reporter: lujie
HI:
while application is success, but before AM unregistered, RM reboot, then one
error happens:
{code:java}
2019-05-29 18:55:11,112 ERROR [Thread-76]
org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator: Exception while
unregistering
org.apache.hadoop.security.token.SecretManager$InvalidToken:
appattempt_1559127208490_0001_01 not found in AMRMTokenSecretManager.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:422)
at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53)
at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateIOException(RPCUtil.java:80)
at org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:119)
at
org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.finishApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:94)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)
at com.sun.proxy.$Proxy84.finishApplicationMaster(Unknown Source)
at
org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.doUnregistration(RMCommunicator.java:227)
at
org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.unregister(RMCommunicator.java:189)
at
org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.serviceStop(RMCommunicator.java:267)
at
org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator.serviceStop(RMContainerAllocator.java:327)
at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
at org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:54)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster$ContainerAllocatorRouter.serviceStop(MRAppMaster.java:985)
at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
at org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:54)
at
org.apache.hadoop.service.ServiceOperations.stopQuietly(ServiceOperations.java:102)
at org.apache.hadoop.service.CompositeService.stop(CompositeService.java:158)
at
org.apache.hadoop.service.CompositeService.serviceStop(CompositeService.java:132)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceStop(MRAppMaster.java:1868)
at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:220)
at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.stop(MRAppMaster.java:1309)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster.shutDownJob(MRAppMaster.java:668)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster$JobFinishEventHandler$1.run(MRAppMaster.java:747)
Caused by:
org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken):
appattempt_1559127208490_0001_01 not found in AMRMTokenSecretManager.
at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1547)
at org.apache.hadoop.ipc.Client.call(Client.java:1493)
at org.apache.hadoop.ipc.Client.call(Client.java:1392)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:233)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118)
at com.sun.proxy.$Proxy83.finishApplicationMaster(Unknown Source)
at
org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.finishApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:92)
... 27 more
{code}
This will make the isLastAMRetry = false; current AM won't clean up the
staging dir:
{code:java}
org.apache.hadoop.mapreduce.v2.app.MRAppMaster: Skipping cleaning up the
staging dir. assuming AM will be retried.
{code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For
[jira] [Commented] (YARN-9248) RMContainerImpl:Invalid event: ACQUIRED at KILLED
[
https://issues.apache.org/jira/browse/YARN-9248?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16776306#comment-16776306
]
lujie commented on YARN-9248:
-
Hi [~cheersyang]
[^YARN-9248_5.patch] is the latest patch with UT.
> RMContainerImpl:Invalid event: ACQUIRED at KILLED
> -
>
> Key: YARN-9248
> URL: https://issues.apache.org/jira/browse/YARN-9248
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Assignee: lujie
>Priority: Major
> Attachments: YARN-9248_1.patch, YARN-9248_2.patch, YARN-9248_3.patch,
> YARN-9248_4.patch, YARN-9248_5.patch, YARN-9248_6.patch
>
>
> {code:java}
> 2019-01-29 11:46:53,596 ERROR
> org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl:
> Can't handle this event at current state
> org.apache.hadoop.yarn.state.InvalidStateTransitionException: Invalid event:
> ACQUIRED at KILLED
> at
> org.apache.hadoop.yarn.state.StateMachineFactory.doTransition(StateMachineFactory.java:305)
> at
> org.apache.hadoop.yarn.state.StateMachineFactory.access$500(StateMachineFactory.java:46)
> at
> org.apache.hadoop.yarn.state.StateMachineFactory$InternalStateMachine.doTransition(StateMachineFactory.java:487)
> at
> org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl.handle(RMContainerImpl.java:475)
> at
> org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl.handle(RMContainerImpl.java:67)
> at
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService.handleNewContainers(OpportunisticContainerAllocatorAMService.java:351)
> at
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService.access$100(OpportunisticContainerAllocatorAMService.java:94)
> at
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService$OpportunisticAMSProcessor.allocate(OpportunisticContainerAllocatorAMService.java:197)
> at
> org.apache.hadoop.yarn.server.resourcemanager.AMSProcessingChain.allocate(AMSProcessingChain.java:92)
> at
> org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService.allocate(ApplicationMasterService.java:424)
> at
> org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl.allocate(ApplicationMasterProtocolPBServiceImpl.java:60)
> at
> org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$2.callBlockingMethod(ApplicationMasterProtocol.java:99)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:530)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1070)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:943)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:878)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2830)
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Comment Edited] (YARN-9238) Avoid allocating opportunistic containers to previous/removed/non-exist application attempt
[
https://issues.apache.org/jira/browse/YARN-9238?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16775139#comment-16775139
]
lujie edited comment on YARN-9238 at 2/22/19 1:32 PM:
--
Hi：[~cheersyang]
One more thing. Could please review the patch that fix YARN-9248? That bug also
happens to opportunistic container.
was (Author: xiaoheipangzi):
Hi：[~cheersyang]
One more thing. Could please review the patch that fix YARN-9248? This bug also
happens to opportunistic container.
> Avoid allocating opportunistic containers to previous/removed/non-exist
> application attempt
> ---
>
> Key: YARN-9238
> URL: https://issues.apache.org/jira/browse/YARN-9238
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Attachments: YARN-9238_1.patch, YARN-9238_2.patch, YARN-9238_3.patch,
> hadoop-test-resourcemanager-hadoop11.log
>
>
> See
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService.OpportunisticAMSProcessor.allocate
> {code:java}
> // Allocate OPPORTUNISTIC containers.
> 171. SchedulerApplicationAttempt appAttempt =
> 172.((AbstractYarnScheduler)rmContext.getScheduler())
> 173. .getApplicationAttempt(appAttemptId);
> 174.
> 175. OpportunisticContainerContext oppCtx =
> 176. appAttempt.getOpportunisticContainerContext();
> 177. oppCtx.updateNodeList(getLeastLoadedNodes());
> {code}
> MRAppmaster crashes before before allocate#171, ResourceManager will start
> the new appAttempt and do
> {code:java}
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication.setCurrentAppAttempt(T
> currentAttempt){
> this.currentAttempt = currentAttempt;
> }{code}
> hence the allocate#171 will get the new appAttmept and its field
> OpportunisticContainerContext hasn't been initialized.
> so oopCtx ==null at and null pointer happens at line 177
> {code:java}
> java.lang.NullPointerException
> at
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService$OpportunisticAMSProcessor.allocate(OpportunisticContainerAllocatorAMService.java:177)
> at
> org.apache.hadoop.yarn.server.resourcemanager.AMSProcessingChain.allocate(AMSProcessingChain.java:92)
> at
> org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService.allocate(ApplicationMasterService.java:424)
> at
> org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl.allocate(ApplicationMasterProtocolPBServiceImpl.java:60)
> at
> org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$2.callBlockingMethod(ApplicationMasterProtocol.java:99)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:530)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1070)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:943)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:878)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2830) {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-9238) Avoid allocating opportunistic containers to previous/removed/non-exist application attempt
[
https://issues.apache.org/jira/browse/YARN-9238?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16775139#comment-16775139
]
lujie commented on YARN-9238:
-
Hi：[~cheersyang]
One more thing. Could please review the patch that fix YARN-9248? This bug also
happens to opportunistic container.
> Avoid allocating opportunistic containers to previous/removed/non-exist
> application attempt
> ---
>
> Key: YARN-9238
> URL: https://issues.apache.org/jira/browse/YARN-9238
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Attachments: YARN-9238_1.patch, YARN-9238_2.patch, YARN-9238_3.patch,
> hadoop-test-resourcemanager-hadoop11.log
>
>
> See
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService.OpportunisticAMSProcessor.allocate
> {code:java}
> // Allocate OPPORTUNISTIC containers.
> 171. SchedulerApplicationAttempt appAttempt =
> 172.((AbstractYarnScheduler)rmContext.getScheduler())
> 173. .getApplicationAttempt(appAttemptId);
> 174.
> 175. OpportunisticContainerContext oppCtx =
> 176. appAttempt.getOpportunisticContainerContext();
> 177. oppCtx.updateNodeList(getLeastLoadedNodes());
> {code}
> MRAppmaster crashes before before allocate#171, ResourceManager will start
> the new appAttempt and do
> {code:java}
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication.setCurrentAppAttempt(T
> currentAttempt){
> this.currentAttempt = currentAttempt;
> }{code}
> hence the allocate#171 will get the new appAttmept and its field
> OpportunisticContainerContext hasn't been initialized.
> so oopCtx ==null at and null pointer happens at line 177
> {code:java}
> java.lang.NullPointerException
> at
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService$OpportunisticAMSProcessor.allocate(OpportunisticContainerAllocatorAMService.java:177)
> at
> org.apache.hadoop.yarn.server.resourcemanager.AMSProcessingChain.allocate(AMSProcessingChain.java:92)
> at
> org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService.allocate(ApplicationMasterService.java:424)
> at
> org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl.allocate(ApplicationMasterProtocolPBServiceImpl.java:60)
> at
> org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$2.callBlockingMethod(ApplicationMasterProtocol.java:99)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:530)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1070)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:943)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:878)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2830) {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-9238) Allocate on previous or removed or non existent application attempt
[
https://issues.apache.org/jira/browse/YARN-9238?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16774871#comment-16774871
]
lujie commented on YARN-9238:
-
Change the title and simplify the describtion
> Allocate on previous or removed or non existent application attempt
> ---
>
> Key: YARN-9238
> URL: https://issues.apache.org/jira/browse/YARN-9238
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Attachments: YARN-9238_1.patch, YARN-9238_2.patch, YARN-9238_3.patch,
> hadoop-test-resourcemanager-hadoop11.log
>
>
> See
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService.OpportunisticAMSProcessor.allocate
> {code:java}
> // Allocate OPPORTUNISTIC containers.
> 171. SchedulerApplicationAttempt appAttempt =
> 172.((AbstractYarnScheduler)rmContext.getScheduler())
> 173. .getApplicationAttempt(appAttemptId);
> 174.
> 175. OpportunisticContainerContext oppCtx =
> 176. appAttempt.getOpportunisticContainerContext();
> 177. oppCtx.updateNodeList(getLeastLoadedNodes());
> {code}
> MRAppmaster crashes before before allocate#171, ResourceManager will start
> the new appAttempt and do
> {code:java}
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication.setCurrentAppAttempt(T
> currentAttempt){
> this.currentAttempt = currentAttempt;
> }{code}
> hence the allocate#171 will get the new appAttmept and its field
> OpportunisticContainerContext hasn't been initialized.
> so oopCtx ==null at and null pointer happens at line 177
> {code:java}
> java.lang.NullPointerException
> at
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService$OpportunisticAMSProcessor.allocate(OpportunisticContainerAllocatorAMService.java:177)
> at
> org.apache.hadoop.yarn.server.resourcemanager.AMSProcessingChain.allocate(AMSProcessingChain.java:92)
> at
> org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService.allocate(ApplicationMasterService.java:424)
> at
> org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl.allocate(ApplicationMasterProtocolPBServiceImpl.java:60)
> at
> org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$2.callBlockingMethod(ApplicationMasterProtocol.java:99)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:530)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1070)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:943)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:878)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2830) {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-9238) Allocate on previous or removed or non existent application attempt
[
https://issues.apache.org/jira/browse/YARN-9238?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-9238:
Description:
See
org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService.OpportunisticAMSProcessor.allocate
{code:java}
// Allocate OPPORTUNISTIC containers.
171. SchedulerApplicationAttempt appAttempt =
172.((AbstractYarnScheduler)rmContext.getScheduler())
173. .getApplicationAttempt(appAttemptId);
174.
175. OpportunisticContainerContext oppCtx =
176. appAttempt.getOpportunisticContainerContext();
177. oppCtx.updateNodeList(getLeastLoadedNodes());
{code}
MRAppmaster crashes before before allocate#171, ResourceManager will start the
new appAttempt and do
{code:java}
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication.setCurrentAppAttempt(T
currentAttempt){
this.currentAttempt = currentAttempt;
}{code}
hence the allocate#171 will get the new appAttmept and its field
OpportunisticContainerContext hasn't been initialized.
so oopCtx ==null at and null pointer happens at line 177
{code:java}
java.lang.NullPointerException
at
org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService$OpportunisticAMSProcessor.allocate(OpportunisticContainerAllocatorAMService.java:177)
at
org.apache.hadoop.yarn.server.resourcemanager.AMSProcessingChain.allocate(AMSProcessingChain.java:92)
at
org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService.allocate(ApplicationMasterService.java:424)
at
org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl.allocate(ApplicationMasterProtocolPBServiceImpl.java:60)
at
org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$2.callBlockingMethod(ApplicationMasterProtocol.java:99)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:530)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1070)
at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:943)
at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:878)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2830) {code}
was:
See
org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService.OpportunisticAMSProcessor.allocate
{code:java}
// Allocate OPPORTUNISTIC containers.
171. SchedulerApplicationAttempt appAttempt =
172.((AbstractYarnScheduler)rmContext.getScheduler())
173. .getApplicationAttempt(appAttemptId);
174.
175. OpportunisticContainerContext oppCtx =
176. appAttempt.getOpportunisticContainerContext();
177. oppCtx.updateNodeList(getLeastLoadedNodes());
{code}
if "allocate" arrive at line#171 and MRAppmaster crashes, ResourceManager will
start the new appAttempt and do
{code:java}
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication.setCurrentAppAttempt(T
currentAttempt){
this.currentAttempt = currentAttempt;
}{code}
the new appAttmept hasn't init its field OpportunisticContainerContext , hence
oopCtx ==null and null pointer happens at line 177
{code:java}
java.lang.NullPointerException
at
org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService$OpportunisticAMSProcessor.allocate(OpportunisticContainerAllocatorAMService.java:177)
at
org.apache.hadoop.yarn.server.resourcemanager.AMSProcessingChain.allocate(AMSProcessingChain.java:92)
at
org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService.allocate(ApplicationMasterService.java:424)
at
org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl.allocate(ApplicationMasterProtocolPBServiceImpl.java:60)
at
org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$2.callBlockingMethod(ApplicationMasterProtocol.java:99)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:530)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1070)
at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:943)
at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:878)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2830) {code}
> Allocate on previous or removed or non existent application attempt
> ---
>
> Key: YARN-9238
> URL: https://issues.apache.org/jira/browse/YARN-9238
> Project: Hadoop YARN
> Issue Type: Bug
>Report
[jira] [Updated] (YARN-9238) Allocate on previous or removed or non existent application attempt
[
https://issues.apache.org/jira/browse/YARN-9238?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-9238:
Description:
See
org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService.OpportunisticAMSProcessor.allocate
{code:java}
// Allocate OPPORTUNISTIC containers.
171. SchedulerApplicationAttempt appAttempt =
172.((AbstractYarnScheduler)rmContext.getScheduler())
173. .getApplicationAttempt(appAttemptId);
174.
175. OpportunisticContainerContext oppCtx =
176. appAttempt.getOpportunisticContainerContext();
177. oppCtx.updateNodeList(getLeastLoadedNodes());
{code}
if "allocate" arrive at line#171 and MRAppmaster crashes, ResourceManager will
start the new appAttempt and do
{code:java}
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication.setCurrentAppAttempt(T
currentAttempt){
this.currentAttempt = currentAttempt;
}{code}
the new appAttmept hasn't init its field OpportunisticContainerContext , hence
oopCtx ==null and null pointer happens at line 177
{code:java}
java.lang.NullPointerException
at
org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService$OpportunisticAMSProcessor.allocate(OpportunisticContainerAllocatorAMService.java:177)
at
org.apache.hadoop.yarn.server.resourcemanager.AMSProcessingChain.allocate(AMSProcessingChain.java:92)
at
org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService.allocate(ApplicationMasterService.java:424)
at
org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl.allocate(ApplicationMasterProtocolPBServiceImpl.java:60)
at
org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$2.callBlockingMethod(ApplicationMasterProtocol.java:99)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:530)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1070)
at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:943)
at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:878)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2830) {code}
was:
We have found a data race that can make an odd situation.
See
org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService.OpportunisticAMSProcessor.allocate{color:#ff}:(code1){color}
{code:java}
// Allocate OPPORTUNISTIC containers.
171. SchedulerApplicationAttempt appAttempt =
172.((AbstractYarnScheduler)rmContext.getScheduler())
173. .getApplicationAttempt(appAttemptId);
174.
175. OpportunisticContainerContext oppCtx =
176. appAttempt.getOpportunisticContainerContext();
177. oppCtx.updateNodeList(getLeastLoadedNodes());
{code}
if we just crash the current AM(its attemptid is appattempt_0) just before
code1#171, when code1#171~173 continue to execute to get the appAttempt by
appattempt_0, the obtained appAttempt should represent the currenct AM. But
we found that the obtained appAttempt represents the new AM and its attempid
is appattempt_1. This obtained appAttempt has not init its oppCtx, so NPE
happnes at line code1#177.
{code:java}
java.lang.NullPointerException
at
org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService$OpportunisticAMSProcessor.allocate(OpportunisticContainerAllocatorAMService.java:177)
at
org.apache.hadoop.yarn.server.resourcemanager.AMSProcessingChain.allocate(AMSProcessingChain.java:92)
at
org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService.allocate(ApplicationMasterService.java:424)
at
org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl.allocate(ApplicationMasterProtocolPBServiceImpl.java:60)
at
org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$2.callBlockingMethod(ApplicationMasterProtocol.java:99)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:530)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1070)
at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:943)
at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:878)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2830)
{code}
So why old appAttempt disappeares and why we use old appattempt_0 but get the
new appAttempt
We have found the reason. Below code({color:#ff}code2{color}) is the
function body of getApplicationAttempt at code1#173
{code:java}
399. public T getApplicationAttempt(ApplicationAttemptI
[jira] [Updated] (YARN-9238) Allocate on previous or removed or non existent application attempt
[
https://issues.apache.org/jira/browse/YARN-9238?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-9238:
Summary: Allocate on previous or removed or non existent application
attempt (was: We get a wrong attempt by an appAttemptId when AM crash at some
point)
> Allocate on previous or removed or non existent application attempt
> ---
>
> Key: YARN-9238
> URL: https://issues.apache.org/jira/browse/YARN-9238
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Attachments: YARN-9238_1.patch, YARN-9238_2.patch, YARN-9238_3.patch,
> hadoop-test-resourcemanager-hadoop11.log
>
>
> We have found a data race that can make an odd situation.
> See
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService.OpportunisticAMSProcessor.allocate{color:#ff}:(code1){color}
> {code:java}
> // Allocate OPPORTUNISTIC containers.
> 171. SchedulerApplicationAttempt appAttempt =
> 172.((AbstractYarnScheduler)rmContext.getScheduler())
> 173. .getApplicationAttempt(appAttemptId);
> 174.
> 175. OpportunisticContainerContext oppCtx =
> 176. appAttempt.getOpportunisticContainerContext();
> 177. oppCtx.updateNodeList(getLeastLoadedNodes());
> {code}
> if we just crash the current AM(its attemptid is appattempt_0) just before
> code1#171, when code1#171~173 continue to execute to get the appAttempt by
> appattempt_0, the obtained appAttempt should represent the currenct AM. But
> we found that the obtained appAttempt represents the new AM and its
> attempid is appattempt_1. This obtained appAttempt has not init its oppCtx,
> so NPE happnes at line code1#177.
> {code:java}
> java.lang.NullPointerException
> at
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService$OpportunisticAMSProcessor.allocate(OpportunisticContainerAllocatorAMService.java:177)
> at
> org.apache.hadoop.yarn.server.resourcemanager.AMSProcessingChain.allocate(AMSProcessingChain.java:92)
> at
> org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService.allocate(ApplicationMasterService.java:424)
> at
> org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl.allocate(ApplicationMasterProtocolPBServiceImpl.java:60)
> at
> org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$2.callBlockingMethod(ApplicationMasterProtocol.java:99)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:530)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1070)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:943)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:878)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2830)
> {code}
> So why old appAttempt disappeares and why we use old appattempt_0 but get
> the new appAttempt
> We have found the reason. Below code({color:#ff}code2{color}) is the
> function body of getApplicationAttempt at code1#173
> {code:java}
> 399. public T getApplicationAttempt(ApplicationAttemptId
> applicationAttemptId) {
> 400 SchedulerApplication app = applications.get(
> 401 applicationAttemptId.getApplicationId());
> 402 return app == null ? null : app.getCurrentAppAttempt();
> 403 }
> {code}
> when old AM Crash, new AM and new appAttempt comes. The currentAttempt of
> app will be setted as the new appAttempt (see code3). So the code2 #402 will
> return the new appAttempt.
> if AM crashes at the head of allocate function(code1), bug won't happens due
> to ApplicationDoesNotExistInCacheException. AM crashed after code1,
> everything is also ok.
> We shoud add the check: whether the the getted appAttempt have the same id
> with given id.
> patch comes soon!
> {color:#ff}code3{color}
> {code:java}
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication.setCurrentAppAttempt(T
> currentAttempt){
> this.currentAttempt = currentAttempt;
> }
> {code}
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-9238) We get a wrong attempt by an appAttemptId when AM crash at some point
[
https://issues.apache.org/jira/browse/YARN-9238?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-9238:
Attachment: YARN-9238_3.patch
> We get a wrong attempt by an appAttemptId when AM crash at some point
> --
>
> Key: YARN-9238
> URL: https://issues.apache.org/jira/browse/YARN-9238
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Attachments: YARN-9238_1.patch, YARN-9238_2.patch, YARN-9238_3.patch,
> hadoop-test-resourcemanager-hadoop11.log
>
>
> We have found a data race that can make an odd situation.
> See
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService.OpportunisticAMSProcessor.allocate{color:#ff}:(code1){color}
> {code:java}
> // Allocate OPPORTUNISTIC containers.
> 171. SchedulerApplicationAttempt appAttempt =
> 172.((AbstractYarnScheduler)rmContext.getScheduler())
> 173. .getApplicationAttempt(appAttemptId);
> 174.
> 175. OpportunisticContainerContext oppCtx =
> 176. appAttempt.getOpportunisticContainerContext();
> 177. oppCtx.updateNodeList(getLeastLoadedNodes());
> {code}
> if we just crash the current AM(its attemptid is appattempt_0) just before
> code1#171, when code1#171~173 continue to execute to get the appAttempt by
> appattempt_0, the obtained appAttempt should represent the currenct AM. But
> we found that the obtained appAttempt represents the new AM and its
> attempid is appattempt_1. This obtained appAttempt has not init its oppCtx,
> so NPE happnes at line code1#177.
> {code:java}
> java.lang.NullPointerException
> at
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService$OpportunisticAMSProcessor.allocate(OpportunisticContainerAllocatorAMService.java:177)
> at
> org.apache.hadoop.yarn.server.resourcemanager.AMSProcessingChain.allocate(AMSProcessingChain.java:92)
> at
> org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService.allocate(ApplicationMasterService.java:424)
> at
> org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl.allocate(ApplicationMasterProtocolPBServiceImpl.java:60)
> at
> org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$2.callBlockingMethod(ApplicationMasterProtocol.java:99)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:530)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1070)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:943)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:878)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2830)
> {code}
> So why old appAttempt disappeares and why we use old appattempt_0 but get
> the new appAttempt
> We have found the reason. Below code({color:#ff}code2{color}) is the
> function body of getApplicationAttempt at code1#173
> {code:java}
> 399. public T getApplicationAttempt(ApplicationAttemptId
> applicationAttemptId) {
> 400 SchedulerApplication app = applications.get(
> 401 applicationAttemptId.getApplicationId());
> 402 return app == null ? null : app.getCurrentAppAttempt();
> 403 }
> {code}
> when old AM Crash, new AM and new appAttempt comes. The currentAttempt of
> app will be setted as the new appAttempt (see code3). So the code2 #402 will
> return the new appAttempt.
> if AM crashes at the head of allocate function(code1), bug won't happens due
> to ApplicationDoesNotExistInCacheException. AM crashed after code1,
> everything is also ok.
> We shoud add the check: whether the the getted appAttempt have the same id
> with given id.
> patch comes soon!
> {color:#ff}code3{color}
> {code:java}
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication.setCurrentAppAttempt(T
> currentAttempt){
> this.currentAttempt = currentAttempt;
> }
> {code}
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-9238) We get a wrong attempt by an appAttemptId when AM crash at some point
[
https://issues.apache.org/jira/browse/YARN-9238?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16770319#comment-16770319
]
lujie commented on YARN-9238:
-
Ping>
Further Simplify the unit test in the latest patch and hope for review.
> We get a wrong attempt by an appAttemptId when AM crash at some point
> --
>
> Key: YARN-9238
> URL: https://issues.apache.org/jira/browse/YARN-9238
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Assignee: lujie
>Priority: Critical
> Attachments: YARN-9238_1.patch, YARN-9238_2.patch, YARN-9238_3.patch,
> hadoop-test-resourcemanager-hadoop11.log
>
>
> We have found a data race that can make an odd situation.
> See
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService.OpportunisticAMSProcessor.allocate{color:#ff}:(code1){color}
> {code:java}
> // Allocate OPPORTUNISTIC containers.
> 171. SchedulerApplicationAttempt appAttempt =
> 172.((AbstractYarnScheduler)rmContext.getScheduler())
> 173. .getApplicationAttempt(appAttemptId);
> 174.
> 175. OpportunisticContainerContext oppCtx =
> 176. appAttempt.getOpportunisticContainerContext();
> 177. oppCtx.updateNodeList(getLeastLoadedNodes());
> {code}
> if we just crash the current AM(its attemptid is appattempt_0) just before
> code1#171, when code1#171~173 continue to execute to get the appAttempt by
> appattempt_0, the obtained appAttempt should represent the currenct AM. But
> we found that the obtained appAttempt represents the new AM and its
> attempid is appattempt_1. This obtained appAttempt has not init its oppCtx,
> so NPE happnes at line code1#177.
> {code:java}
> java.lang.NullPointerException
> at
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService$OpportunisticAMSProcessor.allocate(OpportunisticContainerAllocatorAMService.java:177)
> at
> org.apache.hadoop.yarn.server.resourcemanager.AMSProcessingChain.allocate(AMSProcessingChain.java:92)
> at
> org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService.allocate(ApplicationMasterService.java:424)
> at
> org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl.allocate(ApplicationMasterProtocolPBServiceImpl.java:60)
> at
> org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$2.callBlockingMethod(ApplicationMasterProtocol.java:99)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:530)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1070)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:943)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:878)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2830)
> {code}
> So why old appAttempt disappeares and why we use old appattempt_0 but get
> the new appAttempt
> We have found the reason. Below code({color:#ff}code2{color}) is the
> function body of getApplicationAttempt at code1#173
> {code:java}
> 399. public T getApplicationAttempt(ApplicationAttemptId
> applicationAttemptId) {
> 400 SchedulerApplication app = applications.get(
> 401 applicationAttemptId.getApplicationId());
> 402 return app == null ? null : app.getCurrentAppAttempt();
> 403 }
> {code}
> when old AM Crash, new AM and new appAttempt comes. The currentAttempt of
> app will be setted as the new appAttempt (see code3). So the code2 #402 will
> return the new appAttempt.
> if AM crashes at the head of allocate function(code1), bug won't happens due
> to ApplicationDoesNotExistInCacheException. AM crashed after code1,
> everything is also ok.
> We shoud add the check: whether the the getted appAttempt have the same id
> with given id.
> patch comes soon!
> {color:#ff}code3{color}
> {code:java}
> org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication.setCurrentAppAttempt(T
> currentAttempt){
> this.currentAttempt = currentAttempt;
> }
> {code}
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-9248) RMContainerImpl:Invalid event: ACQUIRED at KILLED
[
https://issues.apache.org/jira/browse/YARN-9248?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16770314#comment-16770314
]
lujie commented on YARN-9248:
-
ping ---> give the simplified patch and hope for review
> RMContainerImpl:Invalid event: ACQUIRED at KILLED
> -
>
> Key: YARN-9248
> URL: https://issues.apache.org/jira/browse/YARN-9248
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Assignee: lujie
>Priority: Major
> Attachments: YARN-9248_1.patch, YARN-9248_2.patch, YARN-9248_3.patch,
> YARN-9248_4.patch, YARN-9248_5.patch, YARN-9248_6.patch
>
>
> {code:java}
> 2019-01-29 11:46:53,596 ERROR
> org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl:
> Can't handle this event at current state
> org.apache.hadoop.yarn.state.InvalidStateTransitionException: Invalid event:
> ACQUIRED at KILLED
> at
> org.apache.hadoop.yarn.state.StateMachineFactory.doTransition(StateMachineFactory.java:305)
> at
> org.apache.hadoop.yarn.state.StateMachineFactory.access$500(StateMachineFactory.java:46)
> at
> org.apache.hadoop.yarn.state.StateMachineFactory$InternalStateMachine.doTransition(StateMachineFactory.java:487)
> at
> org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl.handle(RMContainerImpl.java:475)
> at
> org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl.handle(RMContainerImpl.java:67)
> at
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService.handleNewContainers(OpportunisticContainerAllocatorAMService.java:351)
> at
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService.access$100(OpportunisticContainerAllocatorAMService.java:94)
> at
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService$OpportunisticAMSProcessor.allocate(OpportunisticContainerAllocatorAMService.java:197)
> at
> org.apache.hadoop.yarn.server.resourcemanager.AMSProcessingChain.allocate(AMSProcessingChain.java:92)
> at
> org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService.allocate(ApplicationMasterService.java:424)
> at
> org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl.allocate(ApplicationMasterProtocolPBServiceImpl.java:60)
> at
> org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$2.callBlockingMethod(ApplicationMasterProtocol.java:99)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:530)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1070)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:943)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:878)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2830)
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Updated] (YARN-9248) RMContainerImpl:Invalid event: ACQUIRED at KILLED
[
https://issues.apache.org/jira/browse/YARN-9248?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lujie updated YARN-9248:
Attachment: YARN-9248_6.patch
> RMContainerImpl:Invalid event: ACQUIRED at KILLED
> -
>
> Key: YARN-9248
> URL: https://issues.apache.org/jira/browse/YARN-9248
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Assignee: lujie
>Priority: Major
> Attachments: YARN-9248_1.patch, YARN-9248_2.patch, YARN-9248_3.patch,
> YARN-9248_4.patch, YARN-9248_5.patch, YARN-9248_6.patch
>
>
> {code:java}
> 2019-01-29 11:46:53,596 ERROR
> org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl:
> Can't handle this event at current state
> org.apache.hadoop.yarn.state.InvalidStateTransitionException: Invalid event:
> ACQUIRED at KILLED
> at
> org.apache.hadoop.yarn.state.StateMachineFactory.doTransition(StateMachineFactory.java:305)
> at
> org.apache.hadoop.yarn.state.StateMachineFactory.access$500(StateMachineFactory.java:46)
> at
> org.apache.hadoop.yarn.state.StateMachineFactory$InternalStateMachine.doTransition(StateMachineFactory.java:487)
> at
> org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl.handle(RMContainerImpl.java:475)
> at
> org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl.handle(RMContainerImpl.java:67)
> at
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService.handleNewContainers(OpportunisticContainerAllocatorAMService.java:351)
> at
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService.access$100(OpportunisticContainerAllocatorAMService.java:94)
> at
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService$OpportunisticAMSProcessor.allocate(OpportunisticContainerAllocatorAMService.java:197)
> at
> org.apache.hadoop.yarn.server.resourcemanager.AMSProcessingChain.allocate(AMSProcessingChain.java:92)
> at
> org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService.allocate(ApplicationMasterService.java:424)
> at
> org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl.allocate(ApplicationMasterProtocolPBServiceImpl.java:60)
> at
> org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$2.callBlockingMethod(ApplicationMasterProtocol.java:99)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:530)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1070)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:943)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:878)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2830)
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-9248) RMContainerImpl:Invalid event: ACQUIRED at KILLED
[
https://issues.apache.org/jira/browse/YARN-9248?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16758226#comment-16758226
]
lujie commented on YARN-9248:
-
TestRMAppAttemptTransitions#testContainerRemovedBeforeAllocate fails will be
fixed in YARN-9262
> RMContainerImpl:Invalid event: ACQUIRED at KILLED
> -
>
> Key: YARN-9248
> URL: https://issues.apache.org/jira/browse/YARN-9248
> Project: Hadoop YARN
> Issue Type: Bug
>Reporter: lujie
>Assignee: lujie
>Priority: Major
> Attachments: YARN-9248_1.patch, YARN-9248_2.patch, YARN-9248_3.patch,
> YARN-9248_4.patch, YARN-9248_5.patch
>
>
> {code:java}
> 2019-01-29 11:46:53,596 ERROR
> org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl:
> Can't handle this event at current state
> org.apache.hadoop.yarn.state.InvalidStateTransitionException: Invalid event:
> ACQUIRED at KILLED
> at
> org.apache.hadoop.yarn.state.StateMachineFactory.doTransition(StateMachineFactory.java:305)
> at
> org.apache.hadoop.yarn.state.StateMachineFactory.access$500(StateMachineFactory.java:46)
> at
> org.apache.hadoop.yarn.state.StateMachineFactory$InternalStateMachine.doTransition(StateMachineFactory.java:487)
> at
> org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl.handle(RMContainerImpl.java:475)
> at
> org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl.handle(RMContainerImpl.java:67)
> at
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService.handleNewContainers(OpportunisticContainerAllocatorAMService.java:351)
> at
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService.access$100(OpportunisticContainerAllocatorAMService.java:94)
> at
> org.apache.hadoop.yarn.server.resourcemanager.OpportunisticContainerAllocatorAMService$OpportunisticAMSProcessor.allocate(OpportunisticContainerAllocatorAMService.java:197)
> at
> org.apache.hadoop.yarn.server.resourcemanager.AMSProcessingChain.allocate(AMSProcessingChain.java:92)
> at
> org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService.allocate(ApplicationMasterService.java:424)
> at
> org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl.allocate(ApplicationMasterProtocolPBServiceImpl.java:60)
> at
> org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$2.callBlockingMethod(ApplicationMasterProtocol.java:99)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:530)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1070)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:943)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:878)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2830)
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
