Jiandan Yang created YARN-8664: ----------------------------------- Summary: ApplicationMasterProtocolPBServiceImpl#allocate throw NPE when NM losting Key: YARN-8664 URL: https://issues.apache.org/jira/browse/YARN-8664 Project: Hadoop YARN Issue Type: Bug Components: resourcemanager Affects Versions: 2.8.2 Environment:
Reporter: Jiandan Yang Assignee: Jiandan Yang ResourceManager logs about exception is: {code:java} 2018-08-09 00:52:30,746 WARN [IPC Server handler 5 on 8030] org.apache.hadoop.ipc.Server: IPC Server handler 5 on 8030, call Call#305638 Retry#0 org.apache.hadoop.yarn.api.ApplicationMasterProtocolPB.allocate from 11.13.73.101:51083 java.lang.NullPointerException at org.apache.hadoop.yarn.proto.YarnProtos$ResourceProto.isInitialized(YarnProtos.java:6402) at org.apache.hadoop.yarn.proto.YarnProtos$ResourceProto$Builder.build(YarnProtos.java:6642) at org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl.mergeLocalToProto(ResourcePBImpl.java:254) at org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl.getProto(ResourcePBImpl.java:61) at org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl.convertToProtoFormat(NodeReportPBImpl.java:313) at org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl.mergeLocalToBuilder(NodeReportPBImpl.java:264) at org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl.mergeLocalToProto(NodeReportPBImpl.java:287) at org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl.getProto(NodeReportPBImpl.java:224) at org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateResponsePBImpl.convertToProtoFormat(AllocateResponsePBImpl.java:714) at org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateResponsePBImpl.access$400(AllocateResponsePBImpl.java:69) at org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateResponsePBImpl$6$1.next(AllocateResponsePBImpl.java:680) at org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateResponsePBImpl$6$1.next(AllocateResponsePBImpl.java:669) at com.google.protobuf.AbstractMessageLite$Builder.checkForNullValues(AbstractMessageLite.java:336) at com.google.protobuf.AbstractMessageLite$Builder.addAll(AbstractMessageLite.java:323) at org.apache.hadoop.yarn.proto.YarnServiceProtos$AllocateResponseProto$Builder.addAllUpdatedNodes(YarnServiceProtos.java:12846) at org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateResponsePBImpl.mergeLocalToBuilder(AllocateResponsePBImpl.java:145) at org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateResponsePBImpl.mergeLocalToProto(AllocateResponsePBImpl.java:176) at org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateResponsePBImpl.getProto(AllocateResponsePBImpl.java:97) at org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl.allocate(ApplicationMasterProtocolPBServiceImpl.java:61) at org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$2.callBlockingMethod(ApplicationMasterProtocol.java:99) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:447) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:989) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:846) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:789) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1804) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2457) {code} ApplicationMasterService#allocate will call AllocateResponse#setUpdatedNodes when NM losting, and AllocateResponse#getProto will call ResourceBPImpl#getProto to transform NodeReportPBImpl#capacity into format of PB . Because ResourcePBImpl is not thread safe and multiple AM will call allocate at the same time, ResourcePBImpl#getProto may throw NullPointerException or UnsupportedOperationException. I wrote a test code which can reproduce exception. {code:java} @Test public void testResource1() throws InterruptedException { ResourcePBImpl resource = (ResourcePBImpl) Resource.newInstance(1, 1); for(long i=0;i<100;i++) { resource.setResourceInformation("r" + i, ResourceInformation.newInstance("r" + i, i)); } for (int i =0;i<10;i++ ) { Thread thread = new PBThread(resource); thread.setName("t"+i); thread.start(); } Thread.sleep(100000000); } class PBThread extends Thread { ResourcePBImpl resourcePB; public PBThread(ResourcePBImpl resourcePB) { this.resourcePB = resourcePB; } @Override public void run() { while(true) { this.resourcePB.getProto(); } } } {code} -- This message was sent by Atlassian JIRA (v7.6.3#76005) --------------------------------------------------------------------- To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org