YARN-7386. Duplicate Strings in various places in Yarn memory (mi...@cloudera.com via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a2c150a7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a2c150a7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a2c150a7 Branch: refs/heads/HDFS-7240 Commit: a2c150a7369cc629bbfaa2dfa3a8495b6f9c42e2 Parents: ba81366 Author: Robert Kanter <rkan...@apache.org> Authored: Thu Nov 9 12:07:46 2017 -0800 Committer: Robert Kanter <rkan...@apache.org> Committed: Thu Nov 9 12:12:52 2017 -0800 ---------------------------------------------------------------------- .../impl/pb/ContainerLaunchContextPBImpl.java | 16 ++++++++++++---- .../yarn/api/records/impl/pb/ContainerPBImpl.java | 2 +- .../server/resourcemanager/rmapp/RMAppImpl.java | 7 ++++--- .../rmapp/attempt/RMAppAttemptImpl.java | 3 ++- 4 files changed, 19 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2c150a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java index d722cc5..a9f2ee3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java @@ -27,6 +27,7 @@ import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerRetryContext; @@ -392,7 +393,8 @@ extends ContainerLaunchContext { this.environment = new HashMap<String, String>(); for (StringStringMapProto c : list) { - this.environment.put(c.getKey(), c.getValue()); + this.environment.put(StringInterner.weakIntern(c.getKey()), + StringInterner.weakIntern(c.getValue())); } } @@ -402,7 +404,10 @@ extends ContainerLaunchContext { return; initEnv(); this.environment.clear(); - this.environment.putAll(env); + for (Map.Entry<String, String> e : env.entrySet()) { + this.environment.put(StringInterner.weakIntern(e.getKey()), + StringInterner.weakIntern(e.getValue())); + } } private void addEnvToProto() { @@ -464,7 +469,7 @@ extends ContainerLaunchContext { for (ApplicationACLMapProto aclProto : list) { this.applicationACLS.put(ProtoUtils.convertFromProtoFormat(aclProto - .getAccessType()), aclProto.getAcl()); + .getAccessType()), StringInterner.weakIntern(aclProto.getAcl())); } } @@ -513,7 +518,10 @@ extends ContainerLaunchContext { return; initApplicationACLs(); this.applicationACLS.clear(); - this.applicationACLS.putAll(appACLs); + for (Map.Entry<ApplicationAccessType, String> e : appACLs.entrySet()) { + this.applicationACLS.put(e.getKey(), + StringInterner.weakIntern(e.getValue())); + } } public ContainerRetryContext getContainerRetryContext() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2c150a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java index b6e22d1..be84938 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java @@ -181,7 +181,7 @@ public class ContainerPBImpl extends Container { builder.clearNodeHttpAddress(); return; } - builder.setNodeHttpAddress(nodeHttpAddress); + builder.setNodeHttpAddress(nodeHttpAddress.intern()); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2c150a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index cfb8a74..ae5f6b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -46,6 +46,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.CallerContext; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -425,12 +426,12 @@ public class RMAppImpl implements RMApp, Recoverable { this.systemClock = SystemClock.getInstance(); this.applicationId = applicationId; - this.name = name; + this.name = StringInterner.weakIntern(name); this.rmContext = rmContext; this.dispatcher = rmContext.getDispatcher(); this.handler = dispatcher.getEventHandler(); this.conf = config; - this.user = user; + this.user = StringInterner.weakIntern(user); this.queue = queue; this.submissionContext = submissionContext; this.scheduler = scheduler; @@ -441,7 +442,7 @@ public class RMAppImpl implements RMApp, Recoverable { } else { this.startTime = startTime; } - this.applicationType = applicationType; + this.applicationType = StringInterner.weakIntern(applicationType); this.applicationTags = applicationTags; this.amReqs = amReqs; if (submissionContext.getPriority() != null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2c150a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index d3fbcb5..2d5c6a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -44,6 +44,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -1614,7 +1615,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { ClusterMetrics.getMetrics().addAMRegisterDelay(delay); RMAppAttemptRegistrationEvent registrationEvent = (RMAppAttemptRegistrationEvent) event; - appAttempt.host = registrationEvent.getHost(); + appAttempt.host = StringInterner.weakIntern(registrationEvent.getHost()); appAttempt.rpcPort = registrationEvent.getRpcport(); appAttempt.originalTrackingUrl = sanitizeTrackingUrl(registrationEvent.getTrackingurl()); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
