YARN-5549. AMLauncher#createAMContainerLaunchContext() should not log the command to be launched indiscriminately. (Daniel Templeton via rchiang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/378f624a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/378f624a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/378f624a Branch: refs/heads/YARN-3368 Commit: 378f624a392550770d1db33cb4cee3ef7d5facd4 Parents: 5a8c506 Author: Ray Chiang <rchi...@apache.org> Authored: Fri Sep 2 11:07:39 2016 -0700 Committer: Ray Chiang <rchi...@apache.org> Committed: Fri Sep 2 11:14:35 2016 -0700 ---------------------------------------------------------------------- .../hadoop/yarn/conf/YarnConfiguration.java | 12 +++++++++ .../src/main/resources/yarn-default.xml | 13 ++++++++++ .../resourcemanager/amlauncher/AMLauncher.java | 26 ++++++++++++++++---- 3 files changed, 46 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/378f624a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 46e3323..86e8a95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -534,6 +534,18 @@ public class YarnConfiguration extends Configuration { public static final int DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = 10; + /** + * The {@code AMLauncher.createAMContainerLaunchContext()} method will log the + * command being executed to the RM log if this property is true. Commands + * may contain sensitive information, such as application or service + * passwords, making logging the commands a security risk. In cases where + * the cluster may be running applications with such commands, this property + * should be set to false. Commands are only logged at the debug level. + */ + public static final String RM_AMLAUNCHER_LOG_COMMAND = + RM_PREFIX + "amlauncher.log.command"; + public static final boolean DEFAULT_RM_AMLAUNCHER_LOG_COMMAND = false; + //RM delegation token related keys public static final String RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY = RM_PREFIX + "delegation.key.update-interval"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/378f624a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index e956507..423b78b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -299,6 +299,19 @@ </property> <property> + <description> + The resource manager will log all commands being executed to the RM log + if this property is true. Commands may contain sensitive information, + such as application or service passwords, making logging the commands a + security risk. In cases where the cluster may be running applications with + such commands this property should be set to false. Commands are only + logged at the debug level. + </description> + <name>yarn.resourcemanager.amlauncher.log.command</name> + <value>false</value> + </property> + + <property> <description>The class to use as the resource scheduler.</description> <name>yarn.resourcemanager.scheduler.class</name> <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value> http://git-wip-us.apache.org/repos/asf/hadoop/blob/378f624a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 4aace2c..181463a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; /** * The launch of the AM itself. @@ -81,6 +82,7 @@ public class AMLauncher implements Runnable { private final AMLauncherEventType eventType; private final RMContext rmContext; private final Container masterContainer; + private final boolean logCommandLine; @SuppressWarnings("rawtypes") private final EventHandler handler; @@ -93,6 +95,9 @@ public class AMLauncher implements Runnable { this.rmContext = rmContext; this.handler = rmContext.getDispatcher().getEventHandler(); this.masterContainer = application.getMasterContainer(); + this.logCommandLine = + conf.getBoolean(YarnConfiguration.RM_AMLAUNCHER_LOG_COMMAND, + YarnConfiguration.DEFAULT_RM_AMLAUNCHER_LOG_COMMAND); } private void connect() throws IOException { @@ -188,11 +193,22 @@ public class AMLauncher implements Runnable { // Construct the actual Container ContainerLaunchContext container = applicationMasterContext.getAMContainerSpec(); - LOG.info("Command to launch container " - + containerID - + " : " - + StringUtils.arrayToString(container.getCommands().toArray( - new String[0]))); + + if (LOG.isDebugEnabled()) { + StringBuilder message = new StringBuilder("Command to launch container "); + + message.append(containerID).append(" : "); + + if (logCommandLine) { + message.append(Joiner.on(",").join(container.getCommands())); + } else { + message.append("<REDACTED> -- Set "); + message.append(YarnConfiguration.RM_AMLAUNCHER_LOG_COMMAND); + message.append(" to true to reenable command logging"); + } + + LOG.debug(message.toString()); + } // Populate the current queue name in the environment variable. setupQueueNameEnv(container, applicationMasterContext); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org