Repository: hadoop Updated Branches: refs/heads/branch-3.1 170f1040d -> 0460201c3
YARN-8128. Documenting the per-node per-app file limit in YARN log aggregation. Contributed by Xuan Gong. (cherry picked from commit 328f0847e3d98cad8c368d57499f31081c153237) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0460201c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0460201c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0460201c Branch: refs/heads/branch-3.1 Commit: 0460201c3a505144202bd899ceb1586beb6e6ab6 Parents: 170f104 Author: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vino...@apache.org> Authored: Thu May 17 12:47:10 2018 -0700 Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vino...@apache.org> Committed: Thu May 17 13:09:09 2018 -0700 ---------------------------------------------------------------------- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 10 ++++++++++ .../LogAggregationFileController.java | 17 ++++++----------- .../src/main/resources/yarn-default.xml | 9 +++++++++ 3 files changed, 25 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0460201c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 208f85c..7683821 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1357,6 +1357,16 @@ public class YarnConfiguration extends Configuration { NM_PREFIX + "log-aggregation.roll-monitoring-interval-seconds"; public static final long DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = -1; + + /** + * Define how many aggregated log files per application per NM we can have + * in remote file system. + */ + public static final String NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP + = NM_PREFIX + "log-aggregation.num-log-files-per-app"; + public static final int + DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP = 30; + /** * Number of threads used in log cleanup. Only applicable if Log aggregation * is disabled http://git-wip-us.apache.org/repos/asf/hadoop/blob/0460201c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java index aeef574..5ac89e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java @@ -95,13 +95,6 @@ public abstract class LogAggregationFileController { protected static final FsPermission APP_LOG_FILE_UMASK = FsPermission .createImmutable((short) (0640 ^ 0777)); - // This is temporary solution. The configuration will be deleted once - // we find a more scalable method to only write a single log file per LRS. - private static final String NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP - = YarnConfiguration.NM_PREFIX + "log-aggregation.num-log-files-per-app"; - private static final int - DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP = 30; - // This is temporary solution. The configuration will be deleted once we have // the FileSystem API to check whether append operation is supported or not. public static final String LOG_AGGREGATION_FS_SUPPORT_APPEND @@ -122,12 +115,14 @@ public abstract class LogAggregationFileController { */ public void initialize(Configuration conf, String controllerName) { this.conf = conf; - int configuredRentionSize = - conf.getInt(NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP, - DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP); + int configuredRentionSize = conf.getInt( + YarnConfiguration.NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP, + YarnConfiguration + .DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP); if (configuredRentionSize <= 0) { this.retentionSize = - DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP; + YarnConfiguration + .DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP; } else { this.retentionSize = configuredRentionSize; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0460201c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 58f32bf..25c24d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3012,6 +3012,15 @@ </property> <property> + <description>Define how many aggregated log files per application per NM + we can have in remote file system. By default, the total number of + aggregated log files per application per NM is 30. + </description> + <name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name> + <value>30</value> + </property> + + <property> <description> Enable/disable intermediate-data encryption at YARN level. For now, this only is used by the FileSystemRMStateStore to setup right --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org