Repository: hadoop Updated Branches: refs/heads/branch-2.8 cb7953377 -> 35f9cfda6
YARN-4773. Log aggregation performs extraneous filesystem operations when rolling log aggregation is disabled. Contributed by Jun Gong (cherry picked from commit 948b75807068c304ffe789e32f2b850c0d653e0a) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/35f9cfda Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/35f9cfda Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/35f9cfda Branch: refs/heads/branch-2.8 Commit: 35f9cfda61e989de437be4b8d86ae675ee7bdec2 Parents: cb79533 Author: Jason Lowe <jl...@apache.org> Authored: Mon Mar 28 23:00:56 2016 +0000 Committer: Jason Lowe <jl...@apache.org> Committed: Mon Mar 28 23:03:16 2016 +0000 ---------------------------------------------------------------------- .../logaggregation/AppLogAggregatorImpl.java | 14 ++++++++++---- .../logaggregation/TestLogAggregationService.java | 12 +++++++----- 2 files changed, 17 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/35f9cfda/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index da7fc14..fed4a3b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -124,11 +124,11 @@ public class AppLogAggregatorImpl implements AppLogAggregator { private final long rollingMonitorInterval; private final boolean logAggregationInRolling; private final NodeId nodeId; - // This variable is only for testing - private final AtomicBoolean waiting = new AtomicBoolean(false); - // This variable is only for testing + // These variables are only for testing + private final AtomicBoolean waiting = new AtomicBoolean(false); private int logAggregationTimes = 0; + private int cleanupOldLogTimes = 0; private boolean renameTemporaryLogFileFailed = false; @@ -365,8 +365,9 @@ public class AppLogAggregatorImpl implements AppLogAggregator { // Before upload logs, make sure the number of existing logs // is smaller than the configured NM log aggregation retention size. - if (uploadedLogsInThisCycle) { + if (uploadedLogsInThisCycle && logAggregationInRolling) { cleanOldLogs(); + cleanupOldLogTimes++; } if (writer != null) { @@ -689,4 +690,9 @@ public class AppLogAggregatorImpl implements AppLogAggregator { public int getLogAggregationTimes() { return this.logAggregationTimes; } + + @VisibleForTesting + int getCleanupOldLogTimes() { + return this.cleanupOldLogTimes; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/35f9cfda/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 6239ded..bdb4d1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -2133,7 +2133,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest { Records.newRecord(LogAggregationContext.class); logAggregationContext.setLogAggregationPolicyClassName( FailedOrKilledContainerLogAggregationPolicy.class.getName()); - verifySkipUnnecessaryNNOperations(logAggregationContext, 0, 2); + verifySkipUnnecessaryNNOperations(logAggregationContext, 0, 2, 0); } @Test (timeout = 20000) @@ -2147,13 +2147,13 @@ public class TestLogAggregationService extends BaseContainerManagerTest { AMOnlyLogAggregationPolicy.class.getName()); contextWithAMOnly.setRolledLogsIncludePattern("sys*"); contextWithAMOnly.setRolledLogsExcludePattern("std_final"); - verifySkipUnnecessaryNNOperations(contextWithAMOnly, 1, 4); + verifySkipUnnecessaryNNOperations(contextWithAMOnly, 1, 4, 1); } private void verifySkipUnnecessaryNNOperations( LogAggregationContext logAggregationContext, - int expectedLogAggregationTimes, int expectedAggregationReportNum) - throws Exception { + int expectedLogAggregationTimes, int expectedAggregationReportNum, + int expectedCleanupOldLogsTimes) throws Exception { LogAggregationService logAggregationService = new LogAggregationService( dispatcher, this.context, this.delSrvc, super.dirsHandler); logAggregationService.init(this.conf); @@ -2164,7 +2164,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest { null, this.acls, logAggregationContext)); // Container finishes - String[] logFiles = new String[] { "stdout" }; + String[] logFiles = new String[] { "sysout" }; finishContainer(appId, logAggregationService, ContainerType.APPLICATION_MASTER, 1, 0, logFiles); AppLogAggregatorImpl aggregator = @@ -2184,6 +2184,8 @@ public class TestLogAggregationService extends BaseContainerManagerTest { aggregator.getLogAggregationTimes()); assertEquals(expectedAggregationReportNum, this.context.getLogAggregationStatusForApps().size()); + assertEquals(expectedCleanupOldLogsTimes, + aggregator.getCleanupOldLogTimes()); } private int numOfLogsAvailable(LogAggregationService logAggregationService,