This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push: new c8b090d YARN-10828. Backport YARN-9789 to branch-3.2. Contributed by Tarun Parimi c8b090d is described below commit c8b090d9a492211e3be55e6adb79da57c6db304e Author: Szilard Nemeth <snem...@apache.org> AuthorDate: Thu Jun 24 20:20:23 2021 +0200 YARN-10828. Backport YARN-9789 to branch-3.2. Contributed by Tarun Parimi --- .../capacity/conf/LeveldbConfigurationStore.java | 17 +++++++++---- .../capacity/conf/ZKConfigurationStore.java | 20 ++++++++------- .../conf/TestLeveldbConfigurationStore.java | 29 ++++++++++++++++++++++ 3 files changed, 52 insertions(+), 14 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java index b347aba..1ff53a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java @@ -193,12 +193,14 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { @Override public void logMutation(LogMutation logMutation) throws IOException { - LinkedList<LogMutation> logs = deserLogMutations(db.get(bytes(LOG_KEY))); - logs.add(logMutation); - if (logs.size() > maxLogs) { - logs.removeFirst(); + if(maxLogs > 0) { + LinkedList<LogMutation> logs = deserLogMutations(db.get(bytes(LOG_KEY))); + logs.add(logMutation); + if (logs.size() > maxLogs) { + logs.removeFirst(); + } + db.put(bytes(LOG_KEY), serLogMutations(logs)); } - db.put(bytes(LOG_KEY), serLogMutations(logs)); pendingMutation = logMutation; } @@ -293,6 +295,11 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { return deserLogMutations(db.get(bytes(LOG_KEY))); } + @VisibleForTesting + protected DB getDB() { + return db; + } + @Override public void storeVersion() throws Exception { try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java index 229f680..3742c36 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java @@ -156,16 +156,18 @@ public class ZKConfigurationStore extends YarnConfigurationStore { @Override public void logMutation(LogMutation logMutation) throws Exception { - byte[] storedLogs = getZkData(logsPath); - LinkedList<LogMutation> logs = new LinkedList<>(); - if (storedLogs != null) { - logs = unsafeCast(deserializeObject(storedLogs)); - } - logs.add(logMutation); - if (logs.size() > maxLogs) { - logs.remove(logs.removeFirst()); + if(maxLogs > 0) { + byte[] storedLogs = getZkData(logsPath); + LinkedList<LogMutation> logs = new LinkedList<>(); + if (storedLogs != null) { + logs = unsafeCast(deserializeObject(storedLogs)); + } + logs.add(logMutation); + if (logs.size() > maxLogs) { + logs.remove(logs.removeFirst()); + } + safeSetZkData(logsPath, logs); } - safeSetZkData(logsPath, logs); pendingMutation = logMutation; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java index 4c92120..e88e7de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java @@ -18,7 +18,12 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; import org.apache.hadoop.yarn.server.records.Version; +import org.iq80.leveldb.DB; +import org.iq80.leveldb.DBIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.FileUtil; @@ -36,6 +41,7 @@ import org.junit.Test; import java.io.File; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; /** @@ -119,6 +125,29 @@ public class TestLeveldbConfigurationStore extends rm2.close(); } + @Test + public void testDisableAuditLogs() throws Exception { + conf.setLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, 0); + confStore.initialize(conf, schedConf, rmContext); + + prepareLogMutation("key1", "val1"); + + boolean logKeyPresent = false; + DB db = ((LeveldbConfigurationStore) confStore).getDB(); + DBIterator itr = db.iterator(); + itr.seekToFirst(); + while (itr.hasNext()) { + Map.Entry<byte[], byte[]> entry = itr.next(); + String key = new String(entry.getKey(), StandardCharsets.UTF_8); + if (key.equals("log")) { + logKeyPresent = true; + break; + } + } + assertFalse("Audit Log is not disabled", logKeyPresent); + confStore.close(); + } + @Override public YarnConfigurationStore createConfStore() { return new LeveldbConfigurationStore(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org