YARN-5601. Make the RM epoch base value configurable. Contributed by Subru Krishnan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09ab49db Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09ab49db Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09ab49db Branch: refs/heads/YARN-2915 Commit: 09ab49dbbaaa1926c6419ac9d5dae0d4b6a88ffc Parents: ee2b43e Author: Jian He <jia...@apache.org> Authored: Fri Sep 2 12:23:57 2016 +0800 Committer: Subru Krishnan <su...@apache.org> Committed: Tue Nov 22 18:37:54 2016 -0800 ---------------------------------------------------------------------- .../hadoop-yarn/dev-support/findbugs-exclude.xml | 5 ++++- .../java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 3 +++ .../apache/hadoop/yarn/conf/TestYarnConfigurationFields.java | 2 ++ .../hadoop/yarn/server/resourcemanager/ResourceManager.java | 7 +++++++ .../resourcemanager/recovery/FileSystemRMStateStore.java | 2 +- .../server/resourcemanager/recovery/LeveldbRMStateStore.java | 2 +- .../server/resourcemanager/recovery/MemoryRMStateStore.java | 1 + .../yarn/server/resourcemanager/recovery/RMStateStore.java | 4 ++++ .../yarn/server/resourcemanager/recovery/ZKRMStateStore.java | 2 +- .../resourcemanager/recovery/RMStateStoreTestBase.java | 8 +++++--- .../server/resourcemanager/recovery/TestFSRMStateStore.java | 1 + .../resourcemanager/recovery/TestLeveldbRMStateStore.java | 1 + .../server/resourcemanager/recovery/TestZKRMStateStore.java | 1 + 13 files changed, 32 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ab49db/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index c4424cf..0e94b70 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -297,7 +297,10 @@ </Match> <Match> <Class name="org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore" /> - <Field name="resourceManager"/> + <Or> + <Field name="resourceManager"/> + <Field name="baseEpoch"/> + </Or> <Bug pattern="IS2_INCONSISTENT_SYNC" /> </Match> <Match> http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ab49db/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index b928931..eeb8258 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -141,6 +141,9 @@ public class YarnConfiguration extends Configuration { public static final String RM_HOSTNAME = RM_PREFIX + "hostname"; + public static final String RM_EPOCH = RM_PREFIX + "epoch"; + public static final long DEFAULT_RM_EPOCH = 0L; + /** The address of the applications manager interface in the RM.*/ public static final String RM_ADDRESS = RM_PREFIX + "address"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ab49db/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java index 4c08e71..5958d4d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java @@ -99,6 +99,8 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase { .add(YarnConfiguration.FEDERATION_FAILOVER_ENABLED); configurationPropsToSkipCompare .add(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS); + configurationPropsToSkipCompare + .add(YarnConfiguration.RM_EPOCH); // Ignore blacklisting nodes for AM failures feature since it is still a // "work in progress" http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ab49db/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index b7bdde4..c7693e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -753,6 +753,13 @@ public class ResourceManager extends CompositeService implements Recoverable { LOG.error("Failed to load/recover state", e); throw e; } + } else { + if (HAUtil.isFederationEnabled(conf)) { + long epoch = conf.getLong(YarnConfiguration.RM_EPOCH, + YarnConfiguration.DEFAULT_RM_EPOCH); + rmContext.setEpoch(epoch); + LOG.info("Epoch set for Federation: " + epoch); + } } super.serviceStart(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ab49db/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java index b9a4374..a836ca8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java @@ -197,7 +197,7 @@ public class FileSystemRMStateStore extends RMStateStore { @Override public synchronized long getAndIncrementEpoch() throws Exception { Path epochNodePath = getNodePath(rootDirPath, EPOCH_NODE); - long currentEpoch = 0; + long currentEpoch = baseEpoch; FileStatus status = getFileStatusWithRetries(epochNodePath); if (status != null) { // load current epoch http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ab49db/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java index 02f90dd..dfe78e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java @@ -250,7 +250,7 @@ public class LeveldbRMStateStore extends RMStateStore { @Override public synchronized long getAndIncrementEpoch() throws Exception { - long currentEpoch = 0; + long currentEpoch = baseEpoch; byte[] dbKeyBytes = bytes(EPOCH_NODE); try { byte[] data = db.get(dbKeyBytes); http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ab49db/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java index 8b4ace3..5d7e1eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java @@ -83,6 +83,7 @@ public class MemoryRMStateStore extends RMStateStore { @Override public synchronized void initInternal(Configuration conf) { + epoch = baseEpoch; } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ab49db/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index 0fd346f..fd1ffc7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -99,6 +99,7 @@ public abstract class RMStateStore extends AbstractService { "ReservationSystemRoot"; protected static final String VERSION_NODE = "RMVersionNode"; protected static final String EPOCH_NODE = "EpochNode"; + protected long baseEpoch; protected ResourceManager resourceManager; private final ReadLock readLock; private final WriteLock writeLock; @@ -684,6 +685,9 @@ public abstract class RMStateStore extends AbstractService { dispatcher.register(RMStateStoreEventType.class, rmStateStoreEventHandler); dispatcher.setDrainEventsOnStop(); + // read the base epoch value from conf + baseEpoch = conf.getLong(YarnConfiguration.RM_EPOCH, + YarnConfiguration.DEFAULT_RM_EPOCH); initInternal(conf); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ab49db/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java index cf6380f..b9367b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java @@ -353,7 +353,7 @@ public class ZKRMStateStore extends RMStateStore { @Override public synchronized long getAndIncrementEpoch() throws Exception { String epochNodePath = getNodePath(zkRootNodePath, EPOCH_NODE); - long currentEpoch = 0; + long currentEpoch = baseEpoch; if (exists(epochNodePath)) { // load current epoch http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ab49db/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java index 8544c13..cbd7ae7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java @@ -91,6 +91,8 @@ public class RMStateStoreTestBase { public static final Log LOG = LogFactory.getLog(RMStateStoreTestBase.class); + protected final long epoch = 10L; + static class TestDispatcher implements Dispatcher, EventHandler<Event> { ApplicationAttemptId attemptId; @@ -558,13 +560,13 @@ public class RMStateStoreTestBase { store.setRMDispatcher(new TestDispatcher()); long firstTimeEpoch = store.getAndIncrementEpoch(); - Assert.assertEquals(0, firstTimeEpoch); + Assert.assertEquals(epoch, firstTimeEpoch); long secondTimeEpoch = store.getAndIncrementEpoch(); - Assert.assertEquals(1, secondTimeEpoch); + Assert.assertEquals(epoch + 1, secondTimeEpoch); long thirdTimeEpoch = store.getAndIncrementEpoch(); - Assert.assertEquals(2, thirdTimeEpoch); + Assert.assertEquals(epoch + 2, thirdTimeEpoch); } public void testAppDeletion(RMStateStoreHelper stateStoreHelper) http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ab49db/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java index 6f0d53f..8c968cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java @@ -116,6 +116,7 @@ public class TestFSRMStateStore extends RMStateStoreTestBase { conf.setInt(YarnConfiguration.FS_RM_STATE_STORE_NUM_RETRIES, 8); conf.setLong(YarnConfiguration.FS_RM_STATE_STORE_RETRY_INTERVAL_MS, 900L); + conf.setLong(YarnConfiguration.RM_EPOCH, epoch); if (adminCheckEnable) { conf.setBoolean( YarnConfiguration.YARN_INTERMEDIATE_DATA_ENCRYPTION, true); http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ab49db/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java index 4297e73..d6f77d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java @@ -81,6 +81,7 @@ public class TestLeveldbRMStateStore extends RMStateStoreTestBase { @Test(timeout = 60000) public void testEpoch() throws Exception { + conf.setLong(YarnConfiguration.RM_EPOCH, epoch); LeveldbStateStoreTester tester = new LeveldbStateStoreTester(); testEpoch(tester); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ab49db/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index f71cf25..22eadea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -155,6 +155,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { conf.set(YarnConfiguration.RM_ZK_ADDRESS, curatorTestingServer.getConnectString()); conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode); + conf.setLong(YarnConfiguration.RM_EPOCH, epoch); this.store = new TestZKRMStateStoreInternal(conf, workingZnode); return this.store; } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org