YARN-6354. LeveldbRMStateStore can parse invalid keys when recovering reservations. Contributed by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/318bfb01 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/318bfb01 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/318bfb01 Branch: refs/heads/YARN-2915 Commit: 318bfb01bc6793da09e32e9cc292eb63224b6ca2 Parents: 4d1fac5 Author: Eric Payne <epa...@apache.org> Authored: Fri Mar 31 12:30:35 2017 -0500 Committer: Eric Payne <epa...@apache.org> Committed: Fri Mar 31 12:30:35 2017 -0500 ---------------------------------------------------------------------- .../recovery/LeveldbRMStateStore.java | 8 ++++++++ .../recovery/TestLeveldbRMStateStore.java | 19 +++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/318bfb01/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java index 02f90dd..2ca53db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java @@ -214,6 +214,11 @@ public class LeveldbRMStateStore extends RMStateStore { return db == null; } + @VisibleForTesting + DB getDatabase() { + return db; + } + @Override protected Version loadVersion() throws Exception { Version version = null; @@ -284,6 +289,9 @@ public class LeveldbRMStateStore extends RMStateStore { while (iter.hasNext()) { Entry<byte[],byte[]> entry = iter.next(); String key = asString(entry.getKey()); + if (!key.startsWith(RM_RESERVATION_KEY_PREFIX)) { + break; + } String planReservationString = key.substring(RM_RESERVATION_KEY_PREFIX.length()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/318bfb01/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java index 4297e73..51adbe1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.fusesource.leveldbjni.JniDBFactory; import org.iq80.leveldb.DB; import org.junit.After; import org.junit.Before; @@ -125,17 +126,27 @@ public class TestLeveldbRMStateStore extends RMStateStoreTestBase { public void testCompactionCycle() throws Exception { final DB mockdb = mock(DB.class); conf.setLong(YarnConfiguration.RM_LEVELDB_COMPACTION_INTERVAL_SECS, 1); - LeveldbRMStateStore store = new LeveldbRMStateStore() { + stateStore = new LeveldbRMStateStore() { @Override protected DB openDatabase() throws Exception { return mockdb; } }; - store.init(conf); - store.start(); + stateStore.init(conf); + stateStore.start(); verify(mockdb, timeout(10000)).compactRange( (byte[]) isNull(), (byte[]) isNull()); - store.close(); + } + + @Test + public void testBadKeyIteration() throws Exception { + stateStore = new LeveldbRMStateStore(); + stateStore.init(conf); + stateStore.start(); + DB db = stateStore.getDatabase(); + // add an entry that appears at the end of the database when iterating + db.put(JniDBFactory.bytes("zzz"), JniDBFactory.bytes("z")); + stateStore.loadState(); } class LeveldbStateStoreTester implements RMStateStoreHelper { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
