This is an automated email from the ASF dual-hosted git repository.
jbrennan pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new 6a58e62 YARN-8558. NM recovery level db not cleaned up properly on
container finish. Contributed by Bibin A Chundatt.
6a58e62 is described below
commit 6a58e6293fb586fc6d98dfd959bd3b760837207b
Author: Jim Brennan <[email protected]>
AuthorDate: Tue Nov 17 17:36:22 2020 +0000
YARN-8558. NM recovery level db not cleaned up properly on container
finish. Contributed by Bibin A Chundatt.
---
.../nodemanager/recovery/NMLeveldbStateStoreService.java | 13 +++++++++----
.../recovery/TestNMLeveldbStateStoreService.java | 7 +++++++
2 files changed, 16 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
index bcdcc4b..9e5a361 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
@@ -145,9 +145,9 @@ public class NMLeveldbStateStoreService extends
NMStateStoreService {
NM_TOKENS_KEY_PREFIX + PREV_MASTER_KEY_SUFFIX;
private static final String CONTAINER_TOKENS_KEY_PREFIX =
"ContainerTokens/";
- private static final String CONTAINER_TOKENS_CURRENT_MASTER_KEY =
+ private static final String CONTAINER_TOKEN_SECRETMANAGER_CURRENT_MASTER_KEY
=
CONTAINER_TOKENS_KEY_PREFIX + CURRENT_MASTER_KEY_SUFFIX;
- private static final String CONTAINER_TOKENS_PREV_MASTER_KEY =
+ private static final String CONTAINER_TOKEN_SECRETMANAGER_PREV_MASTER_KEY =
CONTAINER_TOKENS_KEY_PREFIX + PREV_MASTER_KEY_SUFFIX;
private static final String LOG_DELETER_KEY_PREFIX = "LogDeleters/";
@@ -697,6 +697,11 @@ public class NMLeveldbStateStoreService extends
NMStateStoreService {
batch.delete(bytes(keyPrefix + CONTAINER_KILLED_KEY_SUFFIX));
batch.delete(bytes(keyPrefix + CONTAINER_EXIT_CODE_KEY_SUFFIX));
batch.delete(bytes(keyPrefix + CONTAINER_UPDATE_TOKEN_SUFFIX));
+ batch.delete(bytes(keyPrefix + CONTAINER_START_TIME_KEY_SUFFIX));
+ batch.delete(bytes(keyPrefix + CONTAINER_LOG_DIR_KEY_SUFFIX));
+ batch.delete(bytes(keyPrefix + CONTAINER_VERSION_KEY_SUFFIX));
+ batch.delete(bytes(keyPrefix + CONTAINER_REMAIN_RETRIES_KEY_SUFFIX));
+ batch.delete(bytes(keyPrefix + CONTAINER_WORK_DIR_KEY_SUFFIX));
List<String> unknownKeysForContainer = containerUnknownKeySuffixes
.removeAll(containerId);
for (String unknownKeySuffix : unknownKeysForContainer) {
@@ -1319,13 +1324,13 @@ public class NMLeveldbStateStoreService extends
NMStateStoreService {
@Override
public void storeContainerTokenCurrentMasterKey(MasterKey key)
throws IOException {
- storeMasterKey(CONTAINER_TOKENS_CURRENT_MASTER_KEY, key);
+ storeMasterKey(CONTAINER_TOKEN_SECRETMANAGER_CURRENT_MASTER_KEY, key);
}
@Override
public void storeContainerTokenPreviousMasterKey(MasterKey key)
throws IOException {
- storeMasterKey(CONTAINER_TOKENS_PREV_MASTER_KEY, key);
+ storeMasterKey(CONTAINER_TOKEN_SECRETMANAGER_PREV_MASTER_KEY, key);
}
@Override
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
index c64d48f..ccc1bf7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
@@ -28,7 +28,9 @@ import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.isNull;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -488,6 +490,11 @@ public class TestNMLeveldbStateStoreService {
recoveredContainers =
loadContainersState(stateStore.getContainerStateIterator());
assertTrue(recoveredContainers.isEmpty());
+ // recover again to check remove clears all containers
+ restartStateStore();
+ NMStateStoreService nmStoreSpy = spy(stateStore);
+ loadContainersState(nmStoreSpy.getContainerStateIterator());
+ verify(nmStoreSpy,times(0)).removeContainer(any(ContainerId.class));
}
private StartContainerRequest createContainerRequest(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
