[
https://issues.apache.org/jira/browse/YARN-11350?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17643383#comment-17643383
]
ASF GitHub Bot commented on YARN-11350:
---------------------------------------
slfan1989 commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1039707916
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java:
##########
@@ -171,38 +203,293 @@ public void testMetricsInited() throws Exception {
MetricsRecords.assertMetric(record,
"UpdateReservationHomeSubClusterNumOps", expectOps);
}
- @Test(expected = NotImplementedException.class)
+ @Test
public void testStoreNewMasterKey() throws Exception {
- super.testStoreNewMasterKey();
+
+ // Manually create a DelegationKey,
+ // and call the interface storeNewMasterKey to write the data to zk.
+ DelegationKey key = new DelegationKey(1234, Time.now() + 60 * 60,
"keyBytes".getBytes());
+ RouterMasterKey paramRouterMasterKey =
RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ FederationStateStore stateStore = this.getStateStore();
+
+ assertTrue(stateStore instanceof ZookeeperFederationStateStore);
+
+ // Compare the data returned by the storeNewMasterKey
+ // interface with the data queried by zk, and ensure that the data is
consistent.
+ RouterMasterKeyRequest routerMasterKeyRequest =
+ RouterMasterKeyRequest.newInstance(paramRouterMasterKey);
+ RouterMasterKeyResponse response =
stateStore.storeNewMasterKey(routerMasterKeyRequest);
+ assertNotNull(response);
+ RouterMasterKey respRouterMasterKey = response.getRouterMasterKey();
+ assertNotNull(respRouterMasterKey);
+
+ // Get Data From zk.
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + key.getKeyId();
+ String nodePath = ZNODE_MASTER_KEY_PREFIX + nodeName;
+ RouterMasterKey zkRouterMasterKey = getRouterMasterKeyFromZK(nodePath);
+
+ assertNotNull(zkRouterMasterKey);
+ assertEquals(paramRouterMasterKey, respRouterMasterKey);
+ assertEquals(paramRouterMasterKey, zkRouterMasterKey);
+ assertEquals(zkRouterMasterKey, respRouterMasterKey);
}
- @Test(expected = NotImplementedException.class)
+ @Test
public void testGetMasterKeyByDelegationKey() throws YarnException,
IOException {
- super.testGetMasterKeyByDelegationKey();
+
+ // Manually create a DelegationKey,
+ // and call the interface storeNewMasterKey to write the data to zk.
+ DelegationKey key = new DelegationKey(5678, Time.now() + 60 * 60,
"keyBytes".getBytes());
+ RouterMasterKey paramRouterMasterKey =
RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ FederationStateStore stateStore = this.getStateStore();
+
+ assertTrue(stateStore instanceof ZookeeperFederationStateStore);
Review Comment:
Thank you very much for helping to review the code, I will fix it.
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java:
##########
@@ -171,38 +203,293 @@ public void testMetricsInited() throws Exception {
MetricsRecords.assertMetric(record,
"UpdateReservationHomeSubClusterNumOps", expectOps);
}
- @Test(expected = NotImplementedException.class)
+ @Test
public void testStoreNewMasterKey() throws Exception {
- super.testStoreNewMasterKey();
+
+ // Manually create a DelegationKey,
+ // and call the interface storeNewMasterKey to write the data to zk.
+ DelegationKey key = new DelegationKey(1234, Time.now() + 60 * 60,
"keyBytes".getBytes());
+ RouterMasterKey paramRouterMasterKey =
RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ FederationStateStore stateStore = this.getStateStore();
+
+ assertTrue(stateStore instanceof ZookeeperFederationStateStore);
+
+ // Compare the data returned by the storeNewMasterKey
+ // interface with the data queried by zk, and ensure that the data is
consistent.
+ RouterMasterKeyRequest routerMasterKeyRequest =
+ RouterMasterKeyRequest.newInstance(paramRouterMasterKey);
+ RouterMasterKeyResponse response =
stateStore.storeNewMasterKey(routerMasterKeyRequest);
+ assertNotNull(response);
+ RouterMasterKey respRouterMasterKey = response.getRouterMasterKey();
+ assertNotNull(respRouterMasterKey);
+
+ // Get Data From zk.
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + key.getKeyId();
+ String nodePath = ZNODE_MASTER_KEY_PREFIX + nodeName;
+ RouterMasterKey zkRouterMasterKey = getRouterMasterKeyFromZK(nodePath);
+
+ assertNotNull(zkRouterMasterKey);
+ assertEquals(paramRouterMasterKey, respRouterMasterKey);
+ assertEquals(paramRouterMasterKey, zkRouterMasterKey);
+ assertEquals(zkRouterMasterKey, respRouterMasterKey);
}
- @Test(expected = NotImplementedException.class)
+ @Test
public void testGetMasterKeyByDelegationKey() throws YarnException,
IOException {
- super.testGetMasterKeyByDelegationKey();
+
+ // Manually create a DelegationKey,
+ // and call the interface storeNewMasterKey to write the data to zk.
+ DelegationKey key = new DelegationKey(5678, Time.now() + 60 * 60,
"keyBytes".getBytes());
+ RouterMasterKey paramRouterMasterKey =
RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ FederationStateStore stateStore = this.getStateStore();
+
+ assertTrue(stateStore instanceof ZookeeperFederationStateStore);
+
+ // Call the getMasterKeyByDelegationKey interface of stateStore to get the
MasterKey data.
+ RouterMasterKeyRequest routerMasterKeyRequest =
+ RouterMasterKeyRequest.newInstance(paramRouterMasterKey);
+ RouterMasterKeyResponse response =
stateStore.storeNewMasterKey(routerMasterKeyRequest);
+ assertNotNull(response);
+
+ // Get Data From zk.
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + key.getKeyId();
+ String nodePath = ZNODE_MASTER_KEY_PREFIX + nodeName;
+ RouterMasterKey zkRouterMasterKey = getRouterMasterKeyFromZK(nodePath);
+
+ // Call the getMasterKeyByDelegationKey interface to get the returned
result.
+ // The zk data should be consistent with the returned data.
+ RouterMasterKeyResponse response1 =
+ stateStore.getMasterKeyByDelegationKey(routerMasterKeyRequest);
+ assertNotNull(response1);
+ RouterMasterKey respRouterMasterKey = response1.getRouterMasterKey();
+ assertEquals(paramRouterMasterKey, respRouterMasterKey);
+ assertEquals(paramRouterMasterKey, zkRouterMasterKey);
+ assertEquals(zkRouterMasterKey, respRouterMasterKey);
}
- @Test(expected = NotImplementedException.class)
+ @Test
public void testRemoveStoredMasterKey() throws YarnException, IOException {
- super.testRemoveStoredMasterKey();
+
+ // Manually create a DelegationKey,
+ // and call the interface storeNewMasterKey to write the data to zk.
+ DelegationKey key = new DelegationKey(2345, Time.now() + 60 * 60,
"keyBytes".getBytes());
+ RouterMasterKey paramRouterMasterKey =
RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ FederationStateStore stateStore = this.getStateStore();
+
+ assertTrue(stateStore instanceof ZookeeperFederationStateStore);
+
+ // We need to ensure that the returned result is not empty.
+ RouterMasterKeyRequest routerMasterKeyRequest =
+ RouterMasterKeyRequest.newInstance(paramRouterMasterKey);
+ RouterMasterKeyResponse response =
stateStore.storeNewMasterKey(routerMasterKeyRequest);
+ assertNotNull(response);
+
+ // We will check if delegationToken exists in zk.
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + key.getKeyId();
+ String nodePath = ZNODE_MASTER_KEY_PREFIX + nodeName;
+ assertTrue(isExists(nodePath));
+
+ // Call removeStoredMasterKey to remove the MasterKey data in zk.
+ RouterMasterKeyResponse response1 =
stateStore.removeStoredMasterKey(routerMasterKeyRequest);
+ assertNotNull(response1);
+ RouterMasterKey respRouterMasterKey = response1.getRouterMasterKey();
+ assertNotNull(respRouterMasterKey);
+ assertEquals(paramRouterMasterKey, respRouterMasterKey);
+
+ // We have removed the RouterMasterKey data from zk,
+ // the path should be empty at this point.
+ assertFalse(isExists(nodePath));
}
- @Test(expected = NotImplementedException.class)
- public void testStoreNewToken() throws IOException, YarnException {
- super.testStoreNewToken();
+ @Test
+ public void testStoreNewToken() throws YarnException, IOException {
+
+ // We manually generate the DelegationToken,
+ // and then call the StoreNewToken method to store the Token in zk.
+ RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(
+ new Text("owner2"), new Text("renewer2"), new Text("realuser2"));
+ FederationStateStore stateStore = this.getStateStore();
+ int seqNum = stateStore.incrementDelegationTokenSeqNum();
+ identifier.setSequenceNumber(seqNum);
+ Long renewDate = Time.now();
+
+ // Store new rm-token
+ RouterStoreToken paramStoreToken =
RouterStoreToken.newInstance(identifier, renewDate);
+ RouterRMTokenRequest request =
RouterRMTokenRequest.newInstance(paramStoreToken);
+ RouterRMTokenResponse routerRMTokenResponse =
stateStore.storeNewToken(request);
+ assertNotNull(routerRMTokenResponse);
+ RouterStoreToken respStoreToken =
routerRMTokenResponse.getRouterStoreToken();
+ assertNotNull(respStoreToken);
+
+ // Get delegationToken Path
+ String nodeName = ROUTER_RM_DELEGATION_TOKEN_PREFIX +
identifier.getSequenceNumber();
+ String nodePath = getNodePath(ZNODE_DT_PREFIX, nodeName);
+
+ // Check if the path exists, we expect the result to exist.
+ assertTrue(isExists(nodePath));
+
+ // Check whether the token (paramStoreToken)
+ // We generated is consistent with the data stored in zk.
+ // We expect data to be consistent.
+ RouterStoreToken zkRouterStoreToken = getStoreTokenFromZK(nodePath);
+ assertNotNull(zkRouterStoreToken);
+ assertEquals(paramStoreToken, zkRouterStoreToken);
+ assertEquals(respStoreToken, zkRouterStoreToken);
}
- @Test(expected = NotImplementedException.class)
- public void testUpdateStoredToken() throws IOException, YarnException {
- super.testUpdateStoredToken();
+ @Test
+ public void testUpdateStoredToken() throws YarnException, IOException {
+
+ // We manually generate the DelegationToken,
+ // and then call the StoreNewToken method to store the Token in zk.
+ RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(
+ new Text("owner2"), new Text("renewer2"), new Text("realuser2"));
+ FederationStateStore stateStore = this.getStateStore();
+ int seqNum = stateStore.incrementDelegationTokenSeqNum();
+ identifier.setSequenceNumber(seqNum);
+ Long renewDate = Time.now();
+
+ // Store new rm-token()
+ RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier,
renewDate);
+ RouterRMTokenRequest request =
RouterRMTokenRequest.newInstance(storeToken);
+ RouterRMTokenResponse routerRMTokenResponse =
stateStore.storeNewToken(request);
+ Assert.assertNotNull(routerRMTokenResponse);
+
+ // We are ready to update some data renewDate2 & sequenceNumber2
+ Long renewDate2 = Time.now();
+ int sequenceNumber2 = stateStore.incrementDelegationTokenSeqNum();
+ identifier.setSequenceNumber(sequenceNumber2);
+
+ // Update rm-token
+ RouterStoreToken paramStoreToken =
RouterStoreToken.newInstance(identifier, renewDate2);
+ RouterRMTokenRequest updateTokenRequest =
RouterRMTokenRequest.newInstance(paramStoreToken);
+ RouterRMTokenResponse updateTokenResponse =
stateStore.updateStoredToken(updateTokenRequest);
+ Assert.assertNotNull(updateTokenResponse);
+ RouterStoreToken updateTokenResp =
updateTokenResponse.getRouterStoreToken();
+
+ // Get delegationToken Path
+ String nodeName = ROUTER_RM_DELEGATION_TOKEN_PREFIX +
identifier.getSequenceNumber();
+ String nodePath = getNodePath(ZNODE_DT_PREFIX, nodeName);
+
+ // Check if the path exists, we expect the result to exist.
+ assertTrue(isExists(nodePath));
+
+ // Check whether the token (paramStoreToken)
+ // We generated is consistent with the data stored in zk.
+ // We expect data to be consistent.
+ RouterStoreToken zkRouterStoreToken = getStoreTokenFromZK(nodePath);
+ assertNotNull(zkRouterStoreToken);
+ assertEquals(paramStoreToken, zkRouterStoreToken);
+ assertEquals(updateTokenResp, zkRouterStoreToken);
}
- @Test(expected = NotImplementedException.class)
- public void testRemoveStoredToken() throws IOException, YarnException {
- super.testRemoveStoredToken();
+ @Test
+ public void testRemoveStoredToken() throws YarnException, IOException {
+
+ // We manually generate the DelegationToken,
+ // and then call the StoreNewToken method to store the Token in zk.
+ RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(
+ new Text("owner2"), new Text("renewer2"), new Text("realuser2"));
+ FederationStateStore stateStore = this.getStateStore();
+ int seqNum = stateStore.incrementDelegationTokenSeqNum();
+ identifier.setSequenceNumber(seqNum);
+ Long renewDate = Time.now();
+
+ // Store new rm-token
+ RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier,
renewDate);
+ RouterRMTokenRequest request =
RouterRMTokenRequest.newInstance(storeToken);
+ RouterRMTokenResponse routerRMTokenResponse =
stateStore.storeNewToken(request);
+ Assert.assertNotNull(routerRMTokenResponse);
Review Comment:
I will fix it.
> [Federation] Router Support DelegationToken With ZK
> ---------------------------------------------------
>
> Key: YARN-11350
> URL: https://issues.apache.org/jira/browse/YARN-11350
> Project: Hadoop YARN
> Issue Type: Sub-task
> Components: federation, router
> Affects Versions: 3.4.0
> Reporter: Shilun Fan
> Assignee: Shilun Fan
> Priority: Major
> Labels: pull-request-available
>
> [Federation] Router Support DelegationToken WithÂ
> ZookeeperFederationStateStore.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
