[jira] [Commented] (YARN-11349) [Federation] Router Support DelegationToken With SQL
[
https://issues.apache.org/jira/browse/YARN-11349?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17641004#comment-17641004
]
ASF GitHub Bot commented on YARN-11349:
---
slfan1989 commented on code in PR #5169:
URL: https://github.com/apache/hadoop/pull/5169#discussion_r1035403985
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java:
##
@@ -164,6 +170,40 @@ public void setRenewDate(Long renewDate) {
this.builder.setRenewDate(renewDate);
}
+ @Override
+ public byte[] toByteArray() throws IOException {
+return builder.build().toByteArray();
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+builder.mergeFrom((DataInputStream) in);
+ }
+
+ @Override
+ public String getTokenInfo() {
+RouterStoreTokenProtoOrBuilder p = viaProto ? proto : builder;
+if (this.tokenInfo != null) {
+ return this.tokenInfo;
+}
+if (!p.hasTokenInfo()) {
+ return null;
+}
+this.tokenInfo = p.getTokenInfo();
+return this.tokenInfo;
+ }
+
+ @Override
+ protected void setTokenInfo(String tokenInfo) {
+maybeInitBuilder();
+if(tokenInfo == null) {
Review Comment:
i will fix it.
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationRouterRMTokenInputValidator.java:
##
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.utils;
+
+import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier;
+import
org.apache.hadoop.yarn.server.federation.store.exception.FederationStateStoreInvalidInputException;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey;
+import
org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest;
+import
org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public final class FederationRouterRMTokenInputValidator {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FederationRouterRMTokenInputValidator.class);
+
+ private FederationRouterRMTokenInputValidator() {
+ }
+
+ /**
+ * We will check with the RouterRMTokenRequest{@link RouterRMTokenRequest}
+ * to ensure that the request object is not empty and that the
RouterStoreToken is not empty.
+ *
+ * @param request RouterRMTokenRequest Request.
+ * @throws FederationStateStoreInvalidInputException if the request is
invalid.
+ */
+ public static void validate(RouterRMTokenRequest request)
+ throws FederationStateStoreInvalidInputException {
+
+if (request == null) {
+ String message = "Missing RouterRMToken Request."
+ + " Please try again by specifying a router rm token information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+}
+
+RouterStoreToken storeToken = request.getRouterStoreToken();
+if (storeToken == null) {
+ String message = "Missing RouterStoreToken."
+ + " Please try again by specifying a router rm token information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+}
+
+try {
+ YARNDelegationTokenIdentifier identifier =
storeToken.getTokenIdentifier();
+ if (identifier == null) {
+String message = "Missing YARNDelegationTokenIdentifier."
++ " Please try again by specifying a router rm token information.";
+LOG.warn(message);
+throw new FederationStateStoreInvalidInputException(message);
+ }
+} catch (Exception e) {
+ throw new FederationStateStoreInvalidInputException(e);
+}
+ }
+
+ /**
+ * We will check with the RouterMasterKeyRequest{@link
RouterMasterKeyRequest}
+ * to ensure that the request
[jira] [Commented] (YARN-11349) [Federation] Router Support DelegationToken With SQL
[
https://issues.apache.org/jira/browse/YARN-11349?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17641003#comment-17641003
]
ASF GitHub Bot commented on YARN-11349:
---
slfan1989 commented on code in PR #5169:
URL: https://github.com/apache/hadoop/pull/5169#discussion_r1035403875
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java:
##
@@ -1353,45 +1386,560 @@ public Connection getConn() {
return conn;
}
+ /**
+ * SQLFederationStateStore Supports Store New MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an
abstraction for DelegationKey.
+ * @return routerMasterKeyResponse, the response contains the
RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest
request)
throws YarnException, IOException {
-throw new NotImplementedException("Code is not implemented");
+
+// Step1: Verify parameters to ensure that key fields are not empty.
+FederationRouterRMTokenInputValidator.validate(request);
+
+// Step2: Parse the parameters and serialize the DelegationKey as a string.
+DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+int keyId = delegationKey.getKeyId();
+String delegationKeyStr = encodeWritable(delegationKey);
+CallableStatement cstmt = null;
+
+// Step3. store data in database.
+try {
+ // Call procedure
+ cstmt = getCallableStatement(CALL_SP_ADD_MASTERKEY);
+
+ // Set the parameters for the stored procedure
+ // 1）IN keyId_IN bigint
+ cstmt.setInt("keyId_IN", keyId);
+ // 2）IN masterKey_IN varbinary(1024)
+ cstmt.setString("masterKey_IN", delegationKeyStr);
+ // 3) OUT rowCount_OUT int
+ cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER);
+
+ long startTime = clock.getTime();
+ cstmt.executeUpdate();
+ long stopTime = clock.getTime();
+
+ // Get rowCount
+ int rowCount = cstmt.getInt("rowCount_OUT");
+
+ // We hope that 1 record can be written to the database.
+ // If the number of records is not 1, it means that the data was written
incorrectly.
+ if (rowCount != 1) {
+FederationStateStoreUtils.logAndThrowStoreException(LOG,
+"Wrong behavior during the insertion of masterKey, keyId = %s. " +
+"please check the records of the database.",
String.valueOf(keyId));
+ }
+ FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime -
startTime);
+} catch (SQLException e) {
+ FederationStateStoreClientMetrics.failedStateStoreCall();
+ FederationStateStoreUtils.logAndThrowRetriableException(e, LOG,
+ "Unable to insert the newly masterKey, keyId = %s.",
String.valueOf(keyId));
+} finally {
+ // Return to the pool the CallableStatement
+ FederationStateStoreUtils.returnToPool(LOG, cstmt);
+}
+
+// Step4. Query Data from the database and return the result.
+return getMasterKeyByDelegationKey(request);
}
+ /**
+ * SQLFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an
abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the
RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest
request)
throws YarnException, IOException {
-throw new NotImplementedException("Code is not implemented");
+// Step1: Verify parameters to ensure that key fields are not empty.
+FederationRouterRMTokenInputValidator.validate(request);
+
+// Step2: Parse parameters and get KeyId.
+RouterMasterKey paramMasterKey = request.getRouterMasterKey();
+int paramKeyId = paramMasterKey.getKeyId();
+CallableStatement cstmt = null;
+
+// Step3. Clear data from database.
+try {
+ // Defined the sp_deleteMasterKey procedure
+ // This procedure requires 1 input parameters, 1 output parameters
+ // Input parameters
+ // 1）IN keyId_IN int
+ // Output parameters
+ // 2）OUT rowCount_OUT int
+ cstmt = getCallableStatement(CALL_SP_DELETE_MASTERKEY);
+
+ // Set the parameters for the stored procedure
+ // 1）IN keyId_IN int
+ cstmt.setInt("keyId_IN", paramKeyId);
+ // 2）OUT rowCount_OUT int
+ cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER);
+
+ // Execute the query
+ long startTime = clock.getTime();
+ cstmt.executeUpdate();
+
[jira] [Commented] (YARN-11349) [Federation] Router Support DelegationToken With SQL
[
https://issues.apache.org/jira/browse/YARN-11349?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17640990#comment-17640990
]
ASF GitHub Bot commented on YARN-11349:
---
hadoop-yetus commented on PR #5169:
URL: https://github.com/apache/hadoop/pull/5169#issuecomment-1331448445
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 1m 18s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available.
|
| +0 :ok: | buf | 0m 0s | | buf was not available. |
| +0 :ok: | buf | 0m 0s | | buf was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 2 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 14m 57s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 37m 32s | | trunk passed |
| +1 :green_heart: | compile | 42m 33s | | trunk passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| -1 :x: | compile | 14m 20s |
[/branch-compile-root-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5169/5/artifact/out/branch-compile-root-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt)
| root in trunk failed with JDK Private
Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| -0 :warning: | checkstyle | 1m 5s |
[/buildtool-branch-checkstyle-root.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5169/5/artifact/out/buildtool-branch-checkstyle-root.txt)
| The patch fails to run checkstyle in root |
| -1 :x: | mvnsite | 1m 7s |
[/branch-mvnsite-hadoop-common-project_hadoop-common.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5169/5/artifact/out/branch-mvnsite-hadoop-common-project_hadoop-common.txt)
| hadoop-common in trunk failed. |
| -1 :x: | mvnsite | 1m 7s |
[/branch-mvnsite-hadoop-yarn-project_hadoop-yarn.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5169/5/artifact/out/branch-mvnsite-hadoop-yarn-project_hadoop-yarn.txt)
| hadoop-yarn in trunk failed. |
| -1 :x: | mvnsite | 1m 14s |
[/branch-mvnsite-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5169/5/artifact/out/branch-mvnsite-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common.txt)
| hadoop-yarn-server-common in trunk failed. |
| -1 :x: | mvnsite | 1m 8s |
[/branch-mvnsite-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-router.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5169/5/artifact/out/branch-mvnsite-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-router.txt)
| hadoop-yarn-server-router in trunk failed. |
| -1 :x: | javadoc | 1m 12s |
[/branch-javadoc-hadoop-common-project_hadoop-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5169/5/artifact/out/branch-javadoc-hadoop-common-project_hadoop-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt)
| hadoop-common in trunk failed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| -1 :x: | javadoc | 1m 13s |
[/branch-javadoc-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5169/5/artifact/out/branch-javadoc-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt)
| hadoop-yarn in trunk failed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| +1 :green_heart: | javadoc | 7m 52s | | trunk passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| -1 :x: | spotbugs | 2m 45s |
[/branch-spotbugs-hadoop-yarn-project_hadoop-yarn.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5169/5/artifact/out/branch-spotbugs-hadoop-yarn-project_hadoop-yarn.txt)
| hadoop-yarn in trunk failed. |
| +1 :green_heart: | shadedclient | 26m 4s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 27s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 7m 7s | | the patch passed |
| +1 :green_heart: | compile | 24m 48s | | the patch passed with JDK
[jira] [Commented] (YARN-6538) Inter Queue preemption is not happening when DRF is configured
[
https://issues.apache.org/jira/browse/YARN-6538?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17640938#comment-17640938
]
ASF GitHub Bot commented on YARN-6538:
--
hadoop-yetus commented on PR #5174:
URL: https://github.com/apache/hadoop/pull/5174#issuecomment-1331314820
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 0m 50s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 1 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 17m 54s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 25m 49s | | trunk passed |
| +1 :green_heart: | compile | 9m 58s | | trunk passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 8m 51s | | trunk passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 2m 5s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 55s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 33s | | trunk passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 2m 47s | | trunk passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 4m 36s | | trunk passed |
| +1 :green_heart: | shadedclient | 22m 24s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 32s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 35s | | the patch passed |
| +1 :green_heart: | compile | 9m 12s | | the patch passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| -1 :x: | javac | 9m 12s |
[/results-compile-javac-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5174/1/artifact/out/results-compile-javac-hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt)
|
hadoop-yarn-project_hadoop-yarn-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04
with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 generated 3 new + 721
unchanged - 0 fixed = 724 total (was 721) |
| +1 :green_heart: | compile | 8m 42s | | the patch passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| -1 :x: | javac | 8m 42s |
[/results-compile-javac-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5174/1/artifact/out/results-compile-javac-hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt)
|
hadoop-yarn-project_hadoop-yarn-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07
with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 generated 3 new
+ 637 unchanged - 0 fixed = 640 total (was 637) |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| -0 :warning: | checkstyle | 1m 51s |
[/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5174/1/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn.txt)
| hadoop-yarn-project/hadoop-yarn: The patch generated 1 new + 8 unchanged -
0 fixed = 9 total (was 8) |
| +1 :green_heart: | mvnsite | 2m 27s | | the patch passed |
| +1 :green_heart: | javadoc | 2m 16s | | the patch passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 2m 9s | | the patch passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 4m 55s | | the patch passed |
| +1 :green_heart: | shadedclient | 22m 48s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | unit | 5m 56s | | hadoop-yarn-common in the patch
passed. |
| +1 :green_heart: | unit | 100m 1s | |
hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | asflicense | 1m 16s | | The patch does not
generate ASF License warnings. |
| | | 266m 6s | | |
| Subsystem | Report/Notes |
[jira] [Commented] (YARN-10946) AbstractCSQueue: Create separate class for constructing Queue API objects
[
https://issues.apache.org/jira/browse/YARN-10946?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17640877#comment-17640877
]
ASF GitHub Bot commented on YARN-10946:
---
hadoop-yetus commented on PR #5173:
URL: https://github.com/apache/hadoop/pull/5173#issuecomment-1331073970
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 0m 51s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| -1 :x: | test4tests | 0m 0s | | The patch doesn't appear to include
any new or modified tests. Please justify why no new tests are needed for this
patch. Also please list what manual steps were performed to verify this patch.
|
_ trunk Compile Tests _ |
| +1 :green_heart: | mvninstall | 46m 52s | | trunk passed |
| +1 :green_heart: | compile | 1m 11s | | trunk passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 1m 11s | | trunk passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 0s | | trunk passed |
| +1 :green_heart: | mvnsite | 1m 9s | | trunk passed |
| +1 :green_heart: | javadoc | 1m 8s | | trunk passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 0m 54s | | trunk passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 2m 26s | | trunk passed |
| +1 :green_heart: | shadedclient | 23m 54s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +1 :green_heart: | mvninstall | 0m 57s | | the patch passed |
| +1 :green_heart: | compile | 1m 6s | | the patch passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 1m 6s | | the patch passed |
| +1 :green_heart: | compile | 0m 55s | | the patch passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 0m 55s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| +1 :green_heart: | checkstyle | 0m 43s | | the patch passed |
| +1 :green_heart: | mvnsite | 1m 0s | | the patch passed |
| +1 :green_heart: | javadoc | 0m 47s | | the patch passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 0m 43s | | the patch passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 2m 7s | | the patch passed |
| +1 :green_heart: | shadedclient | 23m 9s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | unit | 100m 31s | |
hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | asflicense | 0m 48s | | The patch does not
generate ASF License warnings. |
| | | 212m 52s | | |
| Subsystem | Report/Notes |
|--:|:-|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base:
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5173/2/artifact/out/Dockerfile
|
| GITHUB PR | https://github.com/apache/hadoop/pull/5173 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 4ca2c696c56c 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4
01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / 9f68fc2ea550d84e1790e64c48362f228287cd60 |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions |
/usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04
/usr/lib/jvm/java-8-openjdk-amd64:Private
Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results |
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5173/2/testReport/ |
| Max. process+thread count | 945 (vs. ulimit of 5500) |
| modules | C:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager
U:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager
|
| Console output |
[jira] [Commented] (YARN-11349) [Federation] Router Support DelegationToken With SQL
[
https://issues.apache.org/jira/browse/YARN-11349?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17640863#comment-17640863
]
ASF GitHub Bot commented on YARN-11349:
---
goiri commented on code in PR #5169:
URL: https://github.com/apache/hadoop/pull/5169#discussion_r1035047482
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java:
##
@@ -164,6 +170,40 @@ public void setRenewDate(Long renewDate) {
this.builder.setRenewDate(renewDate);
}
+ @Override
+ public byte[] toByteArray() throws IOException {
+return builder.build().toByteArray();
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+builder.mergeFrom((DataInputStream) in);
+ }
+
+ @Override
+ public String getTokenInfo() {
+RouterStoreTokenProtoOrBuilder p = viaProto ? proto : builder;
+if (this.tokenInfo != null) {
+ return this.tokenInfo;
+}
+if (!p.hasTokenInfo()) {
+ return null;
+}
+this.tokenInfo = p.getTokenInfo();
+return this.tokenInfo;
+ }
+
+ @Override
+ protected void setTokenInfo(String tokenInfo) {
+maybeInitBuilder();
+if(tokenInfo == null) {
Review Comment:
Space
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java:
##
@@ -1353,45 +1386,560 @@ public Connection getConn() {
return conn;
}
+ /**
+ * SQLFederationStateStore Supports Store New MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an
abstraction for DelegationKey.
+ * @return routerMasterKeyResponse, the response contains the
RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest
request)
throws YarnException, IOException {
-throw new NotImplementedException("Code is not implemented");
+
+// Step1: Verify parameters to ensure that key fields are not empty.
+FederationRouterRMTokenInputValidator.validate(request);
+
+// Step2: Parse the parameters and serialize the DelegationKey as a string.
+DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+int keyId = delegationKey.getKeyId();
+String delegationKeyStr = encodeWritable(delegationKey);
+CallableStatement cstmt = null;
+
+// Step3. store data in database.
+try {
+ // Call procedure
+ cstmt = getCallableStatement(CALL_SP_ADD_MASTERKEY);
+
+ // Set the parameters for the stored procedure
+ // 1）IN keyId_IN bigint
+ cstmt.setInt("keyId_IN", keyId);
+ // 2）IN masterKey_IN varbinary(1024)
+ cstmt.setString("masterKey_IN", delegationKeyStr);
+ // 3) OUT rowCount_OUT int
+ cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER);
+
+ long startTime = clock.getTime();
+ cstmt.executeUpdate();
+ long stopTime = clock.getTime();
+
+ // Get rowCount
+ int rowCount = cstmt.getInt("rowCount_OUT");
+
+ // We hope that 1 record can be written to the database.
+ // If the number of records is not 1, it means that the data was written
incorrectly.
+ if (rowCount != 1) {
+FederationStateStoreUtils.logAndThrowStoreException(LOG,
+"Wrong behavior during the insertion of masterKey, keyId = %s. " +
+"please check the records of the database.",
String.valueOf(keyId));
+ }
+ FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime -
startTime);
+} catch (SQLException e) {
+ FederationStateStoreClientMetrics.failedStateStoreCall();
+ FederationStateStoreUtils.logAndThrowRetriableException(e, LOG,
+ "Unable to insert the newly masterKey, keyId = %s.",
String.valueOf(keyId));
+} finally {
+ // Return to the pool the CallableStatement
+ FederationStateStoreUtils.returnToPool(LOG, cstmt);
+}
+
+// Step4. Query Data from the database and return the result.
+return getMasterKeyByDelegationKey(request);
}
+ /**
+ * SQLFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an
abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the
RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest
request)
throws YarnException, IOException {
-throw new NotImplementedException("Code is not implemented");
+//
[jira] [Commented] (YARN-11350) [Federation] Router Support DelegationToken With ZK
[ https://issues.apache.org/jira/browse/YARN-11350?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17640841#comment-17640841 ] ASF GitHub Bot commented on YARN-11350: --- goiri commented on code in PR #5131: URL: https://github.com/apache/hadoop/pull/5131#discussion_r1035037274 ## hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java: ## @@ -22,6 +22,7 @@ import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier; import org.apache.hadoop.yarn.util.Records; +import java.io.DataInput; Review Comment: Why do we need this? We didn't add anything using DataInput to this class. > [Federation] Router Support DelegationToken With ZK > --- > > Key: YARN-11350 > URL: https://issues.apache.org/jira/browse/YARN-11350 > Project: Hadoop YARN > Issue Type: Sub-task > Components: federation, router >Affects Versions: 3.4.0 >Reporter: Shilun Fan >Assignee: Shilun Fan >Priority: Major > Labels: pull-request-available > > [Federation] Router Support DelegationToken With > ZookeeperFederationStateStore. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11226) [Federation] Add createNewReservation, submitReservation, updateReservation, deleteReservation REST APIs for Router
[
https://issues.apache.org/jira/browse/YARN-11226?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17640838#comment-17640838
]
ASF GitHub Bot commented on YARN-11226:
---
hadoop-yetus commented on PR #4892:
URL: https://github.com/apache/hadoop/pull/4892#issuecomment-1330982849
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 1m 4s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 5 new or modified test files. |
_ trunk Compile Tests _ |
| +1 :green_heart: | mvninstall | 41m 27s | | trunk passed |
| +1 :green_heart: | compile | 0m 48s | | trunk passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 0m 45s | | trunk passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 0m 45s | | trunk passed |
| +1 :green_heart: | mvnsite | 0m 48s | | trunk passed |
| +1 :green_heart: | javadoc | 0m 52s | | trunk passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 0m 35s | | trunk passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 1m 12s | | trunk passed |
| +1 :green_heart: | shadedclient | 20m 47s | | branch has no errors
when building and testing our client artifacts. |
| -0 :warning: | patch | 21m 9s | | Used diff version of patch file.
Binary files and potentially other changes not applied. Please rebase and
squash commits if necessary. |
_ Patch Compile Tests _ |
| +1 :green_heart: | mvninstall | 0m 29s | | the patch passed |
| +1 :green_heart: | compile | 0m 29s | | the patch passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 0m 29s | | the patch passed |
| +1 :green_heart: | compile | 0m 26s | | the patch passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 0m 26s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| -0 :warning: | checkstyle | 0m 24s |
[/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-router.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-4892/41/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-router.txt)
|
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router:
The patch generated 1 new + 0 unchanged - 1 fixed = 1 total (was 1) |
| +1 :green_heart: | mvnsite | 0m 29s | | the patch passed |
| +1 :green_heart: | javadoc | 0m 24s | | the patch passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 0m 24s | | the patch passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 0m 58s | | the patch passed |
| +1 :green_heart: | shadedclient | 20m 31s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | unit | 0m 35s | | hadoop-yarn-server-router in
the patch passed. |
| +1 :green_heart: | asflicense | 0m 45s | | The patch does not
generate ASF License warnings. |
| | | 97m 8s | | |
| Subsystem | Report/Notes |
|--:|:-|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base:
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-4892/41/artifact/out/Dockerfile
|
| GITHUB PR | https://github.com/apache/hadoop/pull/4892 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 6ae58f79ac07 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4
01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / 5011e125f8d35a8cd69b736b61f7dacd0840bc45 |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions |
/usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04
/usr/lib/jvm/java-8-openjdk-amd64:Private
[jira] [Commented] (YARN-11226) [Federation] Add createNewReservation, submitReservation, updateReservation, deleteReservation REST APIs for Router
[
https://issues.apache.org/jira/browse/YARN-11226?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17640836#comment-17640836
]
ASF GitHub Bot commented on YARN-11226:
---
hadoop-yetus commented on PR #4892:
URL: https://github.com/apache/hadoop/pull/4892#issuecomment-1330972520
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 1m 0s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 5 new or modified test files. |
_ trunk Compile Tests _ |
| +1 :green_heart: | mvninstall | 41m 34s | | trunk passed |
| +1 :green_heart: | compile | 0m 41s | | trunk passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 0m 39s | | trunk passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 0m 36s | | trunk passed |
| +1 :green_heart: | mvnsite | 0m 40s | | trunk passed |
| +1 :green_heart: | javadoc | 0m 44s | | trunk passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 0m 32s | | trunk passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 1m 10s | | trunk passed |
| +1 :green_heart: | shadedclient | 23m 58s | | branch has no errors
when building and testing our client artifacts. |
| -0 :warning: | patch | 24m 23s | | Used diff version of patch file.
Binary files and potentially other changes not applied. Please rebase and
squash commits if necessary. |
_ Patch Compile Tests _ |
| -1 :x: | mvninstall | 0m 21s |
[/patch-mvninstall-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-router.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-4892/40/artifact/out/patch-mvninstall-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-router.txt)
| hadoop-yarn-server-router in the patch failed. |
| -1 :x: | compile | 0m 23s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-router-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-4892/40/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-router-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt)
| hadoop-yarn-server-router in the patch failed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| -1 :x: | javac | 0m 23s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-router-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-4892/40/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-router-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt)
| hadoop-yarn-server-router in the patch failed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| -1 :x: | compile | 0m 21s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-router-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-4892/40/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-router-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt)
| hadoop-yarn-server-router in the patch failed with JDK Private
Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| -1 :x: | javac | 0m 21s |
[/patch-compile-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-router-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-4892/40/artifact/out/patch-compile-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-router-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt)
| hadoop-yarn-server-router in the patch failed with JDK Private
Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| -0 :warning: | checkstyle | 0m 19s |
[jira] [Updated] (YARN-6538) Inter Queue preemption is not happening when DRF is configured
[
https://issues.apache.org/jira/browse/YARN-6538?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
ASF GitHub Bot updated YARN-6538:
-
Labels: pull-request-available (was: )
> Inter Queue preemption is not happening when DRF is configured
> --
>
> Key: YARN-6538
> URL: https://issues.apache.org/jira/browse/YARN-6538
> Project: Hadoop YARN
> Issue Type: Sub-task
> Components: capacity scheduler, scheduler preemption
>Affects Versions: 2.8.0
>Reporter: Sunil G
>Assignee: Sunil G
>Priority: Major
> Labels: pull-request-available
>
> Cluster capacity of . Here memory is more and vcores
> are less. If applications have more demand, vcores might be exhausted.
> Inter queue preemption ideally has to be kicked in once vcores is over
> utilized. However preemption is not happening.
> Analysis:
> In {{AbstractPreemptableResourceCalculator.computeFixpointAllocation}},
> {code}
> // assign all cluster resources until no more demand, or no resources are
> // left
> while (!orderedByNeed.isEmpty() && Resources.greaterThan(rc, totGuarant,
> unassigned, Resources.none())) {
> {code}
> will loop even when vcores are 0 (because memory is still +ve). Hence we are
> having more vcores in idealAssigned which cause no-preemption cases.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Assigned] (YARN-6538) Inter Queue preemption is not happening when DRF is configured
[
https://issues.apache.org/jira/browse/YARN-6538?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Benjamin Teke reassigned YARN-6538:
---
Assignee: Benjamin Teke (was: Sunil G)
> Inter Queue preemption is not happening when DRF is configured
> --
>
> Key: YARN-6538
> URL: https://issues.apache.org/jira/browse/YARN-6538
> Project: Hadoop YARN
> Issue Type: Sub-task
> Components: capacity scheduler, scheduler preemption
>Affects Versions: 2.8.0
>Reporter: Sunil G
>Assignee: Benjamin Teke
>Priority: Major
> Labels: pull-request-available
>
> Cluster capacity of . Here memory is more and vcores
> are less. If applications have more demand, vcores might be exhausted.
> Inter queue preemption ideally has to be kicked in once vcores is over
> utilized. However preemption is not happening.
> Analysis:
> In {{AbstractPreemptableResourceCalculator.computeFixpointAllocation}},
> {code}
> // assign all cluster resources until no more demand, or no resources are
> // left
> while (!orderedByNeed.isEmpty() && Resources.greaterThan(rc, totGuarant,
> unassigned, Resources.none())) {
> {code}
> will loop even when vcores are 0 (because memory is still +ve). Hence we are
> having more vcores in idealAssigned which cause no-preemption cases.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-6538) Inter Queue preemption is not happening when DRF is configured
[
https://issues.apache.org/jira/browse/YARN-6538?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17640826#comment-17640826
]
Benjamin Teke commented on YARN-6538:
-
Based on offline discussion taking this over from [~sunilg]
> Inter Queue preemption is not happening when DRF is configured
> --
>
> Key: YARN-6538
> URL: https://issues.apache.org/jira/browse/YARN-6538
> Project: Hadoop YARN
> Issue Type: Sub-task
> Components: capacity scheduler, scheduler preemption
>Affects Versions: 2.8.0
>Reporter: Sunil G
>Assignee: Sunil G
>Priority: Major
> Labels: pull-request-available
>
> Cluster capacity of . Here memory is more and vcores
> are less. If applications have more demand, vcores might be exhausted.
> Inter queue preemption ideally has to be kicked in once vcores is over
> utilized. However preemption is not happening.
> Analysis:
> In {{AbstractPreemptableResourceCalculator.computeFixpointAllocation}},
> {code}
> // assign all cluster resources until no more demand, or no resources are
> // left
> while (!orderedByNeed.isEmpty() && Resources.greaterThan(rc, totGuarant,
> unassigned, Resources.none())) {
> {code}
> will loop even when vcores are 0 (because memory is still +ve). Hence we are
> having more vcores in idealAssigned which cause no-preemption cases.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-6538) Inter Queue preemption is not happening when DRF is configured
[
https://issues.apache.org/jira/browse/YARN-6538?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17640825#comment-17640825
]
ASF GitHub Bot commented on YARN-6538:
--
brumi1024 opened a new pull request, #5174:
URL: https://github.com/apache/hadoop/pull/5174
### How was this patch tested?
Extended the UTs and run the existing ones.
### For code changes:
- [ ] Does the title or this PR starts with the corresponding JIRA issue id
(e.g. 'HADOOP-17799. Your PR title ...')?
- [ ] Object storage: have the integration tests been executed and the
endpoint declared according to the connector-specific documentation?
- [ ] If adding new dependencies to the code, are these dependencies
licensed in a way that is compatible for inclusion under [ASF
2.0](http://www.apache.org/legal/resolved.html#category-a)?
- [ ] If applicable, have you updated the `LICENSE`, `LICENSE-binary`,
`NOTICE-binary` files?
> Inter Queue preemption is not happening when DRF is configured
> --
>
> Key: YARN-6538
> URL: https://issues.apache.org/jira/browse/YARN-6538
> Project: Hadoop YARN
> Issue Type: Sub-task
> Components: capacity scheduler, scheduler preemption
>Affects Versions: 2.8.0
>Reporter: Sunil G
>Assignee: Sunil G
>Priority: Major
>
> Cluster capacity of . Here memory is more and vcores
> are less. If applications have more demand, vcores might be exhausted.
> Inter queue preemption ideally has to be kicked in once vcores is over
> utilized. However preemption is not happening.
> Analysis:
> In {{AbstractPreemptableResourceCalculator.computeFixpointAllocation}},
> {code}
> // assign all cluster resources until no more demand, or no resources are
> // left
> while (!orderedByNeed.isEmpty() && Resources.greaterThan(rc, totGuarant,
> unassigned, Resources.none())) {
> {code}
> will loop even when vcores are 0 (because memory is still +ve). Hence we are
> having more vcores in idealAssigned which cause no-preemption cases.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
-
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11350) [Federation] Router Support DelegationToken With ZK
[ https://issues.apache.org/jira/browse/YARN-11350?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17640767#comment-17640767 ] ASF GitHub Bot commented on YARN-11350: --- slfan1989 closed pull request #5111: YARN-11350. [Federation] Router Support DelegationToken With ZK.(Old Version) URL: https://github.com/apache/hadoop/pull/5111 > [Federation] Router Support DelegationToken With ZK > --- > > Key: YARN-11350 > URL: https://issues.apache.org/jira/browse/YARN-11350 > Project: Hadoop YARN > Issue Type: Sub-task > Components: federation, router >Affects Versions: 3.4.0 >Reporter: Shilun Fan >Assignee: Shilun Fan >Priority: Major > Labels: pull-request-available > > [Federation] Router Support DelegationToken With > ZookeeperFederationStateStore. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11350) [Federation] Router Support DelegationToken With ZK
[ https://issues.apache.org/jira/browse/YARN-11350?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17640766#comment-17640766 ] ASF GitHub Bot commented on YARN-11350: --- slfan1989 commented on PR #5111: URL: https://github.com/apache/hadoop/pull/5111#issuecomment-1330800385 We have submitted a new pr (#5131), this pr will be closed. > [Federation] Router Support DelegationToken With ZK > --- > > Key: YARN-11350 > URL: https://issues.apache.org/jira/browse/YARN-11350 > Project: Hadoop YARN > Issue Type: Sub-task > Components: federation, router >Affects Versions: 3.4.0 >Reporter: Shilun Fan >Assignee: Shilun Fan >Priority: Major > Labels: pull-request-available > > [Federation] Router Support DelegationToken With > ZookeeperFederationStateStore. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11350) [Federation] Router Support DelegationToken With ZK
[
https://issues.apache.org/jira/browse/YARN-11350?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17640761#comment-17640761
]
ASF GitHub Bot commented on YARN-11350:
---
slfan1989 commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1034873671
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java:
##
@@ -886,45 +975,554 @@ public UpdateReservationHomeSubClusterResponse
updateReservationHomeSubCluster(
return UpdateReservationHomeSubClusterResponse.newInstance();
}
+ /**
+ * ZookeeperFederationStateStore Supports Store NewMasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an
abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the
RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest
request)
+ public synchronized RouterMasterKeyResponse
storeNewMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
-throw new NotImplementedException("Code is not implemented");
+
+long start = clock.getTime();
+// For the verification of the request, after passing the verification,
+// the request and the internal objects will not be empty and can be used
directly.
+FederationRouterRMTokenInputValidator.validate(request);
+
+// Parse the delegationKey from the request and get the ZK storage path.
+DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+String nodeCreatePath =
getMasterKeyZNodePathByDelegationKey(delegationKey);
+LOG.debug("Storing RMDelegationKey_{}, ZkNodePath = {}.",
delegationKey.getKeyId(),
+nodeCreatePath);
+
+// Write master key data to zk.
+try(ByteArrayOutputStream os = new ByteArrayOutputStream();
+DataOutputStream fsOut = new DataOutputStream(os)) {
+ delegationKey.write(fsOut);
+ put(nodeCreatePath, os.toByteArray(), false);
+}
+
+// Get the stored masterKey from zk.
+RouterMasterKey masterKeyFromZK = getRouterMasterKeyFromZK(nodeCreatePath);
+long end = clock.getTime();
+opDurations.addStoreNewMasterKeyDuration(start, end);
+return RouterMasterKeyResponse.newInstance(masterKeyFromZK);
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an
abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the
RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest
request)
+ public synchronized RouterMasterKeyResponse
removeStoredMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
-throw new NotImplementedException("Code is not implemented");
+
+long start = clock.getTime();
+// For the verification of the request, after passing the verification,
+// the request and the internal objects will not be empty and can be used
directly.
+FederationRouterRMTokenInputValidator.validate(request);
+
+try {
+ // Parse the delegationKey from the request and get the ZK storage path.
+ RouterMasterKey masterKey = request.getRouterMasterKey();
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeRemovePath =
getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Removing RMDelegationKey_{}, ZkNodePath = {}.",
delegationKey.getKeyId(),
+ nodeRemovePath);
+
+ // Check if the path exists, Throws an exception if the path does not
exist.
+ if (!exists(nodeRemovePath)) {
+throw new YarnException("ZkNodePath = " + nodeRemovePath + " not
exists!");
+ }
+
+ // try to remove masterKey.
+ zkManager.delete(nodeRemovePath);
+ long end = clock.getTime();
+ opDurations.removeStoredMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKey);
+} catch (Exception e) {
+ throw new YarnException(e);
+}
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an
abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the
RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse
[jira] [Commented] (YARN-11383) Workflow priority mappings is case sensitive
[
https://issues.apache.org/jira/browse/YARN-11383?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17640743#comment-17640743
]
ASF GitHub Bot commented on YARN-11383:
---
hadoop-yetus commented on PR #5171:
URL: https://github.com/apache/hadoop/pull/5171#issuecomment-1330738006
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 1m 1s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 1 new or modified test files. |
_ trunk Compile Tests _ |
| +1 :green_heart: | mvninstall | 41m 22s | | trunk passed |
| +1 :green_heart: | compile | 1m 17s | | trunk passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 1m 4s | | trunk passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 4s | | trunk passed |
| +1 :green_heart: | mvnsite | 1m 9s | | trunk passed |
| +1 :green_heart: | javadoc | 1m 2s | | trunk passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 0m 50s | | trunk passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 2m 16s | | trunk passed |
| +1 :green_heart: | shadedclient | 24m 54s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +1 :green_heart: | mvninstall | 0m 56s | | the patch passed |
| +1 :green_heart: | compile | 1m 4s | | the patch passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 1m 4s | | the patch passed |
| +1 :green_heart: | compile | 0m 54s | | the patch passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 0m 54s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| +1 :green_heart: | checkstyle | 0m 43s | | the patch passed |
| +1 :green_heart: | mvnsite | 0m 58s | | the patch passed |
| +1 :green_heart: | javadoc | 0m 45s | | the patch passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 0m 39s | | the patch passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 2m 6s | | the patch passed |
| +1 :green_heart: | shadedclient | 24m 50s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| -1 :x: | unit | 102m 8s |
[/patch-unit-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5171/3/artifact/out/patch-unit-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager.txt)
| hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | asflicense | 0m 39s | | The patch does not
generate ASF License warnings. |
| | | 211m 11s | | |
| Reason | Tests |
|---:|:--|
| Failed junit tests |
hadoop.yarn.server.resourcemanager.reservation.TestCapacityOverTimePolicy |
| Subsystem | Report/Notes |
|--:|:-|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base:
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5171/3/artifact/out/Dockerfile
|
| GITHUB PR | https://github.com/apache/hadoop/pull/5171 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 9d129e64f201 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4
01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / c511e7fa6067fd35ae7cd87f2257e33b4fbbc143 |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions |
/usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04
/usr/lib/jvm/java-8-openjdk-amd64:Private
Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results |
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5171/3/testReport/ |
| Max. process+thread
[jira] [Commented] (YARN-5607) Document TestContainerResourceUsage#waitForContainerCompletion
[ https://issues.apache.org/jira/browse/YARN-5607?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17640733#comment-17640733 ] ASF GitHub Bot commented on YARN-5607: -- ashutoshcipher commented on PR #5119: URL: https://github.com/apache/hadoop/pull/5119#issuecomment-1330722591 Thanks @susheel-gupta , I will look into it > Document TestContainerResourceUsage#waitForContainerCompletion > -- > > Key: YARN-5607 > URL: https://issues.apache.org/jira/browse/YARN-5607 > Project: Hadoop YARN > Issue Type: Test > Components: resourcemanager, test >Affects Versions: 2.9.0 >Reporter: Karthik Kambatla >Assignee: Susheel Gupta >Priority: Major > Labels: newbie, pull-request-available > > The logic in TestContainerResourceUsage#waitForContainerCompletion > (introduced in YARN-5024) is not immediately obvious. It could use some > documentation. Also, this seems like a useful helper method. Should this be > moved to one of the mock classes or to a util class? -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-10946) AbstractCSQueue: Create separate class for constructing Queue API objects
[
https://issues.apache.org/jira/browse/YARN-10946?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17640714#comment-17640714
]
ASF GitHub Bot commented on YARN-10946:
---
hadoop-yetus commented on PR #5173:
URL: https://github.com/apache/hadoop/pull/5173#issuecomment-1330664079
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 0m 45s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| -1 :x: | test4tests | 0m 0s | | The patch doesn't appear to include
any new or modified tests. Please justify why no new tests are needed for this
patch. Also please list what manual steps were performed to verify this patch.
|
_ trunk Compile Tests _ |
| +1 :green_heart: | mvninstall | 38m 28s | | trunk passed |
| +1 :green_heart: | compile | 1m 22s | | trunk passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 1m 5s | | trunk passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 1s | | trunk passed |
| +1 :green_heart: | mvnsite | 1m 17s | | trunk passed |
| +1 :green_heart: | javadoc | 1m 5s | | trunk passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 1m 0s | | trunk passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 2m 24s | | trunk passed |
| +1 :green_heart: | shadedclient | 21m 56s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +1 :green_heart: | mvninstall | 0m 55s | | the patch passed |
| +1 :green_heart: | compile | 0m 59s | | the patch passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 0m 59s | | the patch passed |
| +1 :green_heart: | compile | 0m 54s | | the patch passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 0m 54s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| -0 :warning: | checkstyle | 0m 47s |
[/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5173/1/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager.txt)
|
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager:
The patch generated 2 new + 23 unchanged - 0 fixed = 25 total (was 23) |
| +1 :green_heart: | mvnsite | 0m 57s | | the patch passed |
| +1 :green_heart: | javadoc | 0m 42s | | the patch passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 0m 46s | | the patch passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 2m 0s | | the patch passed |
| +1 :green_heart: | shadedclient | 21m 50s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | unit | 99m 3s | |
hadoop-yarn-server-resourcemanager in the patch passed. |
| -1 :x: | asflicense | 0m 43s |
[/results-asflicense.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5173/1/artifact/out/results-asflicense.txt)
| The patch generated 1 ASF License warnings. |
| | | 199m 33s | | |
| Subsystem | Report/Notes |
|--:|:-|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base:
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5173/1/artifact/out/Dockerfile
|
| GITHUB PR | https://github.com/apache/hadoop/pull/5173 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux bcf2130f3e5c 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4
01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / 844f6e8313ca38e238fce8e15aa07bdd3c4c90c5 |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions |
[jira] [Resolved] (YARN-4944) Handle lack of ResourceCalculatorPlugin gracefully
[ https://issues.apache.org/jira/browse/YARN-4944?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Susheel Gupta resolved YARN-4944. - Resolution: Duplicate > Handle lack of ResourceCalculatorPlugin gracefully > -- > > Key: YARN-4944 > URL: https://issues.apache.org/jira/browse/YARN-4944 > Project: Hadoop YARN > Issue Type: Improvement > Components: nodemanager >Affects Versions: 2.8.0 >Reporter: Karthik Kambatla >Assignee: Susheel Gupta >Priority: Major > Labels: newbie++, trivial > > On some systems (e.g. mac), the NM might not be able to instantiate a > ResourceCalculatorPlugin and leads to logging a bunch of error messages. We > could improve the way we handle this. -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11349) [Federation] Router Support DelegationToken With SQL
[
https://issues.apache.org/jira/browse/YARN-11349?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17640590#comment-17640590
]
ASF GitHub Bot commented on YARN-11349:
---
hadoop-yetus commented on PR #5169:
URL: https://github.com/apache/hadoop/pull/5169#issuecomment-1330402586
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 0m 57s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available.
|
| +0 :ok: | buf | 0m 0s | | buf was not available. |
| +0 :ok: | buf | 0m 0s | | buf was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 2 new or modified test files. |
_ trunk Compile Tests _ |
| +0 :ok: | mvndep | 15m 59s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 29m 10s | | trunk passed |
| +1 :green_heart: | compile | 25m 30s | | trunk passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 22m 8s | | trunk passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 4m 24s | | trunk passed |
| +1 :green_heart: | mvnsite | 8m 52s | | trunk passed |
| +1 :green_heart: | javadoc | 5m 47s | | trunk passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 4m 42s | | trunk passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 18m 2s | | trunk passed |
| +1 :green_heart: | shadedclient | 23m 49s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 24s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 7m 2s | | the patch passed |
| +1 :green_heart: | compile | 24m 59s | | the patch passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | cc | 24m 59s | | the patch passed |
| +1 :green_heart: | javac | 24m 59s | | the patch passed |
| +1 :green_heart: | compile | 22m 13s | | the patch passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | cc | 22m 13s | | the patch passed |
| +1 :green_heart: | javac | 22m 13s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| +1 :green_heart: | checkstyle | 4m 17s | | the patch passed |
| +1 :green_heart: | mvnsite | 8m 44s | | the patch passed |
| -1 :x: | javadoc | 1m 0s |
[/results-javadoc-javadoc-hadoop-common-project_hadoop-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5169/4/artifact/out/results-javadoc-javadoc-hadoop-common-project_hadoop-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt)
|
hadoop-common-project_hadoop-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04
with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 generated 1 new + 0
unchanged - 0 fixed = 1 total (was 0) |
| +1 :green_heart: | javadoc | 4m 40s | | the patch passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 18m 34s | | the patch passed |
| +1 :green_heart: | shadedclient | 23m 39s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | unit | 18m 33s | | hadoop-common in the patch
passed. |
| +1 :green_heart: | unit | 255m 5s | | hadoop-yarn in the patch
passed. |
| +1 :green_heart: | unit | 3m 33s | | hadoop-yarn-server-common in
the patch passed. |
| +1 :green_heart: | unit | 1m 1s | | hadoop-yarn-server-router in
the patch passed. |
| +1 :green_heart: | asflicense | 1m 16s | | The patch does not
generate ASF License warnings. |
| | | 563m 18s | | |
| Subsystem | Report/Notes |
|--:|:-|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base:
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5169/4/artifact/out/Dockerfile
|
| GITHUB PR | https://github.com/apache/hadoop/pull/5169 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets cc
[jira] [Updated] (YARN-10946) AbstractCSQueue: Create separate class for constructing Queue API objects
[ https://issues.apache.org/jira/browse/YARN-10946?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] ASF GitHub Bot updated YARN-10946: -- Labels: pull-request-available (was: ) > AbstractCSQueue: Create separate class for constructing Queue API objects > - > > Key: YARN-10946 > URL: https://issues.apache.org/jira/browse/YARN-10946 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Szilard Nemeth >Assignee: Peter Szucs >Priority: Minor > Labels: pull-request-available > > Relevant methods are: > - > org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue#getQueueConfigurations > - > org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue#getQueueInfo > - > org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue#getQueueStatistics -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-10946) AbstractCSQueue: Create separate class for constructing Queue API objects
[ https://issues.apache.org/jira/browse/YARN-10946?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17640587#comment-17640587 ] ASF GitHub Bot commented on YARN-10946: --- p-szucs opened a new pull request, #5173: URL: https://github.com/apache/hadoop/pull/5173 Change-Id: If4da3cd24e9b2a94689ef8e1ea5b69dc1b84288b ### Description of PR AbstractCSQueue: Create separate class for constructing Queue API objects Relevant methods are: - org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue#getQueueConfigurations - org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue#getQueueInfo - org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue#getQueueStatistics ### How was this patch tested? ### For code changes: - [x] Does the title or this PR starts with the corresponding JIRA issue id (e.g. 'HADOOP-17799. Your PR title ...')? - [ ] Object storage: have the integration tests been executed and the endpoint declared according to the connector-specific documentation? - [ ] If adding new dependencies to the code, are these dependencies licensed in a way that is compatible for inclusion under [ASF 2.0](http://www.apache.org/legal/resolved.html#category-a)? - [ ] If applicable, have you updated the `LICENSE`, `LICENSE-binary`, `NOTICE-binary` files? > AbstractCSQueue: Create separate class for constructing Queue API objects > - > > Key: YARN-10946 > URL: https://issues.apache.org/jira/browse/YARN-10946 > Project: Hadoop YARN > Issue Type: Sub-task >Reporter: Szilard Nemeth >Assignee: Peter Szucs >Priority: Minor > > Relevant methods are: > - > org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue#getQueueConfigurations > - > org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue#getQueueInfo > - > org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue#getQueueStatistics -- This message was sent by Atlassian Jira (v8.20.10#820010) - To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
[jira] [Commented] (YARN-11383) Workflow priority mappings is case sensitive
[
https://issues.apache.org/jira/browse/YARN-11383?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17640465#comment-17640465
]
ASF GitHub Bot commented on YARN-11383:
---
hadoop-yetus commented on PR #5171:
URL: https://github.com/apache/hadoop/pull/5171#issuecomment-1330268950
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|::|--:|:|::|:---:|
| +0 :ok: | reexec | 0m 40s | | Docker mode activated. |
_ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files
found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available.
|
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain
any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to
include 1 new or modified test files. |
_ trunk Compile Tests _ |
| +1 :green_heart: | mvninstall | 39m 17s | | trunk passed |
| +1 :green_heart: | compile | 1m 22s | | trunk passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 1m 5s | | trunk passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 10s | | trunk passed |
| +1 :green_heart: | mvnsite | 1m 10s | | trunk passed |
| +1 :green_heart: | javadoc | 1m 5s | | trunk passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 1m 2s | | trunk passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 2m 25s | | trunk passed |
| +1 :green_heart: | shadedclient | 21m 43s | | branch has no errors
when building and testing our client artifacts. |
_ Patch Compile Tests _ |
| +1 :green_heart: | mvninstall | 0m 54s | | the patch passed |
| +1 :green_heart: | compile | 1m 3s | | the patch passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 1m 3s | | the patch passed |
| +1 :green_heart: | compile | 0m 58s | | the patch passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 0m 58s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks
issues. |
| -0 :warning: | checkstyle | 0m 45s |
[/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5171/2/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager.txt)
|
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager:
The patch generated 1 new + 9 unchanged - 0 fixed = 10 total (was 9) |
| +1 :green_heart: | mvnsite | 1m 2s | | the patch passed |
| +1 :green_heart: | javadoc | 0m 42s | | the patch passed with JDK
Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 0m 41s | | the patch passed with JDK
Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 2m 0s | | the patch passed |
| +1 :green_heart: | shadedclient | 21m 17s | | patch has no errors
when building and testing our client artifacts. |
_ Other Tests _ |
| +1 :green_heart: | unit | 99m 24s | |
hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | asflicense | 0m 41s | | The patch does not
generate ASF License warnings. |
| | | 199m 47s | | |
| Subsystem | Report/Notes |
|--:|:-|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base:
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5171/2/artifact/out/Dockerfile
|
| GITHUB PR | https://github.com/apache/hadoop/pull/5171 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall
mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 78d6ddd4f4ef 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4
01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / 55f2ab31a365526749e7c10e6ebc8a8a066daa47 |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions |
/usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04
/usr/lib/jvm/java-8-openjdk-amd64:Private
Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results |
https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5171/2/testReport/ |
| Max.
