[hadoop] branch trunk updated (7cb22eb72d5 -> ac958777afd)
This is an automated email from the ASF dual-hosted git repository. zanderxu pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git from 7cb22eb72d5 YARN-11371. [Federation] Refactor FederationInterceptorREST#createNewApplication\submitApplication Use FederationActionRetry. (#5130) add ac958777afd HDFS-16813. Remove parameter validation logic such as dfs.namenode.decommission.blocks.per.interval in DatanodeAdminManager#activate (#5063) No new revisions were added by this update. Summary of changes: .../DatanodeAdminDefaultMonitor.java | 9 ++ .../blockmanagement/DatanodeAdminManager.java | 33 +- .../blockmanagement/DatanodeAdminMonitorBase.java | 4 +++ 3 files changed, 14 insertions(+), 32 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-11371. [Federation] Refactor FederationInterceptorREST#createNewApplication\submitApplication Use FederationActionRetry. (#5130)
This is an automated email from the ASF dual-hosted git repository.
inigoiri pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7cb22eb72d5 YARN-11371. [Federation] Refactor
FederationInterceptorREST#createNewApplication\submitApplication Use
FederationActionRetry. (#5130)
7cb22eb72d5 is described below
commit 7cb22eb72d587d4b721c3e072bb2811c767db9ab
Author: slfan1989 <55643692+slfan1...@users.noreply.github.com>
AuthorDate: Wed Nov 23 06:38:24 2022 +0800
YARN-11371. [Federation] Refactor
FederationInterceptorREST#createNewApplication\submitApplication Use
FederationActionRetry. (#5130)
---
.../utils/FederationStateStoreFacade.java | 170 ++-
.../yarn/server/router/RouterServerUtil.java | 45 ---
.../clientrm/FederationClientInterceptor.java | 96 +-
.../router/webapp/FederationInterceptorREST.java | 324 +
.../webapp/TestFederationInterceptorRESTRetry.java | 17 +-
5 files changed, 334 insertions(+), 318 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
index 47cb9e9e35c..fc1e442ab9c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
@@ -22,8 +22,10 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.List;
+import java.util.ArrayList;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import java.util.Random;
import javax.cache.Cache;
import javax.cache.CacheManager;
@@ -38,6 +40,8 @@ import javax.cache.integration.CacheLoader;
import javax.cache.integration.CacheLoaderException;
import javax.cache.spi.CachingProvider;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.retry.RetryPolicies;
@@ -50,6 +54,8 @@ import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
+import
org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
import
org.apache.hadoop.yarn.server.federation.store.exception.FederationStateStoreRetriableException;
@@ -110,6 +116,8 @@ public final class FederationStateStoreFacade {
private static final FederationStateStoreFacade FACADE =
new FederationStateStoreFacade();
+ private static Random rand = new Random(System.currentTimeMillis());
+
private FederationStateStore stateStore;
private int cacheTimeToLive;
private Configuration conf;
@@ -496,6 +504,7 @@ public final class FederationStateStoreFacade {
* @param defaultValue the default implementation for fallback
* @param type the class for which a retry proxy is required
* @param retryPolicy the policy for retrying method call failures
+ * @param The type of the instance
* @return a retry proxy for the specified interface
*/
public static Object createRetryInstance(Configuration conf,
@@ -731,7 +740,7 @@ public final class FederationStateStoreFacade {
return stateStore;
}
- /*
+ /**
* The Router Supports Store NewMasterKey (RouterMasterKey{@link
RouterMasterKey}).
*
* @param newKey Key used for generating and verifying delegation tokens
@@ -849,4 +858,163 @@ public final class FederationStateStoreFacade {
RouterRMTokenRequest request =
RouterRMTokenRequest.newInstance(storeToken);
return stateStore.getTokenByRouterStoreToken(request);
}
+
+ /**
+ * Get the number of active cluster nodes.
+ *
+ * @return number of active cluster nodes.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ */
+ public int getActiveSubClustersCount() throws YarnException {
+Map activeSubClusters = getSubClusters(true);
+if (activeSubClusters == null || activeSubClusters.isEmpty()) {
+ return 0;
+} else {
+ return
[hadoop] branch trunk updated: YARN-8262. get_executable in container-executor should provide meaningful error codes. Contributed by Susheel Gupta
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 3c37a01654e YARN-8262. get_executable in container-executor should
provide meaningful error codes. Contributed by Susheel Gupta
3c37a01654e is described below
commit 3c37a01654e0ddca3d88a9fe7ba7b0830ac44ea6
Author: Szilard Nemeth
AuthorDate: Tue Nov 22 13:37:55 2022 +0100
YARN-8262. get_executable in container-executor should provide meaningful
error codes. Contributed by Susheel Gupta
---
.../server/nodemanager/LinuxContainerExecutor.java | 8 ++--
.../container-executor/impl/get_executable.c | 22 +++---
.../src/main/native/container-executor/impl/util.c | 10 ++
.../src/main/native/container-executor/impl/util.h | 7 ++-
4 files changed, 33 insertions(+), 14 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index e899215291b..ea4595dffc4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -175,8 +175,12 @@ public class LinuxContainerExecutor extends
ContainerExecutor {
COULD_NOT_CREATE_WORK_DIRECTORIES(35),
COULD_NOT_CREATE_APP_LOG_DIRECTORIES(36),
COULD_NOT_CREATE_TMP_DIRECTORIES(37),
-ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS(38);
-
+ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS(38),
+CANNOT_GET_EXECUTABLE_NAME_FROM_READLINK(80),
+TOO_LONG_EXECUTOR_PATH(81),
+CANNOT_GET_EXECUTABLE_NAME_FROM_KERNEL(82),
+CANNOT_GET_EXECUTABLE_NAME_FROM_PID(83),
+WRONG_PATH_OF_EXECUTABLE(84);
private final int code;
ExitCode(int exitCode) {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
index e1ec293cd47..b027e51bd31 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
@@ -56,17 +56,17 @@ char *__get_exec_readproc(char *procfn) {
filename = malloc(EXECUTOR_PATH_MAX);
if (!filename) {
fprintf(ERRORFILE,"cannot allocate memory for filename before readlink:
%s\n",strerror(errno));
-exit(-1);
+exit(OUT_OF_MEMORY);
}
len = readlink(procfn, filename, EXECUTOR_PATH_MAX);
if (len == -1) {
-fprintf(ERRORFILE,"Can't get executable name from %s - %s\n", procfn,
+fprintf(ERRORFILE,"Cannot get executable name from %s - %s\n", procfn,
strerror(errno));
-exit(-1);
+exit(CANNOT_GET_EXECUTABLE_NAME_FROM_READLINK);
} else if (len >= EXECUTOR_PATH_MAX) {
fprintf(ERRORFILE,"Resolved path for %s [%s] is longer than %d
characters.\n",
procfn, filename, EXECUTOR_PATH_MAX);
-exit(-1);
+exit(TOO_LONG_EXECUTOR_PATH);
}
filename[len] = '\0';
return filename;
@@ -88,14 +88,14 @@ char *__get_exec_sysctl(int *mib)
len = sizeof(buffer);
if (sysctl(mib, 4, buffer, , NULL, 0) == -1) {
-fprintf(ERRORFILE,"Can't get executable name from kernel: %s\n",
+fprintf(ERRORFILE,"Cannot get executable name from kernel: %s\n",
strerror(errno));
-exit(-1);
+exit(CANNOT_GET_EXECUTABLE_NAME_FROM_KERNEL);
}
filename=malloc(EXECUTOR_PATH_MAX);
if (!filename) {
fprintf(ERRORFILE,"cannot allocate memory for filename after sysctl:
%s\n",strerror(errno));
-exit(-1);
+exit(OUT_OF_MEMORY);
}
snprintf(filename,EXECUTOR_PATH_MAX,"%s",buffer);
return filename;
@@ -120,13 +120,13 @@ char* get_executable(char *argv0) {
filename = malloc(PROC_PIDPATHINFO_MAXSIZE);
if (!filename) {
fprintf(ERRORFILE,"cannot allocate memory for filename before
proc_pidpath: %s\n",strerror(errno));
-exit(-1);
+exit(OUT_OF_MEMORY);
}
pid = getpid();
if (proc_pidpath(pid,filename,PROC_PIDPATHINFO_MAXSIZE) <= 0) {
-fprintf(ERRORFILE,"Can't get executable name from pid %u - %s\n", pid,
+fprintf(ERRORFILE,"Cannot get executable name from pid %u - %s\n", pid,
