(hadoop) branch trunk updated: YARN-11444. Improve YARN md documentation format. (#6711) Contributed by Shilun Fan.
This is an automated email from the ASF dual-hosted git repository. slfan1989 pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 8c378d1ea1d6 YARN-11444. Improve YARN md documentation format. (#6711) Contributed by Shilun Fan. 8c378d1ea1d6 is described below commit 8c378d1ea1d6026511981d6bda6c2b22bed03986 Author: slfan1989 <55643692+slfan1...@users.noreply.github.com> AuthorDate: Sun Apr 7 20:50:46 2024 +0800 YARN-11444. Improve YARN md documentation format. (#6711) Contributed by Shilun Fan. Reviewed-by: Ayush Saxena Signed-off-by: Shilun Fan --- .../hadoop-yarn-site/src/site/markdown/CapacityScheduler.md | 4 ++-- .../src/site/markdown/DevelopYourOwnDevicePlugin.md | 2 +- .../hadoop-yarn-site/src/site/markdown/DockerContainers.md | 6 +++--- .../hadoop-yarn-site/src/site/markdown/GracefulDecommission.md | 4 ++-- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md | 2 +- .../hadoop-yarn-site/src/site/markdown/NodeManagerCGroupsMemory.md | 2 +- .../hadoop-yarn-site/src/site/markdown/PluggableDeviceFramework.md | 2 +- .../hadoop-yarn-site/src/site/markdown/ReservationSystem.md | 2 +- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceModel.md | 2 +- .../hadoop-yarn-site/src/site/markdown/RuncContainers.md| 2 +- .../hadoop-yarn-site/src/site/markdown/TimelineServer.md| 6 +++--- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingGpus.md | 2 +- .../hadoop-yarn-site/src/site/markdown/WebServicesIntro.md | 6 +++--- .../hadoop-yarn-site/src/site/markdown/YarnApplicationSecurity.md | 2 +- 14 files changed, 22 insertions(+), 22 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index 1b94d6ff33e3..2b46c68bd7f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -633,7 +633,7 @@ The following configuration parameters can be configured in yarn-site.xml to con | `yarn.resourcemanager.reservation-system.planfollower.time-step` | *Optional* parameter: the frequency in milliseconds of the `PlanFollower` timer. Long value expected. The default value is *1000*. | -The `ReservationSystem` is integrated with the `CapacityScheduler` queue hierachy and can be configured for any **LeafQueue** currently. The `CapacityScheduler` supports the following parameters to tune the `ReservationSystem`: +The `ReservationSystem` is integrated with the `CapacityScheduler` queue hierarchy and can be configured for any **LeafQueue** currently. The `CapacityScheduler` supports the following parameters to tune the `ReservationSystem`: | Property | Description | |: |: | @@ -879,7 +879,7 @@ Changing queue/scheduler properties and adding/removing queues can be done in tw Remove the queue configurations from the file and run refresh as described above ### Enabling periodic configuration refresh -Enabling queue configuration periodic refresh allows reloading and applying the configuration by editing the *conf/capacity-scheduler.xml* without the necessicity of calling yarn rmadmin -refreshQueues. +Enabling queue configuration periodic refresh allows reloading and applying the configuration by editing the *conf/capacity-scheduler.xml* without the necessity of calling yarn rmadmin -refreshQueues. | Property | Description | |: |: | diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DevelopYourOwnDevicePlugin.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DevelopYourOwnDevicePlugin.md index 0331f7261531..e129c6c2e0bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DevelopYourOwnDevicePlugin.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DevelopYourOwnDevicePlugin.md @@ -173,5 +173,5 @@ class and want to give it a try in your Hadoop cluster. Firstly, put the jar file under a directory in Hadooop classpath. -(recommend $HADOOP_COMMOND_HOME/share/hadoop/yarn). Secondly, +(recommend $HADOOP_COMMAND_HOME/share/hadoop/yarn). Secondly, follow the configurations described in [Pluggable Device Framework](./PluggableDeviceFramework.html) and restart YARN. \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md index fe7d1e1ad39e..e512363d022b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md +++ b/hadoop-yarn-
(hadoop) branch branch-3.4 updated: HADOOP-19135. Remove Jcache 1.0-alpha. (#6695) Contributed by Shilun Fan.
This is an automated email from the ASF dual-hosted git repository.
slfan1989 pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new d02ca966ea31 HADOOP-19135. Remove Jcache 1.0-alpha. (#6695)
Contributed by Shilun Fan.
d02ca966ea31 is described below
commit d02ca966ea3161f8fa80ef0cab8439b6cbc2be6f
Author: slfan1989 <55643692+slfan1...@users.noreply.github.com>
AuthorDate: Fri Apr 5 22:09:15 2024 +0800
HADOOP-19135. Remove Jcache 1.0-alpha. (#6695) Contributed by Shilun Fan.
Reviewed-by: Steve Loughran
Signed-off-by: Shilun Fan
---
LICENSE-binary | 2 +-
hadoop-project/pom.xml | 12 ++--
.../hadoop-yarn-server/hadoop-yarn-server-common/pom.xml | 10 ++
3 files changed, 9 insertions(+), 15 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index 182938cfb933..fb910908c0de 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -310,7 +310,6 @@ org.apache.commons:commons-validator:1.6
org.apache.curator:curator-client:5.2.0
org.apache.curator:curator-framework:5.2.0
org.apache.curator:curator-recipes:5.2.0
-org.apache.geronimo.specs:geronimo-jcache_1.0_spec:1.0-alpha-1
org.apache.hbase:hbase-annotations:1.7.1
org.apache.hbase:hbase-client:1.7.1
org.apache.hbase:hbase-common:1.7.1
@@ -507,6 +506,7 @@ com.sun.jersey:jersey-server:1.19.4
com.sun.jersey:jersey-servlet:1.19.4
com.sun.xml.bind:jaxb-impl:2.2.3-1
javax.annotation:javax.annotation-api:1.3.2
+javax.cache:cache-api:1.1.1
javax.servlet:javax.servlet-api:3.1.0
javax.servlet.jsp:jsp-api:2.1
javax.websocket:javax.websocket-api:1.0
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 8ca757650579..702b9d9a1167 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -132,8 +132,8 @@
1.10.0
2.0.3
-1.0-alpha-1
3.8.2
+1.1.1
4.0.3
10.14.2.0
6.2.1.jre7
@@ -1943,11 +1943,6 @@
kerb-core
${kerby.version}
-
- org.apache.geronimo.specs
- geronimo-jcache_1.0_spec
- ${jcache.version}
-
org.ehcache
ehcache
@@ -2078,6 +2073,11 @@
log4j-web
${log4j2.version}
+
+javax.cache
+cache-api
+${cache.api.version}
+
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index 51f0b0960388..a6f9fd79dd6c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -124,14 +124,8 @@
leveldbjni-all
- org.apache.geronimo.specs
- geronimo-jcache_1.0_spec
-
-
- org.osgi
- org.osgi.core
-
-
+ javax.cache
+ cache-api
org.ehcache
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
(hadoop) branch branch-3.4 updated: YARN-11663. [Federation] Add Cache Entity Nums Limit. (#6662) Contributed by Shilun Fan.
This is an automated email from the ASF dual-hosted git repository.
slfan1989 pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 80662741870a YARN-11663. [Federation] Add Cache Entity Nums Limit.
(#6662) Contributed by Shilun Fan.
80662741870a is described below
commit 80662741870ae93493e0da53651dcd486cf13dd8
Author: slfan1989 <55643692+slfan1...@users.noreply.github.com>
AuthorDate: Tue Apr 2 07:47:59 2024 +0800
YARN-11663. [Federation] Add Cache Entity Nums Limit. (#6662) Contributed
by Shilun Fan.
Reviewed-by: Dinesh Chitlangia
Signed-off-by: Shilun Fan
---
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 4 ++
.../src/main/resources/yarn-default.xml| 9
.../federation/cache/FederationGuavaCache.java | 12 -
.../server/federation/cache/FederationJCache.java | 60 +++---
.../utils/TestFederationStateStoreFacade.java | 4 +-
5 files changed, 56 insertions(+), 33 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 0ab4107c1320..650e82d67381 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -4031,6 +4031,10 @@ public class YarnConfiguration extends Configuration {
// 5 minutes
public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
+ public static final String FEDERATION_CACHE_ENTITY_NUMS =
+ FEDERATION_PREFIX + "cache-entity.nums";
+ public static final int DEFAULT_FEDERATION_CACHE_ENTITY_NUMS = 1000;
+
public static final String FEDERATION_FLUSH_CACHE_FOR_RM_ADDR =
FEDERATION_PREFIX + "flush-cache-for-rm-addr";
public static final boolean DEFAULT_FEDERATION_FLUSH_CACHE_FOR_RM_ADDR =
true;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 41e38f601cbd..6b2d2cd817c6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3787,6 +3787,15 @@
300
+
+
+ The number of entries in the Federation cache.
+ default is 1000.
+
+yarn.federation.cache-entity.nums
+1000
+
+
The registry base directory for federation.
yarn.federation.registry.base-dir
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/cache/FederationGuavaCache.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/cache/FederationGuavaCache.java
index 5ab0ef77218d..2ba9e2869fe8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/cache/FederationGuavaCache.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/cache/FederationGuavaCache.java
@@ -27,15 +27,20 @@ import
org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import
org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.Map;
import java.util.concurrent.TimeUnit;
public class FederationGuavaCache extends FederationCache {
+ private static final Logger LOG =
LoggerFactory.getLogger(FederationCache.class);
+
private Cache> cache;
private int cacheTimeToLive;
+ private long cacheEntityNums;
private String className = this.getClass().getSimpleName();
@@ -52,6 +57,8 @@ public class FederationGuavaCache extends FederationCache {
// no conflict or pick up a specific one in the future.
cacheTimeToLive =
pConf.getInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS,
YarnConfiguration.DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS);
+cacheEntityNums =
pConf.getLong(YarnConfiguration.FEDERATION_CACHE_ENTITY_NUMS,
+YarnConfiguration.DEFAULT_FEDERATION_CACHE_ENTITY_NUMS);
if (cacheTimeToLive <= 0) {
isCachingEnabled = false;
return;
@@ -59,8 +66,11 @@ public class FederationGuavaCache extends Federa
