This is an automated email from the ASF dual-hosted git repository.
slfan1989 pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9f6c997662c5 YARN-11471. [Federation] FederationStateStoreFacade Cache
Support Caffeine. (#6795) Contributed by Shilun Fan.
9f6c997662c5 is described below
commit 9f6c997662c5212bd7b542c1e5188ad4ede3f840
Author: slfan1989 <55643692+slfan1...@users.noreply.github.com>
AuthorDate: Sat Jun 1 06:15:20 2024 +0800
YARN-11471. [Federation] FederationStateStoreFacade Cache Support Caffeine.
(#6795) Contributed by Shilun Fan.
Reviewed-by: Inigo Goiri <inigo...@apache.org>
Signed-off-by: Shilun Fan <slfan1...@apache.org>
---
LICENSE-binary | 1 +
hadoop-project/pom.xml | 6 ++
.../hadoop-yarn-server-common/pom.xml | 14 ++++
...ionJCache.java => FederationCaffeineCache.java} | 86 ++++++++--------------
.../server/federation/cache/FederationJCache.java | 2 +-
.../federation/cache/TestFederationCache.java | 3 +-
.../pom.xml | 4 +
.../src/site/markdown/Federation.md | 8 +-
8 files changed, 66 insertions(+), 58 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index c0258e9311b1..32f9f06ae15d 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -226,6 +226,7 @@
com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.12.7
com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.12.7
com.fasterxml.uuid:java-uuid-generator:3.1.4
com.fasterxml.woodstox:woodstox-core:5.4.0
+com.github.ben-manes.caffeine:caffeine:2.9.3
com.github.davidmoten:rxjava-extras:0.8.0.17
com.github.stephenc.jcip:jcip-annotations:1.0-1
com.google:guice:4.0
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index ba7631189a1a..0345925e9994 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -134,6 +134,7 @@
<kerby.version>2.0.3</kerby.version>
<ehcache.version>3.8.2</ehcache.version>
<cache.api.version>1.1.1</cache.api.version>
+ <caffeine.version>2.9.3</caffeine.version>
<hikari.version>4.0.3</hikari.version>
<derby.version>10.14.2.0</derby.version>
<mssql.version>6.2.1.jre7</mssql.version>
@@ -1975,6 +1976,11 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ <artifactId>caffeine</artifactId>
+ <version>${caffeine.version}</version>
+ </dependency>
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index 1f762d31800d..e768ad5e4845 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -131,6 +131,20 @@
<groupId>org.ehcache</groupId>
<artifactId>ehcache</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ <artifactId>caffeine</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.checkerframework</groupId>
+ <artifactId>checker-qual</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.errorprone</groupId>
+ <artifactId>error_prone_annotations</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/cache/FederationJCache.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/cache/FederationCaffeineCache.java
similarity index 60%
copy from
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/cache/FederationJCache.java
copy to
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/cache/FederationCaffeineCache.java
index b4dbefe1278a..cbf3e9db3db3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/cache/FederationJCache.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/cache/FederationCaffeineCache.java
@@ -14,10 +14,10 @@
* License for the specific language governing permissions and limitations
under
* the License.
*/
-
package org.apache.hadoop.yarn.server.federation.cache;
-import org.apache.hadoop.classification.VisibleForTesting;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -26,31 +26,28 @@ import
org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import
org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
-import org.ehcache.Cache;
-import org.ehcache.CacheManager;
-import org.ehcache.config.builders.CacheConfigurationBuilder;
-import org.ehcache.config.builders.CacheManagerBuilder;
-import org.ehcache.config.builders.ExpiryPolicyBuilder;
-import org.ehcache.config.builders.ResourcePoolsBuilder;
-import org.ehcache.expiry.ExpiryPolicy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.time.Duration;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
-public class FederationJCache extends FederationCache {
+/**
+ * CaffeineCache is a high-performance caching library for Java, offering
better performance compared to Ehcache and Guava Cache.
+ * We are integrating this cache to store information about application and
homesubclusters etc.
+ */
+public class FederationCaffeineCache extends FederationCache {
- private static final Logger LOG =
LoggerFactory.getLogger(FederationJCache.class);
+ private static final Logger LOG =
LoggerFactory.getLogger(FederationCaffeineCache.class);
private Cache<String, CacheRequest> cache;
private int cacheTimeToLive;
private long cacheEntityNums;
- private boolean isCachingEnabled = false;
+ private String className = this.getClass().getSimpleName();
- private final String className = this.getClass().getSimpleName();
+ private boolean isCachingEnabled = false;
@Override
public boolean isCachingEnabled() {
@@ -59,48 +56,39 @@ public class FederationJCache extends FederationCache {
@Override
public void initCache(Configuration pConf, FederationStateStore pStateStore)
{
- // Picking the JCache provider from classpath, need to make sure there's
- // no conflict or pick up a specific one in the future
cacheTimeToLive =
pConf.getInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS,
YarnConfiguration.DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS);
cacheEntityNums =
pConf.getLong(YarnConfiguration.FEDERATION_CACHE_ENTITY_NUMS,
YarnConfiguration.DEFAULT_FEDERATION_CACHE_ENTITY_NUMS);
if (cacheTimeToLive <= 0) {
isCachingEnabled = false;
+ LOG.warn("Federation cache is not enabled. If we want to enable
federation cache, " +
+ "we need to set yarn.federation.cache-ttl.secs greater than 0.");
return;
}
this.setStateStore(pStateStore);
- CacheManager cacheManager =
CacheManagerBuilder.newCacheManagerBuilder().build(true);
-
- if (this.cache == null) {
- LOG.info("Creating a JCache Manager with name {}. " +
- "Cache TTL Time = {} secs. Cache Entity Nums = {}.", className,
cacheTimeToLive,
- cacheEntityNums);
- // Set the number of caches
- ResourcePoolsBuilder poolsBuilder =
ResourcePoolsBuilder.heap(cacheEntityNums);
- ExpiryPolicy expiryPolicy = ExpiryPolicyBuilder.timeToLiveExpiration(
- Duration.ofSeconds(cacheTimeToLive));
- CacheConfigurationBuilder<String, CacheRequest> configurationBuilder =
- CacheConfigurationBuilder.newCacheConfigurationBuilder(
- String.class, CacheRequest.class, poolsBuilder)
- .withExpiry(expiryPolicy);
- cache = cacheManager.createCache(className, configurationBuilder);
- }
- isCachingEnabled = true;
+
+ // Initialize Cache.
+ LOG.info("Creating a JCache Manager with name {}. " +
+ "Cache TTL Time = {} secs. Cache Entity Nums = {}.", className,
cacheTimeToLive,
+ cacheEntityNums);
+
+ this.cache = Caffeine.newBuilder().maximumSize(cacheEntityNums)
+ .expireAfterWrite(cacheTimeToLive, TimeUnit.SECONDS).build();
}
@Override
public void clearCache() {
-
+ this.cache.cleanUp();
this.cache = null;
}
@Override
- public Map<SubClusterId, SubClusterInfo> getSubClusters(boolean
filterInactiveSubClusters)
- throws YarnException {
+ public Map<SubClusterId, SubClusterInfo> getSubClusters(
+ boolean filterInactiveSubClusters) throws YarnException {
final String cacheKey = buildCacheKey(className, GET_SUBCLUSTERS_CACHEID,
- Boolean.toString(filterInactiveSubClusters));
- CacheRequest<String, ?> cacheRequest = cache.get(cacheKey);
+ Boolean.toString(filterInactiveSubClusters));
+ CacheRequest<String, ?> cacheRequest = cache.getIfPresent(cacheKey);
if (cacheRequest == null) {
cacheRequest = buildGetSubClustersCacheRequest(className,
filterInactiveSubClusters);
cache.put(cacheKey, cacheRequest);
@@ -112,7 +100,7 @@ public class FederationJCache extends FederationCache {
public Map<String, SubClusterPolicyConfiguration> getPoliciesConfigurations()
throws Exception {
final String cacheKey = buildCacheKey(className,
GET_POLICIES_CONFIGURATIONS_CACHEID);
- CacheRequest<String, ?> cacheRequest = cache.get(cacheKey);
+ CacheRequest<String, ?> cacheRequest = cache.getIfPresent(cacheKey);
if(cacheRequest == null){
cacheRequest = buildGetPoliciesConfigurationsCacheRequest(className);
cache.put(cacheKey, cacheRequest);
@@ -121,17 +109,16 @@ public class FederationJCache extends FederationCache {
}
@Override
- public SubClusterId getApplicationHomeSubCluster(ApplicationId appId)
- throws Exception {
+ public SubClusterId getApplicationHomeSubCluster(ApplicationId appId) throws
Exception {
final String cacheKey = buildCacheKey(className,
GET_APPLICATION_HOME_SUBCLUSTER_CACHEID,
appId.toString());
- CacheRequest<String, ?> cacheRequest = cache.get(cacheKey);
+ CacheRequest<String, ?> cacheRequest = cache.getIfPresent(cacheKey);
if (cacheRequest == null) {
cacheRequest = buildGetApplicationHomeSubClusterRequest(className,
appId);
cache.put(cacheKey, cacheRequest);
}
CacheResponse<SubClusterId> response =
-
ApplicationHomeSubClusterCacheResponse.class.cast(cacheRequest.getValue());
+
ApplicationHomeSubClusterCacheResponse.class.cast(cacheRequest.getValue());
return response.getItem();
}
@@ -139,17 +126,6 @@ public class FederationJCache extends FederationCache {
public void removeSubCluster(boolean flushCache) {
final String cacheKey = buildCacheKey(className, GET_SUBCLUSTERS_CACHEID,
Boolean.toString(flushCache));
- cache.remove(cacheKey);
- }
-
- @VisibleForTesting
- public Cache<String, CacheRequest> getCache() {
- return cache;
- }
-
- @VisibleForTesting
- public String getAppHomeSubClusterCacheKey(ApplicationId appId) {
- return buildCacheKey(className, GET_APPLICATION_HOME_SUBCLUSTER_CACHEID,
- appId.toString());
+ cache.invalidate(cacheKey);
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/cache/FederationJCache.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/cache/FederationJCache.java
index b4dbefe1278a..07f300e65f6b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/cache/FederationJCache.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/cache/FederationJCache.java
@@ -91,7 +91,7 @@ public class FederationJCache extends FederationCache {
@Override
public void clearCache() {
-
+ this.cache.clear();
this.cache = null;
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/cache/TestFederationCache.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/cache/TestFederationCache.java
index 8e0f15802bcc..8873e6093904 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/cache/TestFederationCache.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/cache/TestFederationCache.java
@@ -49,7 +49,8 @@ public class TestFederationCache {
@Parameterized.Parameters
public static Collection<Class[]> getParameters() {
- return Arrays.asList(new Class[][] {{FederationGuavaCache.class},
{FederationJCache.class}});
+ return Arrays.asList(new Class[][]{{FederationGuavaCache.class},
{FederationJCache.class},
+ {FederationCaffeineCache.class}});
}
private final long clusterTs = System.currentTimeMillis();
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml
index 7f3c711fe793..976c21e2dd2e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml
@@ -66,6 +66,10 @@
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</exclusion>
+ <exclusion>
+ <artifactId>error_prone_annotations</artifactId>
+ <groupId>com.google.errorprone</groupId>
+ </exclusion>
</exclusions>
</dependency>
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
index 3886f54041e6..fcb36c250c2c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
@@ -536,7 +536,7 @@ To enable cross-origin support (CORS) for the Yarn Router,
please set the follow
#### How to configure Router Cache
Cache is enabled by default. When we set the `yarn.federation.cache-ttl.secs`
parameter and its value is greater than 0, Cache will be enabled.
-We currently provide two Cache implementations: `JCache` and `GuavaCache`.
+We currently provide three Cache implementations: `JCache`, `GuavaCache`,
`CaffeineCache`
- JCache
@@ -550,6 +550,12 @@ If we want to use JCache, we can configure
`yarn.federation.cache.class` to `org
This is a Cache implemented based on the Guava framework.
If we want to use it, we can configure `yarn.federation.cache.class` to
`org.apache.hadoop.yarn.server.federation.cache.FederationGuavaCache`.
+- CaffeineCache
+
+[CaffeineCache](https://github.com/ben-manes/caffeine) is a high-performance
caching library for Java, offering better performance compared to Ehcache and
Guava Cache.
+If we want to use it, we can configure `yarn.federation.cache.class` to
`org.apache.hadoop.yarn.server.federation.cache.FederationCaffeineCache`.
+
+
#### How to configure Router AuditLog
We can enable the AuditLog configuration for the Router and collect the
AuditLog in a separate log file. We need to modify the configuration related to
RouterAuditLog in the **conf/log4j.properties** file.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
