This is an automated email from the ASF dual-hosted git repository. pinal pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push: new 2b794b2 ATLAS-4347 : DSL Search : Caching Mechanism for translated dsl queries 2b794b2 is described below commit 2b794b28dbeb9d297cab3d1fa6e7c2a0acfa6c48 Author: Pinal <pinal-shah> AuthorDate: Tue Jul 6 17:13:40 2021 +0530 ATLAS-4347 : DSL Search : Caching Mechanism for translated dsl queries Signed-off-by: Pinal <pinal-shah> --- .../java/org/apache/atlas/AtlasConfiguration.java | 1 + .../query/executors/TraversalBasedExecutor.java | 68 +++++++++++++++++----- 2 files changed, 56 insertions(+), 13 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 9ef8487..2f2c8a5 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -78,6 +78,7 @@ public enum AtlasConfiguration { PROCESS_NAME_UPDATE_PATCH("atlas.process.name.update.patch", false), STORE_DIFFERENTIAL_AUDITS("atlas.entity.audit.differential", false), DSL_EXECUTOR_TRAVERSAL("atlas.dsl.executor.traversal", true), + DSL_CACHED_TRANSLATOR("atlas.dsl.cached.translator", true), DEBUG_METRICS_ENABLED("atlas.debug.metrics.enabled", false), TASKS_USE_ENABLED("atlas.tasks.enabled", true); diff --git a/repository/src/main/java/org/apache/atlas/query/executors/TraversalBasedExecutor.java b/repository/src/main/java/org/apache/atlas/query/executors/TraversalBasedExecutor.java index 724cb82..ecd82e3 100644 --- a/repository/src/main/java/org/apache/atlas/query/executors/TraversalBasedExecutor.java +++ b/repository/src/main/java/org/apache/atlas/query/executors/TraversalBasedExecutor.java @@ -17,6 +17,7 @@ */ package org.apache.atlas.query.executors; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.query.AtlasDSL; @@ -28,6 +29,7 @@ import org.apache.atlas.repository.graphdb.AtlasGraphTraversal; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.LruCache; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.slf4j.Logger; @@ -44,10 +46,18 @@ public class TraversalBasedExecutor implements DSLQueryExecutor { private static final String DEFAULT_LIMIT_OFFSET_TEMPLATE = " limit %d offset %d"; private static final String CLAUSE_OFFSET_ZERO = " offset 0"; + private static final Translator translator; + private final AtlasTypeRegistry typeRegistry; private final AtlasGraph graph; private final EntityGraphRetriever entityRetriever; + static { + translator = AtlasConfiguration.DSL_CACHED_TRANSLATOR.getBoolean() + ? new CachedTranslator() + : new Translator(); + } + public TraversalBasedExecutor(AtlasTypeRegistry typeRegistry, AtlasGraph graph, EntityGraphRetriever entityRetriever) { this.typeRegistry = typeRegistry; this.graph = graph; @@ -106,28 +116,60 @@ public class TraversalBasedExecutor implements DSLQueryExecutor { } private GremlinQuery toTraversal(String query, int limit, int offset) throws AtlasBaseException { - QueryParams params = QueryParams.getNormalizedParams(limit, offset); - - query = getStringWithLimitOffset(query, params); - - AtlasDSL.Translator dslTranslator = new AtlasDSL.Translator(query, typeRegistry, params.offset(), params.limit()); - GremlinQuery gremlinQuery = dslTranslator.translate(); - AtlasGraphTraversal result = GremlinClauseToTraversalTranslator.run(this.graph, gremlinQuery.getClauses()); + GremlinQuery gremlinQuery = translator.translate(typeRegistry, query, limit, offset); + AtlasGraphTraversal result = GremlinClauseToTraversalTranslator.run(this.graph, gremlinQuery.getClauses()); gremlinQuery.setResult(result); return gremlinQuery; } - private String getStringWithLimitOffset(String query, QueryParams params) { - if (!query.contains(DSL_KEYWORD_LIMIT) && !query.contains(DSL_KEYWORD_OFFSET)) { - query += String.format(DEFAULT_LIMIT_OFFSET_TEMPLATE, params.limit(), params.offset()); + private static class Translator { + + public Translator() { + } + + public GremlinQuery translate(AtlasTypeRegistry typeRegistry, String query, int limit, int offset) throws AtlasBaseException { + QueryParams params = QueryParams.getNormalizedParams(limit, offset); + query = getStringWithLimitOffset(query, params); + + AtlasDSL.Translator dslTranslator = new AtlasDSL.Translator(query, typeRegistry, params.offset(), params.limit()); + GremlinQuery gremlinQuery = dslTranslator.translate(); + + return gremlinQuery; + } + + private String getStringWithLimitOffset(String query, QueryParams params) { + if (!query.contains(DSL_KEYWORD_LIMIT) && !query.contains(DSL_KEYWORD_OFFSET)) { + query += String.format(DEFAULT_LIMIT_OFFSET_TEMPLATE, params.limit(), params.offset()); + } + + if (query.contains(DSL_KEYWORD_LIMIT) && !query.contains(DSL_KEYWORD_OFFSET)) { + query += CLAUSE_OFFSET_ZERO; + } + + return query; } + } - if (query.contains(DSL_KEYWORD_LIMIT) && !query.contains(DSL_KEYWORD_OFFSET)) { - query += CLAUSE_OFFSET_ZERO; + private static class CachedTranslator extends Translator { + private final static int DSLQUERY_CACHE_SIZE = 100; + private final Map<String, GremlinQuery> cachedQuery; + + public CachedTranslator() { + this.cachedQuery = new LruCache<>(DSLQUERY_CACHE_SIZE, 100000); } - return query; + @Override + public GremlinQuery translate(AtlasTypeRegistry typeRegistry, String query, int limit, int offset) throws AtlasBaseException { + String key = String.format("%s-%s-%s", query, limit, offset); + + if (!cachedQuery.containsKey(key)) { + GremlinQuery gremlinQuery = super.translate(typeRegistry, query, limit, offset); + cachedQuery.put(key, gremlinQuery); + } + + return cachedQuery.get(key); + } } }