Repository: incubator-atlas Updated Branches: refs/heads/master ce20d6f59 -> 9db4d2619
ATLAS-1436: (Part 3) ignoreCache and query tuning for MetricsService Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/9db4d261 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/9db4d261 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/9db4d261 Branch: refs/heads/master Commit: 9db4d26191b4250b714fa3a9cd6d708cca172b83 Parents: ce20d6f Author: apoorvnaik <[email protected]> Authored: Sun Feb 12 23:51:41 2017 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Mon Feb 13 00:02:02 2017 -0800 ---------------------------------------------------------------------- .../apache/atlas/services/MetricsService.java | 27 +++----- .../atlas/services/MetricsServiceTest.java | 65 +++++--------------- .../atlas/web/resources/AdminResource.java | 4 +- 3 files changed, 26 insertions(+), 70 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9db4d261/repository/src/main/java/org/apache/atlas/services/MetricsService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/MetricsService.java b/repository/src/main/java/org/apache/atlas/services/MetricsService.java index e2cc369..7de9bd0 100644 --- a/repository/src/main/java/org/apache/atlas/services/MetricsService.java +++ b/repository/src/main/java/org/apache/atlas/services/MetricsService.java @@ -87,8 +87,8 @@ public class MetricsService { } @SuppressWarnings("unchecked") - public AtlasMetrics getMetrics() { - if (!isCacheValid()) { + public AtlasMetrics getMetrics(boolean ignoreCache) { + if (ignoreCache || !isCacheValid()) { AtlasMetrics metrics = new AtlasMetrics(); for (MetricQuery metricQuery : MetricQuery.values()) { @@ -96,18 +96,7 @@ public class MetricsService { if (LOG.isDebugEnabled()) { LOG.debug("Executing query: {}", metricQuery); } - - if (metricQuery == MetricQuery.ENTITIES_PER_TYPE) { - Collection<String> entityDefNames = atlasTypeRegistry.getAllEntityDefNames(); - - for (String entityDefName : entityDefNames) { - String formattedQuery = String.format(metricQuery.query, entityDefName); - - executeGremlinQuery(metrics, metricQuery.type, entityDefName, formattedQuery); - } - } else { - executeGremlinQuery(metrics, metricQuery.type, metricQuery.name, metricQuery.query); - } + executeGremlinQuery(metrics, metricQuery.type, metricQuery.name, metricQuery.query); } catch (ScriptException e) { LOG.error("Gremlin execution failed for metric {}", metricQuery, e); } @@ -130,8 +119,10 @@ public class MetricsService { if (result instanceof Number) { metrics.addData(type, name, ((Number) result).intValue()); } else if (result instanceof List) { - for (Map resultMap : (List<Map>) result) { - metrics.addData(type, (String) resultMap.get("key"), ((Number) resultMap.get("value")).intValue()); + for (Map<String, Number> resultMap : (List<Map<String, Number>>) result) { + for (Map.Entry<String, Number> entry : resultMap.entrySet()) { + metrics.addData(type, entry.getKey(), entry.getValue().intValue()); + } } } else { String returnClassName = result != null ? result.getClass().getSimpleName() : "null"; @@ -176,10 +167,10 @@ public class MetricsService { TAGS_COUNT(GENERAL, METRIC_TAG_COUNT, "g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() == 'TRAIT'}).count()"), DELETED_ENTITY_COUNT(GENERAL, METRIC_ENTITY_DELETED, "g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__status', 'DELETED').count()"), - ENTITIES_PER_TYPE(ENTITY, METRIC_TYPE_ENTITIES, "g.V().has('__typeName', T.in, ['%s']).count()"), + ENTITIES_PER_TYPE(ENTITY, METRIC_TYPE_ENTITIES, "g.V().has('__typeName', T.in, g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() != 'TRAIT'}).'__type.name'.toSet()).groupCount{it.'__typeName'}.cap.toList()"), TAGGED_ENTITIES(ENTITY, METRIC_TAGGED_ENTITIES, "g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__traitNames').count()"), - TAGS_PER_ENTITY(TAG, METRIC_TAGS_PER_ENTITY, "g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__traitNames').transform{[ key: it.'Referenceable.qualifiedName', value: it.'__traitNames'.size()]}.dedup().toList()"), + ENTITIES_WITH_SPECIFIC_TAG(TAG, METRIC_TAGS_PER_ENTITY, "g.V().has('__typeName', T.in, g.V().has('__type', 'typeSystem').filter{it.'__type.category'.name() == 'TRAIT'}.'__type.name'.toSet()).groupCount{it.'__typeName'}.cap.toList()"), ; private final String type; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9db4d261/repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java b/repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java index fef6b2f..353e106 100644 --- a/repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java @@ -1,23 +1,5 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.apache.atlas.services; -import org.apache.atlas.AtlasException; import org.apache.atlas.model.metrics.AtlasMetrics; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.type.AtlasTypeRegistry; @@ -50,37 +32,20 @@ public class MetricsServiceTest { private Number mockCount = 10; @BeforeClass - public void init() throws ScriptException, AtlasException { - Map<String, Object> aMockMap = new HashMap<>(); - Map<String, Object> bMockMap = new HashMap<>(); - Map<String, Object> cMockMap = new HashMap<>(); - aMockMap.put("key", "a"); - aMockMap.put("value", 1); - - bMockMap.put("key", "b"); - bMockMap.put("value", 2); - - cMockMap.put("key", "c"); - cMockMap.put("value", 3); - mockMapList.add(aMockMap); - mockMapList.add(bMockMap); - mockMapList.add(cMockMap); + public void init() throws ScriptException { + Map<String, Object> mockMap = new HashMap<>(); + mockMap.put("a", 1); + mockMap.put("b", 2); + mockMap.put("c", 3); + mockMapList.add(mockMap); when(mockConfig.getInt(anyString(), anyInt())).thenReturn(5); - when(mockConfig.getString(anyString(), anyString())) - // we have seven count queries so stubbing 7 counts - .thenReturn("dummyTestQuery.count()") - .thenReturn("dummyTestQuery.count()") - .thenReturn("dummyTestQuery.count()") - .thenReturn("dummyTestQuery.count()") - .thenReturn("dummyTestQuery.count()") - .thenReturn("dummyTestQuery.count()") - .thenReturn("dummyTestQuery.count()") - // The last query is a map - .thenReturn("dummyTestQuery"); assertEquals(mockConfig.getInt("test", 1), 5); + when(mockConfig.getString(anyString(), anyString())) + .thenReturn("count()", "count()", "count()", "count()", "count()", "toList()", "count()", "toList()"); when(mockTypeRegistry.getAllEntityDefNames()).thenReturn(Arrays.asList("a", "b", "c")); setupMockGraph(); + metricsService = new MetricsService(mockConfig, mockGraph, mockTypeRegistry); } @@ -101,19 +66,19 @@ public class MetricsServiceTest { @Test public void testGetMetrics() throws InterruptedException, ScriptException { assertNotNull(metricsService); - AtlasMetrics metrics = metricsService.getMetrics(); + AtlasMetrics metrics = metricsService.getMetrics(false); assertNotNull(metrics); Number aCount = metrics.getMetric("entity", "a"); assertNotNull(aCount); - assertEquals(aCount, 10); + assertEquals(aCount, 1); Number bCount = metrics.getMetric("entity", "b"); assertNotNull(bCount); - assertEquals(bCount, 10); + assertEquals(bCount, 2); Number cCount = metrics.getMetric("entity", "c"); assertNotNull(cCount); - assertEquals(cCount, 10); + assertEquals(cCount, 3); Number aTags = metrics.getMetric("tag", "a"); assertNotNull(aTags); @@ -130,12 +95,12 @@ public class MetricsServiceTest { verify(mockGraph, atLeastOnce()).executeGremlinScript(anyString(), anyBoolean()); // Subsequent call within the cache timeout window - metricsService.getMetrics(); + metricsService.getMetrics(false); verifyZeroInteractions(mockGraph); // Now test the cache refresh Thread.sleep(6000); - metricsService.getMetrics(); + metricsService.getMetrics(true); verify(mockGraph, atLeastOnce()).executeGremlinScript(anyString(), anyBoolean()); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9db4d261/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java index c1450c1..440d75c 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java @@ -254,12 +254,12 @@ public class AdminResource { @GET @Path("metrics") @Produces(Servlets.JSON_MEDIA_TYPE) - public AtlasMetrics getMetrics() { + public AtlasMetrics getMetrics(@QueryParam("ignoreCache") boolean ignoreCache) { if (LOG.isDebugEnabled()) { LOG.debug("==> AdminResource.getMetrics()"); } - AtlasMetrics metrics = metricsService.getMetrics(); + AtlasMetrics metrics = metricsService.getMetrics(ignoreCache); if (LOG.isDebugEnabled()) { LOG.debug("<== AdminResource.getMetrics()");
