Repository: atlas Updated Branches: refs/heads/master 85aef3c05 -> 4a938d873
ATLAS-2578: Update metric queries for faster execution Change-Id: Ic17ce74f8309fa61f9946d7295b1d161374ebe26 Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/4a938d87 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/4a938d87 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/4a938d87 Branch: refs/heads/master Commit: 4a938d873d63280d06678596af0a4f6c3f0a8255 Parents: 85aef3c Author: apoorvnaik <apoorvn...@apache.org> Authored: Wed Apr 11 20:30:02 2018 -0700 Committer: apoorvnaik <apoorvn...@apache.org> Committed: Wed Apr 18 07:54:04 2018 -0700 ---------------------------------------------------------------------- .../atlas/model/metrics/AtlasMetrics.java | 42 ++-- .../apache/atlas/services/MetricsService.java | 196 +++++++++++-------- .../atlas/util/AtlasGremlin2QueryProvider.java | 21 +- .../atlas/util/AtlasGremlin3QueryProvider.java | 8 +- .../atlas/util/AtlasGremlinQueryProvider.java | 2 +- .../atlas/services/MetricsServiceTest.java | 91 ++++----- .../atlas/web/resources/AdminResource.java | 18 +- 7 files changed, 201 insertions(+), 177 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/4a938d87/intg/src/main/java/org/apache/atlas/model/metrics/AtlasMetrics.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/metrics/AtlasMetrics.java b/intg/src/main/java/org/apache/atlas/model/metrics/AtlasMetrics.java index cfccc96..c3304cc 100644 --- a/intg/src/main/java/org/apache/atlas/model/metrics/AtlasMetrics.java +++ b/intg/src/main/java/org/apache/atlas/model/metrics/AtlasMetrics.java @@ -36,13 +36,13 @@ import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) @JsonIgnoreProperties(ignoreUnknown=true) public class AtlasMetrics { - private Map<String, Map<String, Number>> data; + private Map<String, Map<String, Object>> data; public AtlasMetrics() { setData(null); } - public AtlasMetrics(Map<String, Map<String, Number>> data) { + public AtlasMetrics(Map<String, Map<String, Object>> data) { setData(data); } @@ -52,41 +52,41 @@ public class AtlasMetrics { } } - public Map<String, Map<String, Number>> getData() { + public Map<String, Map<String, Object>> getData() { return data; } - public void setData(Map<String, Map<String, Number>> data) { + public void setData(Map<String, Map<String, Object>> data) { this.data = data; } @JsonIgnore - public void addData(String groupKey, String key, Number value) { - Map<String, Map<String, Number>> data = this.data; + public void addMetric(String groupKey, String key, Object value) { + Map<String, Map<String, Object>> data = this.data; if (data == null) { data = new HashMap<>(); } - Map<String, Number> metricMap = data.get(groupKey); - if (metricMap == null) { - metricMap = new HashMap<>(); - data.put(groupKey, metricMap); - } + Map<String, Object> metricMap = data.computeIfAbsent(groupKey, k -> new HashMap<>()); metricMap.put(key, value); setData(data); } @JsonIgnore - public Number getMetric(String groupKey, String key) { - Map<String, Map<String, Number>> data = this.data; - if (data == null) { - return null; - } else { - Map<String, Number> metricMap = data.get(groupKey); - if (metricMap == null || metricMap.isEmpty()) { - return null; - } else { - return metricMap.get(key); + public Number getNumericMetric(String groupKey, String key) { + Object metric = getMetric(groupKey, key); + return metric instanceof Number ? (Number) metric : null; + } + + @JsonIgnore + public Object getMetric(String groupKey, String key) { + Map<String, Map<String, Object>> data = this.data; + Object ret = null; + if (data != null) { + Map<String, Object> metricMap = data.get(groupKey); + if (metricMap != null && !metricMap.isEmpty()) { + ret = metricMap.get(key); } } + return ret; } } http://git-wip-us.apache.org/repos/asf/atlas/blob/4a938d87/repository/src/main/java/org/apache/atlas/services/MetricsService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/MetricsService.java b/repository/src/main/java/org/apache/atlas/services/MetricsService.java index 0fa68b2..bc05ed2 100644 --- a/repository/src/main/java/org/apache/atlas/services/MetricsService.java +++ b/repository/src/main/java/org/apache/atlas/services/MetricsService.java @@ -18,21 +18,25 @@ package org.apache.atlas.services; import com.google.common.annotations.VisibleForTesting; -import org.apache.atlas.ApplicationProperties; -import org.apache.atlas.AtlasException; import org.apache.atlas.annotation.AtlasService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.metrics.AtlasMetrics; import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.util.AtlasGremlinQueryProvider; import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.configuration.Configuration; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.inject.Inject; +import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @AtlasService public class MetricsService { @@ -51,6 +55,7 @@ public class MetricsService { protected static final String METRIC_ENTITY_COUNT = ENTITY + "Count"; protected static final String METRIC_ENTITY_DELETED = ENTITY + "Deleted"; + protected static final String METRIC_ENTITY_ACTIVE = ENTITY + "Active"; protected static final String METRIC_TAGGED_ENTITIES = ENTITY + "Tagged"; protected static final String METRIC_TAGS_PER_ENTITY = ENTITY + "Tags"; @@ -63,29 +68,30 @@ public class MetricsService { public static final String METRIC_COLLECTION_TIME = "collectionTime"; - private static Configuration configuration = null; + private static Configuration configuration = null; private static AtlasGremlinQueryProvider gremlinQueryProvider = null; - private final AtlasGraph atlasGraph; - private final int cacheTTLInSecs; + private final AtlasGraph atlasGraph; + private final AtlasTypeRegistry typeRegistry; + private final int cacheTTLInSecs; private AtlasMetrics cachedMetrics = null; private long cacheExpirationTime = 0; @Inject - public MetricsService(AtlasGraph atlasGraph) throws AtlasException { - this(ApplicationProperties.get(), atlasGraph); + public MetricsService(final Configuration configuration, final AtlasGraph graph, final AtlasTypeRegistry typeRegistry) { + this(configuration, graph, typeRegistry, AtlasGremlinQueryProvider.INSTANCE); } @VisibleForTesting - MetricsService(Configuration configuration, AtlasGraph graph) { + MetricsService(Configuration configuration, AtlasGraph graph, AtlasTypeRegistry typeRegistry, AtlasGremlinQueryProvider queryProvider) { MetricsService.configuration = configuration; - - atlasGraph = graph; - cacheTTLInSecs = configuration != null ? configuration.getInt(METRIC_QUERY_CACHE_TTL, DEFAULT_CACHE_TTL_IN_SECS) - : DEFAULT_CACHE_TTL_IN_SECS; - gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE; + atlasGraph = graph; + cacheTTLInSecs = configuration != null ? configuration.getInt(METRIC_QUERY_CACHE_TTL, DEFAULT_CACHE_TTL_IN_SECS) + : DEFAULT_CACHE_TTL_IN_SECS; + gremlinQueryProvider = queryProvider; + this.typeRegistry = typeRegistry; } @SuppressWarnings("unchecked") @@ -93,48 +99,120 @@ public class MetricsService { if (ignoreCache || !isCacheValid()) { AtlasMetrics metrics = new AtlasMetrics(); - for (MetricQuery metricQuery : MetricQuery.values()) { - try { - if (LOG.isDebugEnabled()) { - LOG.debug("Executing query: {}", metricQuery); - } - executeGremlinQuery(metrics, metricQuery.group, metricQuery.name, metricQuery.query); - } catch (AtlasBaseException e) { - if (LOG.isDebugEnabled()) { - LOG.debug("Gremlin execution failed for metric {}", metricQuery, e); - } else { - LOG.warn("Gremlin execution failed for metric {}", metricQuery); - } + int typeCount = 0, unusedTypeCount = 0; + + Collection<String> typeNames = typeRegistry.getAllTypeNames(); + if (CollectionUtils.isNotEmpty(typeNames)) { + typeCount = typeNames.size(); + } + metrics.addMetric(GENERAL, METRIC_TYPE_COUNT, typeCount); + + int tagCount = 0; + + Collection<String> classificationDefNames = typeRegistry.getAllClassificationDefNames() + .stream() + .map(x -> "'" + x + "'") + .collect(Collectors.toSet()); + String classificationNamesCSV = StringUtils.join(classificationDefNames, ','); + if (CollectionUtils.isNotEmpty(classificationDefNames)) { + tagCount = classificationDefNames.size(); + } + metrics.addMetric(GENERAL, METRIC_TAG_COUNT, tagCount); + + Collection<String> entityDefNames = typeRegistry.getAllEntityDefNames() + .stream() + .map(x -> "'" + x + "'") + .collect(Collectors.toList()); + String entityNamesCSV = StringUtils.join(entityDefNames, ','); + String entityAndClassificationNamesCSV = entityNamesCSV + "," + classificationNamesCSV; + + String query = String.format(gremlinQueryProvider.getQuery(AtlasGremlinQuery.ENTITY_ACTIVE_METRIC), entityAndClassificationNamesCSV); + Map<String, Number> activeCountMap = extractCounts(query); + + query = String.format(gremlinQueryProvider.getQuery(AtlasGremlinQuery.ENTITY_DELETED_METRIC), entityAndClassificationNamesCSV); + Map<String, Number> deletedCountMap = extractCounts(query); + + int totalEntities = 0; + + Map<String, Number> activeEntityCount = new HashMap<>(); + Map<String, Number> deletedEntityCount = new HashMap<>(); + + for (String entityDefName : typeRegistry.getAllEntityDefNames()) { + Number activeCount = activeCountMap.getOrDefault(entityDefName, null); + Number deletedCount = deletedCountMap.getOrDefault(entityDefName, null); + + if (activeCount != null) { + activeEntityCount.put(entityDefName, activeCount); + totalEntities += activeCount.intValue(); + } + if (deletedCount != null) { + deletedEntityCount.put(entityDefName, deletedCount); + totalEntities += deletedCount.intValue(); + } + if (activeCount == null && deletedCount == null) { + unusedTypeCount++; } } + metrics.addMetric(GENERAL, METRIC_TYPE_UNUSED_COUNT, unusedTypeCount); + metrics.addMetric(GENERAL, METRIC_ENTITY_COUNT, totalEntities); + metrics.addMetric(ENTITY, METRIC_ENTITY_ACTIVE, activeEntityCount); + metrics.addMetric(ENTITY, METRIC_ENTITY_DELETED, deletedEntityCount); + + Map<String, Number> taggedEntityCount = new HashMap<>(); + for (String classificationName : typeRegistry.getAllClassificationDefNames()) { + Object count = activeCountMap.getOrDefault(classificationName, null); + if (count != null) { + taggedEntityCount.put(classificationName, (Number) count); + } + } + metrics.addMetric(TAG, METRIC_ENTITIES_PER_TAG, taggedEntityCount); + + + // Miscellaneous metrics long collectionTime = System.currentTimeMillis(); - metrics.addData(GENERAL, METRIC_COLLECTION_TIME, collectionTime); + metrics.addMetric(GENERAL, METRIC_COLLECTION_TIME, collectionTime); - this.cachedMetrics = metrics; + this.cachedMetrics = metrics; this.cacheExpirationTime = (collectionTime + cacheTTLInSecs * 1000); } return cachedMetrics; } - private void executeGremlinQuery(AtlasMetrics metrics, String type, String name, String query) throws AtlasBaseException { - Object result = atlasGraph.executeGremlinScript(query, false); + private Map<String, Number> extractCounts(final String query) { + Map<String, Number> ret = new HashMap<>(); + try { + if (LOG.isDebugEnabled()) { + LOG.debug("Executing query: {}", query); + } - if (result instanceof Number) { - metrics.addData(type, name, ((Number) result).intValue()); - } else if (result instanceof List) { - for (Map<String, Number> resultMap : (List<Map<String, Number>>) result) { - for (Map.Entry<String, Number> entry : resultMap.entrySet()) { - metrics.addData(type, entry.getKey(), entry.getValue().intValue()); + Object result = executeQuery(query); + if (result instanceof List) { + for (Object entry : (List) result) { + if (entry instanceof Map) { + ret.putAll((Map<String, Number>) entry); + } } + } else if (result instanceof Map) { + ret.putAll((Map<String, Number>) result); + } else { + String returnClassName = result != null ? result.getClass().getSimpleName() : "null"; + LOG.warn("Unhandled return type {} for {}. Ignoring", returnClassName, query); + } + } catch (AtlasBaseException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Gremlin execution failed for metric {}", query, e); + } else { + LOG.warn("Gremlin execution failed for metric {}", query); } - } else { - String returnClassName = result != null ? result.getClass().getSimpleName() : "null"; - - LOG.warn("Unhandled return type {} for {}. Ignoring", returnClassName, query); } + return ret; + } + + private Object executeQuery(final String query) throws AtlasBaseException { + return atlasGraph.executeGremlinScript(query, false); } private boolean isCacheValid() { @@ -151,7 +229,7 @@ public class MetricsService { private static String getQuery(String type, String name, String defaultQuery) { String ret = configuration != null ? configuration.getString(METRIC_QUERY_PREFIX + type + "." + name, defaultQuery) - : defaultQuery; + : defaultQuery; if (LOG.isDebugEnabled()) { LOG.debug("query for {}.{}: {}", type, name, ret); @@ -159,42 +237,4 @@ public class MetricsService { return ret; } - - /** - * MetricQuery enum has the capability of reading the queries from the externalized config. - * - * The default behavior is to read from the properties and override the statically type query if the configured - * query is not blank/empty. - */ - private enum MetricQuery { - TYPE_COUNT(GENERAL, METRIC_TYPE_COUNT, AtlasGremlinQuery.TYPE_COUNT_METRIC), - UNUSED_TYPE_COUNT(GENERAL, METRIC_TYPE_UNUSED_COUNT, AtlasGremlinQuery.TYPE_UNUSED_COUNT_METRIC), - ENTITY_COUNT(GENERAL, METRIC_ENTITY_COUNT, AtlasGremlinQuery.ENTITY_COUNT_METRIC), - TAGS_COUNT(GENERAL, METRIC_TAG_COUNT, AtlasGremlinQuery.TAG_COUNT_METRIC), - DELETED_ENTITY_COUNT(GENERAL, METRIC_ENTITY_DELETED, AtlasGremlinQuery.ENTITY_DELETED_METRIC), - - ENTITIES_PER_TYPE(ENTITY, METRIC_TYPE_ENTITIES, AtlasGremlinQuery.ENTITIES_PER_TYPE_METRIC), - TAGGED_ENTITIES(ENTITY, METRIC_TAGGED_ENTITIES, AtlasGremlinQuery.TAGGED_ENTITIES_METRIC), - - ENTITIES_WITH_SPECIFIC_TAG(TAG, METRIC_ENTITIES_PER_TAG, AtlasGremlinQuery.ENTITIES_FOR_TAG_METRIC), - ; - - private final String group; - private final String name; - private final String query; - - MetricQuery(String group, String name, AtlasGremlinQuery gremlinQuery) { - this.group = group; - this.name = name; - this.query = MetricsService.getQuery(group, name, gremlinQueryProvider.getQuery(gremlinQuery)); - } - - @Override - public String toString() { - return "MetricQuery{" + "group='" + group + '\'' + - ", name='" + name + '\'' + - ", query='" + query + '\'' + - '}'; - } - } } http://git-wip-us.apache.org/repos/asf/atlas/blob/4a938d87/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java index ca6446b..82243ad 100644 --- a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java +++ b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java @@ -21,23 +21,12 @@ public class AtlasGremlin2QueryProvider extends AtlasGremlinQueryProvider { @Override public String getQuery(final AtlasGremlinQuery gremlinQuery) { switch (gremlinQuery) { - case TYPE_COUNT_METRIC: - return "g.V().has('__type', 'typeSystem').filter({!it.'__type.category'.name().matches('TRAIT')}).count()"; - case TYPE_UNUSED_COUNT_METRIC: - return "g.V('__type', 'typeSystem').filter({ !it.getProperty('__type.category').name().matches('TRAIT') && it.inE().count() == 0}).count()"; - case ENTITY_COUNT_METRIC: - return "g.V().has('__superTypeNames', T.in, ['Referenceable']).count()"; - case TAG_COUNT_METRIC: - return "g.V().has('__type', 'typeSystem').filter({it.getProperty('__type.category').name().matches('TRAIT')}).count()"; + case ENTITY_ACTIVE_METRIC: + return "g.V().has('__typeName', T.in, [%s]).has('__state', 'ACTIVE').groupCount{it.getProperty('__typeName')}.cap.toList()"; case ENTITY_DELETED_METRIC: - return "g.V().has('__typeName', T.in, g.V().has('__type', 'typeSystem').filter{it.getProperty('__type.category').name().matches('CLASS')}.'__type.name'.toSet()).has('__status', 'DELETED').count()"; - case ENTITIES_PER_TYPE_METRIC: - return "g.V().has('__typeName', T.in, g.V().has('__type', 'typeSystem').filter{it.getProperty('__type.category').name() == 'CLASS'}.'__type.name'.toSet()).groupCount{it.getProperty('__typeName')}.cap.toList()"; - case TAGGED_ENTITIES_METRIC: - return "g.V().has('__traitNames', T.in, g.V().has('__type', 'typeSystem').filter{it.getProperty('__type.category').name() == 'TRAIT'}.'__type.name'.toSet()).count()"; - case ENTITIES_FOR_TAG_METRIC: - return "g.V().has('__typeName', T.in, g.V().has('__type', 'typeSystem').filter{it.getProperty('__type.category').name() == 'TRAIT'}.'__type.name'.toSet()).groupCount{it.getProperty('__typeName')}.cap.toList()"; - case EXPORT_BY_GUID_FULL: + return "g.V().has('__typeName', T.in, [%s]).has('__state', 'DELETED').groupCount{it.getProperty('__typeName')}.cap.toList()"; + + case EXPORT_BY_GUID_FULL: return "g.V('__guid', startGuid).bothE().bothV().has('__guid').transform{[__guid:it.__guid,isProcess:(it.__superTypeNames != null) ? it.__superTypeNames.contains('Process') : false ]}.dedup().toList()"; case EXPORT_BY_GUID_CONNECTED_IN_EDGE: return "g.V('__guid', startGuid).inE().outV().has('__guid').transform{[__guid:it.__guid,isProcess:(it.__superTypeNames != null) ? it.__superTypeNames.contains('Process') : false ]}.dedup().toList()"; http://git-wip-us.apache.org/repos/asf/atlas/blob/4a938d87/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java index 7876672..e82c658 100644 --- a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java +++ b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java @@ -23,10 +23,10 @@ public class AtlasGremlin3QueryProvider extends AtlasGremlin2QueryProvider { // In case any overrides are necessary, a specific switch case can be added here to // return Gremlin 3 specific query otherwise delegate to super.getQuery switch (gremlinQuery) { - case TYPE_UNUSED_COUNT_METRIC: - return "g.V().has('__type', 'typeSystem').filter({ !it.getProperty('__type.category').name().matches('TRAIT') && it.inE().count() == 0}).count()"; - case ENTITY_COUNT_METRIC: - return "g.V().has('__superTypeNames', within(['Referenceable'])).count()"; + case ENTITY_ACTIVE_METRIC: + return "g.V().has('__typeName', within(%s)).has('__state', 'ACTIVE').groupCount().by('__typeName').toList()"; + case ENTITY_DELETED_METRIC: + return "g.V().has('__typeName', within(%s)).has('__state', 'DELETED').groupCount().by('__typeName').toList()"; case EXPORT_TYPE_STARTS_WITH: return "g.V().has('__typeName',typeName).filter({it.get().value(attrName).startsWith(attrValue)}).has('__guid').values('__guid').toList()"; case EXPORT_TYPE_ENDS_WITH: http://git-wip-us.apache.org/repos/asf/atlas/blob/4a938d87/repository/src/main/java/org/apache/atlas/util/AtlasGremlinQueryProvider.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/util/AtlasGremlinQueryProvider.java b/repository/src/main/java/org/apache/atlas/util/AtlasGremlinQueryProvider.java index b73ada2..e2992f1 100644 --- a/repository/src/main/java/org/apache/atlas/util/AtlasGremlinQueryProvider.java +++ b/repository/src/main/java/org/apache/atlas/util/AtlasGremlinQueryProvider.java @@ -34,8 +34,8 @@ public abstract class AtlasGremlinQueryProvider { // Metrics related Queries TYPE_COUNT_METRIC, TYPE_UNUSED_COUNT_METRIC, - ENTITY_COUNT_METRIC, TAG_COUNT_METRIC, + ENTITY_ACTIVE_METRIC, ENTITY_DELETED_METRIC, ENTITIES_PER_TYPE_METRIC, TAGGED_ENTITIES_METRIC, http://git-wip-us.apache.org/repos/asf/atlas/blob/4a938d87/repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java b/repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java index bdc0f03..a20b0fd 100644 --- a/repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java @@ -17,19 +17,16 @@ */ package org.apache.atlas.services; -import org.apache.atlas.TestModules; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.metrics.AtlasMetrics; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.runner.LocalSolrRunner; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.util.AtlasGremlin3QueryProvider; import org.apache.commons.configuration.Configuration; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import org.testng.SkipException; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; -import org.testng.annotations.Guice; import org.testng.annotations.Test; import java.util.ArrayList; @@ -45,34 +42,40 @@ import static org.mockito.Mockito.*; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; -@Guice(modules = TestModules.TestOnlyModule.class) public class MetricsServiceTest { - private Configuration mockConfig = mock(Configuration.class); - private AtlasTypeRegistry mockTypeRegistry = mock(AtlasTypeRegistry.class); - private AtlasGraph mockGraph = mock(AtlasGraph.class); - private MetricsService metricsService; + private Configuration mockConfig = mock(Configuration.class); + private AtlasTypeRegistry mockTypeRegistry = mock(AtlasTypeRegistry.class); + private AtlasGraph mockGraph = mock(AtlasGraph.class); + private MetricsService metricsService; - private List<Map> mockMapList = new ArrayList<>(); - private Number mockCount = 10; + private List<Map> activeEntityCountList = new ArrayList<>(); + private List<Map> deletedEntityCountList = new ArrayList<>(); @BeforeClass public void init() throws Exception { try { - Map<String, Object> mockMap = new HashMap<>(); - mockMap.put("a", 1); - mockMap.put("b", 2); - mockMap.put("c", 3); - mockMapList.add(mockMap); + Map<String, Object> activeEntityCount = new HashMap<>(); + activeEntityCount.put("a", 1); + activeEntityCount.put("b", 2); + activeEntityCount.put("d", 5); + activeEntityCount.put("e", 10); + activeEntityCount.put("f", 15); + activeEntityCountList.add(activeEntityCount); + + Map<String, Object> deletedEntityCount = new HashMap<>(); + deletedEntityCount.put("b", 5); + deletedEntityCountList.add(deletedEntityCount); when(mockConfig.getInt(anyString(), anyInt())).thenReturn(5); assertEquals(mockConfig.getInt("test", 1), 5); - when(mockConfig.getString(anyString(), anyString())) - .thenReturn("count()", "count()", "count()", "count()", "count()", "toList()", "count()", "toList()"); - when(mockTypeRegistry.getAllEntityDefNames()).thenReturn(Arrays.asList("a", "b", "c")); + when(mockConfig.getString(anyString(), anyString())).thenReturn("toList()", "toList()"); + when(mockTypeRegistry.getAllTypeNames()).thenReturn(Arrays.asList("a", "b", "c", "d", "e", "f")); when(mockTypeRegistry.getAllEntityDefNames()).thenReturn(Arrays.asList("a", "b", "c")); + when(mockTypeRegistry.getAllClassificationDefNames()).thenReturn(Arrays.asList("d", "e", "f")); +// when(mockTypeRegistry.getAllEntityDefNames()).thenReturn(Arrays.asList("a", "b", "c")); setupMockGraph(); - metricsService = new MetricsService(mockConfig, mockGraph); + metricsService = new MetricsService(mockConfig, mockGraph, mockTypeRegistry, new AtlasGremlin3QueryProvider()); } catch(Exception e) { throw new SkipException("MetricsServicesTest: init failed!", e); @@ -88,16 +91,9 @@ public class MetricsServiceTest { private void setupMockGraph() throws AtlasBaseException { if (mockGraph == null) mockGraph = mock(AtlasGraph.class); - when(mockGraph.executeGremlinScript(anyString(), eq(false))).thenAnswer(new Answer<Object>() { - @Override - public Object answer(InvocationOnMock invocationOnMock) throws Throwable { - if (((String)invocationOnMock.getArguments()[0]).contains("count()")) { - return mockCount; - } else { - return mockMapList; - } - } - }); + when(mockGraph.executeGremlinScript(anyString(), eq(false))) + .thenReturn(activeEntityCountList) + .thenReturn(deletedEntityCountList); } @Test @@ -105,29 +101,28 @@ public class MetricsServiceTest { assertNotNull(metricsService); AtlasMetrics metrics = metricsService.getMetrics(false); assertNotNull(metrics); - Number aCount = metrics.getMetric("entity", "a"); - assertNotNull(aCount); - assertEquals(aCount, 1); + Map activeMetrics = (Map) metrics.getMetric("entity", "entityActive"); + assertNotNull(activeMetrics); + assertEquals(activeMetrics.get("a"), 1); + assertEquals(activeMetrics.get("b"), 2); - Number bCount = metrics.getMetric("entity", "b"); - assertNotNull(bCount); - assertEquals(bCount, 2); + Map deletedMetrics = (Map) metrics.getMetric("entity", "entityDeleted"); + assertEquals(deletedMetrics.get("b"), 5); - Number cCount = metrics.getMetric("entity", "c"); - assertNotNull(cCount); - assertEquals(cCount, 3); + Number unusedTypeCount = metrics.getNumericMetric("general", "typeUnusedCount"); + assertEquals(unusedTypeCount, 1); - Number aTags = metrics.getMetric("tag", "a"); - assertNotNull(aTags); - assertEquals(aTags, 1); + Number cCount = metrics.getNumericMetric("general", "entityCount"); + assertEquals(cCount, 8); - Number bTags = metrics.getMetric("tag", "b"); - assertNotNull(bTags); - assertEquals(bTags, 2); + Number aTags = (Number) metrics.getMetric("general", "tagCount"); + assertEquals(aTags, 3); - Number cTags = metrics.getMetric("tag", "c"); - assertNotNull(cTags); - assertEquals(cTags, 3); + Map taggedEntityMetric = (Map) metrics.getMetric("tag", "tagEntities"); + assertNotNull(taggedEntityMetric); + assertEquals(taggedEntityMetric.get("d"), 5); + assertEquals(taggedEntityMetric.get("e"), 10); + assertEquals(taggedEntityMetric.get("f"), 15); verify(mockGraph, atLeastOnce()).executeGremlinScript(anyString(), anyBoolean()); http://git-wip-us.apache.org/repos/asf/atlas/blob/4a938d87/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java index b648bc1..5012d51 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java @@ -99,15 +99,15 @@ public class AdminResource { private Response version; - private final ServiceState serviceState; - private final MetricsService metricsService; - private static Configuration atlasProperties; - private final ExportService exportService; - private final ImportService importService; - private final SearchTracker activeSearches; - private final AtlasTypeRegistry typeRegistry; - private final MigrationProgressService migrationProgressService; - private final ReentrantLock importExportOperationLock; + private static Configuration atlasProperties; + private final ServiceState serviceState; + private final MetricsService metricsService; + private final ExportService exportService; + private final ImportService importService; + private final SearchTracker activeSearches; + private final AtlasTypeRegistry typeRegistry; + private final MigrationProgressService migrationProgressService; + private final ReentrantLock importExportOperationLock; static { try {
