Repository: incubator-atlas Updated Branches: refs/heads/master 34d235f39 -> 10e82ff47
ATLAS-1471: updated logging for better troubleshooting and reduce overhead Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/10e82ff4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/10e82ff4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/10e82ff4 Branch: refs/heads/master Commit: 10e82ff47c915bb7f3d750b9fbc398b2dd04f88d Parents: 34d235f Author: Madhan Neethiraj <[email protected]> Authored: Wed Jan 18 17:28:13 2017 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Wed Jan 18 18:35:40 2017 -0800 ---------------------------------------------------------------------- .../atlas/repository/graph/FullTextMapper.java | 2 - .../graph/GraphToTypedInstanceMapper.java | 4 - .../graph/TypedInstanceToGraphMapper.java | 4 - .../org/apache/atlas/MonitoredAspectTest.java | 47 --- typesystem/src/main/resources/atlas-log4j.xml | 16 + .../atlas/web/resources/AdminResource.java | 70 +++- .../web/resources/DataSetLineageResource.java | 40 ++- .../atlas/web/resources/EntityResource.java | 351 ++++++++++++++++--- .../atlas/web/resources/LineageResource.java | 54 ++- .../resources/MetadataDiscoveryResource.java | 56 ++- .../atlas/web/resources/TypesResource.java | 79 ++++- 11 files changed, 571 insertions(+), 152 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/10e82ff4/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java index 0d7487e..911b1ad 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java @@ -18,7 +18,6 @@ package org.apache.atlas.repository.graph; import org.apache.atlas.AtlasException; -import org.apache.atlas.aspect.Monitored; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.typesystem.ITypedInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance; @@ -50,7 +49,6 @@ public class FullTextMapper { instanceCache = new HashMap<>(); } - @Monitored public String mapRecursive(AtlasVertex instanceVertex, boolean followReferences) throws AtlasException { String guid = GraphHelper.getGuid(instanceVertex); ITypedReferenceableInstance typedReference; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/10e82ff4/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java index 4220c35..7b2b753 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java @@ -19,7 +19,6 @@ package org.apache.atlas.repository.graph; import com.google.inject.Singleton; import org.apache.atlas.AtlasException; -import org.apache.atlas.aspect.Monitored; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; @@ -66,7 +65,6 @@ public final class GraphToTypedInstanceMapper { this.graph = graph; } - @Monitored public ITypedReferenceableInstance mapGraphToTypedInstance(String guid, AtlasVertex instanceVertex) throws AtlasException { @@ -104,7 +102,6 @@ public final class GraphToTypedInstanceMapper { return typedInstance; } - @Monitored private void mapVertexToInstanceTraits(AtlasVertex instanceVertex, ITypedReferenceableInstance typedInstance, List<String> traits) throws AtlasException { for (String traitName : traits) { @@ -117,7 +114,6 @@ public final class GraphToTypedInstanceMapper { } } - @Monitored public void mapVertexToInstance(AtlasVertex instanceVertex, ITypedInstance typedInstance, Map<String, AttributeInfo> fields) throws AtlasException { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/10e82ff4/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java index a6000cd..4e55bbc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java @@ -20,7 +20,6 @@ package org.apache.atlas.repository.graph; import com.google.inject.Inject; import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContext; -import org.apache.atlas.aspect.Monitored; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.RepositoryException; import org.apache.atlas.repository.graphdb.AtlasEdge; @@ -85,7 +84,6 @@ public final class TypedInstanceToGraphMapper { UPDATE_FULL } - @Monitored void mapTypedInstanceToGraph(Operation operation, ITypedReferenceableInstance... typedInstances) throws AtlasException { RequestContext requestContext = RequestContext.get(); @@ -156,7 +154,6 @@ public final class TypedInstanceToGraphMapper { return guids; } - @Monitored private String addOrUpdateAttributesAndTraits(Operation operation, ITypedReferenceableInstance typedInstance) throws AtlasException { if (LOG.isDebugEnabled()) { @@ -254,7 +251,6 @@ public final class TypedInstanceToGraphMapper { } } - @Monitored private TypeUtils.Pair<List<ITypedReferenceableInstance>, List<ITypedReferenceableInstance>> createVerticesAndDiscoverInstances( Collection<IReferenceableInstance> instances) throws AtlasException { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/10e82ff4/server-api/src/test/aspect/org/apache/atlas/MonitoredAspectTest.java ---------------------------------------------------------------------- diff --git a/server-api/src/test/aspect/org/apache/atlas/MonitoredAspectTest.java b/server-api/src/test/aspect/org/apache/atlas/MonitoredAspectTest.java deleted file mode 100644 index a53d97c..0000000 --- a/server-api/src/test/aspect/org/apache/atlas/MonitoredAspectTest.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas; - -import org.apache.atlas.aspect.Monitored; -import org.apache.atlas.metrics.Metrics; -import org.testng.annotations.Test; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; - -public class MonitoredAspectTest { - - @Monitored - public void monitoredMethod() throws InterruptedException { - Thread.sleep(1); - } - - @Test - public void testMonitoredAspect() throws Exception { - RequestContext.clear(); - monitoredMethod(); - - Metrics metrics = RequestContext.getMetrics(); - Metrics.Counters counters = metrics.getCounters("MonitoredAspectTest.monitoredMethod"); - assertNotNull(counters); - assertEquals(counters.getInvocations(), 1); - assertTrue(counters.getTotalTimeMSecs() > 0); - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/10e82ff4/typesystem/src/main/resources/atlas-log4j.xml ---------------------------------------------------------------------- diff --git a/typesystem/src/main/resources/atlas-log4j.xml b/typesystem/src/main/resources/atlas-log4j.xml index a58898c..8312657 100755 --- a/typesystem/src/main/resources/atlas-log4j.xml +++ b/typesystem/src/main/resources/atlas-log4j.xml @@ -41,6 +41,22 @@ <appender-ref ref="console"/> </logger> + <!-- uncomment this block to generate performance traces + <appender name="perf_appender" class="org.apache.log4j.DailyRollingFileAppender"> + <param name="file" value="${atlas.log.dir}/atlas_perf.log" /> + <param name="datePattern" value="'.'yyyy-MM-dd" /> + <param name="append" value="true" /> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d|%t|%m%n" /> + </layout> + </appender> + + <logger name="org.apache.atlas.perf" additivity="false"> + <level value="debug" /> + <appender-ref ref="perf_appender" /> + </logger> + --> + <appender name="FAILED" class="org.apache.log4j.DailyRollingFileAppender"> <param name="File" value="${atlas.log.dir}/failed.log"/> <param name="Append" value="true"/> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/10e82ff4/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java index c20e978..02d7488 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java @@ -20,8 +20,6 @@ package org.apache.atlas.web.resources; import com.google.inject.Inject; import org.apache.atlas.AtlasClient; -import org.apache.atlas.aspect.Monitored; -import org.apache.atlas.utils.AtlasPerfTracer; import org.apache.atlas.web.filters.AtlasCSRFPreventionFilter; import org.apache.atlas.web.service.ServiceState; import org.apache.atlas.web.util.Servlets; @@ -31,6 +29,7 @@ import org.apache.commons.lang.StringUtils; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.security.core.Authentication; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.core.context.SecurityContextHolder; @@ -52,7 +51,7 @@ import java.util.Set; @Path("admin") @Singleton public class AdminResource { - private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.AdminResource"); + private static final Logger LOG = LoggerFactory.getLogger(AdminResource.class); private static final String isCSRF_ENABLED = "atlas.rest-csrf.enabled"; private static final String BROWSER_USER_AGENT_PARAM = "atlas.rest-csrf.browser-useragents-regex"; @@ -73,11 +72,14 @@ public class AdminResource { * * @return json representing the thread stack dump. */ - @Monitored @GET @Path("stack") @Produces(MediaType.TEXT_PLAIN) public String getThreadDump() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AdminResource.getThreadDump()"); + } + ThreadGroup topThreadGroup = Thread.currentThread().getThreadGroup(); while (topThreadGroup.getParent() != null) { @@ -93,6 +95,11 @@ public class AdminResource { String stackTrace = StringUtils.join(threads[i].getStackTrace(), "\n"); builder.append(stackTrace); } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AdminResource.getThreadDump()"); + } + return builder.toString(); } @@ -101,11 +108,14 @@ public class AdminResource { * * @return json representing the version. */ - @Monitored @GET @Path("version") @Produces(Servlets.JSON_MEDIA_TYPE) public Response getVersion() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AdminResource.getVersion()"); + } + if (version == null) { try { PropertiesConfiguration configProperties = new PropertiesConfiguration("atlas-buildinfo.properties"); @@ -124,34 +134,51 @@ public class AdminResource { } } + if (LOG.isDebugEnabled()) { + LOG.debug("<== AdminResource.getVersion()"); + } + return version; } - @Monitored @GET @Path("status") @Produces(Servlets.JSON_MEDIA_TYPE) public Response getStatus() { - JSONObject responseData = new JSONObject(); + if (LOG.isDebugEnabled()) { + LOG.debug("==> AdminResource.getStatus()"); + } + + Response response; + try { + JSONObject responseData = new JSONObject(); responseData.put(AtlasClient.STATUS, serviceState.getState().toString()); - Response response = Response.ok(responseData).build(); - return response; + response = Response.ok(responseData).build(); } catch (JSONException e) { throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AdminResource.getStatus()"); + } + + return response; } - @Monitored @GET @Path("session") @Produces(Servlets.JSON_MEDIA_TYPE) public Response getUserProfile() { - JSONObject responseData = new JSONObject(); - Boolean enableTaxonomy = null; + if (LOG.isDebugEnabled()) { + LOG.debug("==> AdminResource.getUserProfile()"); + } + + Response response; + try { PropertiesConfiguration configProperties = new PropertiesConfiguration("atlas-application.properties"); - enableTaxonomy = configProperties.getBoolean(isTaxonomyEnabled, false); + Boolean enableTaxonomy = configProperties.getBoolean(isTaxonomyEnabled, false); Authentication auth = SecurityContextHolder.getContext().getAuthentication(); String userName = null; Set<String> groups = new HashSet<>(); @@ -161,20 +188,27 @@ public class AdminResource { for (GrantedAuthority c : authorities) { groups.add(c.getAuthority()); } - } - + } + + JSONObject responseData = new JSONObject(); + responseData.put(isCSRF_ENABLED, AtlasCSRFPreventionFilter.isCSRF_ENABLED); responseData.put(BROWSER_USER_AGENT_PARAM, AtlasCSRFPreventionFilter.BROWSER_USER_AGENTS_DEFAULT); responseData.put(CUSTOM_METHODS_TO_IGNORE_PARAM, AtlasCSRFPreventionFilter.METHODS_TO_IGNORE_DEFAULT); responseData.put(CUSTOM_HEADER_PARAM, AtlasCSRFPreventionFilter.HEADER_DEFAULT); - responseData.put(isTaxonomyEnabled, enableTaxonomy); - responseData.put("userName", userName); responseData.put("groups", groups); - return Response.ok(responseData).build(); + + response = Response.ok(responseData).build(); } catch (JSONException | ConfigurationException e) { throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AdminResource.getUserProfile()"); + } + + return response; } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/10e82ff4/webapp/src/main/java/org/apache/atlas/web/resources/DataSetLineageResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/DataSetLineageResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/DataSetLineageResource.java index 8192b07..c3799cb 100644 --- a/webapp/src/main/java/org/apache/atlas/web/resources/DataSetLineageResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/DataSetLineageResource.java @@ -19,7 +19,6 @@ package org.apache.atlas.web.resources; import org.apache.atlas.AtlasClient; -import org.apache.atlas.aspect.Monitored; import org.apache.atlas.discovery.DiscoveryException; import org.apache.atlas.discovery.LineageService; import org.apache.atlas.typesystem.exception.EntityNotFoundException; @@ -69,15 +68,22 @@ public class DataSetLineageResource { * * @param tableName table name */ - @Monitored @GET @Path("table/{tableName}/inputs/graph") @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) public Response inputsGraph(@Context HttpServletRequest request, @PathParam("tableName") String tableName) { - LOG.info("Fetching lineage inputs graph for tableName={}", tableName); + if (LOG.isDebugEnabled()) { + LOG.debug("==> DataSetLineageResource.inputsGraph({})", tableName); + } + + AtlasPerfTracer perf = null; try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DataSetLineageResource.inputsGraph(tableName=" + tableName + ")"); + } + final String jsonResult = lineageService.getInputsGraph(tableName); JSONObject response = new JSONObject(); @@ -95,6 +101,8 @@ public class DataSetLineageResource { } catch (Throwable e) { LOG.error("Unable to get lineage inputs graph for table {}", tableName, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); } } @@ -103,15 +111,22 @@ public class DataSetLineageResource { * * @param tableName table name */ - @Monitored @GET @Path("table/{tableName}/outputs/graph") @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) public Response outputsGraph(@Context HttpServletRequest request, @PathParam("tableName") String tableName) { - LOG.info("Fetching lineage outputs graph for tableName={}", tableName); + if (LOG.isDebugEnabled()) { + LOG.debug("==> DataSetLineageResource.outputsGraph({})", tableName); + } + + AtlasPerfTracer perf = null; try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DataSetLineageResource.outputsGraph(tableName=" + tableName + ")"); + } + final String jsonResult = lineageService.getOutputsGraph(tableName); JSONObject response = new JSONObject(); @@ -129,6 +144,8 @@ public class DataSetLineageResource { } catch (Throwable e) { LOG.error("Unable to get lineage outputs graph for table {}", tableName, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); } } @@ -137,15 +154,22 @@ public class DataSetLineageResource { * * @param tableName table name */ - @Monitored @GET @Path("table/{tableName}/schema") @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) public Response schema(@Context HttpServletRequest request, @PathParam("tableName") String tableName) { - LOG.info("Fetching schema for tableName={}", tableName); + if (LOG.isDebugEnabled()) { + LOG.debug("==> DataSetLineageResource.schema({})", tableName); + } + + AtlasPerfTracer perf = null; try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DataSetLineageResource.schema(tableName=" + tableName + ")"); + } + final String jsonResult = lineageService.getSchema(tableName); JSONObject response = new JSONObject(); @@ -163,6 +187,8 @@ public class DataSetLineageResource { } catch (Throwable e) { LOG.error("Unable to get schema for table {}", tableName, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/10e82ff4/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java index 7d65d97..17c8237 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java @@ -24,7 +24,6 @@ import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasConstants; import org.apache.atlas.AtlasException; import org.apache.atlas.EntityAuditEvent; -import org.apache.atlas.aspect.Monitored; import org.apache.atlas.services.MetadataService; import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.Referenceable; @@ -95,13 +94,21 @@ public class EntityResource { * The body contains the JSONArray of entity json. The service takes care of de-duping the entities based on any * unique attribute for the give type. */ - @Monitored @POST @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) @Produces(Servlets.JSON_MEDIA_TYPE) public Response submit(@Context HttpServletRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> EntityResource.submit()"); + } + String entityJson = null; + AtlasPerfTracer perf = null; try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.submit()"); + } + String entities = Servlets.getRequestPayload(request); //Handle backward compatibility - if entities is not JSONArray, convert to JSONArray @@ -115,10 +122,17 @@ public class EntityResource { } entityJson = AtlasClient.toString(new JSONArray(entities)); - LOG.info("submitting entities {} ", entityJson); + + if (LOG.isDebugEnabled()) { + LOG.debug("submitting entities {} ", entityJson); + } final List<String> guids = metadataService.createEntities(entities); - LOG.info("Created entities {}", guids); + + if (LOG.isDebugEnabled()) { + LOG.debug("Created entities {}", guids); + } + JSONObject response = getResponse(new AtlasClient.EntityResult(guids, null, null)); URI locationURI = getLocationURI(guids); @@ -137,6 +151,13 @@ public class EntityResource { } catch (Throwable e) { LOG.error("Unable to persist entity instance entityDef={}", entityJson, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== EntityResource.submit()"); + } + } } @@ -174,20 +195,34 @@ public class EntityResource { * Adds/Updates given entities identified by its GUID or unique attribute * @return response payload as json */ - @Monitored @PUT @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) @Produces(Servlets.JSON_MEDIA_TYPE) public Response updateEntities(@Context HttpServletRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> EntityResource.updateEntities()"); + } + String entityJson = null; + AtlasPerfTracer perf = null; try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.updateEntities()"); + } + final String entities = Servlets.getRequestPayload(request); entityJson = AtlasClient.toString(new JSONArray(entities)); - LOG.info("updating entities {} ", entityJson); + + if (LOG.isDebugEnabled()) { + LOG.info("updating entities {} ", entityJson); + } AtlasClient.EntityResult entityResult = metadataService.updateEntities(entities); - LOG.info("Updated entities: {}", entityResult); + + if (LOG.isDebugEnabled()) { + LOG.debug("Updated entities: {}", entityResult); + } JSONObject response = getResponse(entityResult); return Response.ok(response).build(); @@ -203,6 +238,12 @@ public class EntityResource { } catch (Throwable e) { LOG.error("Unable to persist entity instance entityDef={}", entityJson, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== EntityResource.updateEntities()"); + } } } @@ -235,7 +276,6 @@ public class EntityResource { * The body contains the JSONArray of entity json. The service takes care of de-duping the entities based on any * unique attribute for the give type. */ - @Monitored @POST @Path("qualifiedName") @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) @@ -243,18 +283,32 @@ public class EntityResource { public Response updateByUniqueAttribute(@QueryParam("type") String entityType, @QueryParam("property") String attribute, @QueryParam("value") String value, @Context HttpServletRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> EntityResource.updateByUniqueAttribute({}, {}, {})", entityType, attribute, value); + } + + AtlasPerfTracer perf = null; String entityJson = null; try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.updateByUniqueAttribute(" + entityType + ", " + attribute + ", " + value + ")"); + } + entityJson = Servlets.getRequestPayload(request); - LOG.info("Partially updating entity by unique attribute {} {} {} {} ", entityType, attribute, value, entityJson); + if (LOG.isDebugEnabled()) { + LOG.debug("Partially updating entity by unique attribute {} {} {} {} ", entityType, attribute, value, entityJson); + } Referenceable updatedEntity = InstanceSerialization.fromJsonReferenceable(entityJson, true); AtlasClient.EntityResult entityResult = metadataService.updateEntityByUniqueAttribute(entityType, attribute, value, updatedEntity); - LOG.info("Updated entities: {}", entityResult); + + if (LOG.isDebugEnabled()) { + LOG.debug("Updated entities: {}", entityResult); + } JSONObject response = getResponse(entityResult); return Response.ok(response).build(); @@ -273,6 +327,12 @@ public class EntityResource { } catch (Throwable e) { LOG.error("Unable to partially update entity {} {}:{}.{}", entityJson, entityType, attribute, value, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== EntityResource.updateByUniqueAttribute({}, {}, {})", entityType, attribute, value); + } } } @@ -291,26 +351,48 @@ public class EntityResource { @Produces(Servlets.JSON_MEDIA_TYPE) public Response updateEntityByGuid(@PathParam("guid") String guid, @QueryParam("property") String attribute, @Context HttpServletRequest request) { - if (StringUtils.isEmpty(attribute)) { - return updateEntityPartialByGuid(guid, request); - } else { - return updateEntityAttributeByGuid(guid, attribute, request); + if (LOG.isDebugEnabled()) { + LOG.debug("==> EntityResource.updateEntityByGuid({}, {})", guid, attribute); + } + + AtlasPerfTracer perf = null; + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.updateEntityByGuid(" + guid + ", " + attribute + ")"); + } + + if (StringUtils.isEmpty(attribute)) { + return updateEntityPartialByGuid(guid, request); + } else { + return updateEntityAttributeByGuid(guid, attribute, request); + } + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== EntityResource.updateEntityByGuid({}, {})", guid, attribute); + } } } - @Monitored private Response updateEntityPartialByGuid(String guid, HttpServletRequest request) { String entityJson = null; try { guid = ParamChecker.notEmpty(guid, "Guid property cannot be null"); entityJson = Servlets.getRequestPayload(request); - LOG.info("partially updating entity for guid {} : {} ", guid, entityJson); + + if (LOG.isDebugEnabled()) { + LOG.debug("partially updating entity for guid {} : {} ", guid, entityJson); + } Referenceable updatedEntity = InstanceSerialization.fromJsonReferenceable(entityJson, true); AtlasClient.EntityResult entityResult = metadataService.updateEntityPartialByGuid(guid, updatedEntity); - LOG.info("Updated entities: {}", entityResult); + + if (LOG.isDebugEnabled()) { + LOG.debug("Updated entities: {}", entityResult); + } JSONObject response = getResponse(entityResult); return Response.ok(response).build(); @@ -336,7 +418,6 @@ public class EntityResource { * @postbody property's value * @return response payload as json */ - @Monitored private Response updateEntityAttributeByGuid(String guid, String property, HttpServletRequest request) { String value = null; try { @@ -344,9 +425,15 @@ public class EntityResource { value = Servlets.getRequestPayload(request); Preconditions.checkNotNull(value, "Entity value cannot be null"); - LOG.info("Updating entity {} for property {} = {}", guid, property, value); + if (LOG.isDebugEnabled()) { + LOG.debug("Updating entity {} for property {} = {}", guid, property, value); + } + AtlasClient.EntityResult entityResult = metadataService.updateEntityAttributeByGuid(guid, property, value); - LOG.info("Updated entities: {}", entityResult); + + if (LOG.isDebugEnabled()) { + LOG.debug("Updated entities: {}", entityResult); + } JSONObject response = getResponse(entityResult); return Response.ok(response).build(); @@ -374,23 +461,41 @@ public class EntityResource { * @param value the unique attribute value used to identify the entity * @return response payload as json - including guids of entities(including composite references from that entity) that were deleted */ - @Monitored @DELETE @Produces(Servlets.JSON_MEDIA_TYPE) public Response deleteEntities(@QueryParam("guid") List<String> guids, @QueryParam("type") String entityType, @QueryParam("property") String attribute, @QueryParam("value") String value) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> EntityResource.deleteEntities({}, {}, {}, {})", guids, entityType, attribute, value); + } + + AtlasPerfTracer perf = null; try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.deleteEntities(" + guids + ", " + entityType + ", " + attribute + ", " + value + ")"); + } + AtlasClient.EntityResult entityResult; if (guids != null && !guids.isEmpty()) { - LOG.info("Deleting entities {}", guids); + if (LOG.isDebugEnabled()) { + LOG.debug("Deleting entities {}", guids); + } + entityResult = metadataService.deleteEntities(guids); } else { - LOG.info("Deleting entity type={} with property {}={}", entityType, attribute, value); + if (LOG.isDebugEnabled()) { + LOG.debug("Deleting entity type={} with property {}={}", entityType, attribute, value); + } + entityResult = metadataService.deleteEntityByUniqueAttribute(entityType, attribute, value); } - LOG.info("Deleted entity result: {}", entityResult); + + if (LOG.isDebugEnabled()) { + LOG.debug("Deleted entity result: {}", entityResult); + } + JSONObject response = getResponse(entityResult); return Response.ok(response).build(); } catch (EntityNotFoundException e) { @@ -406,6 +511,12 @@ public class EntityResource { } catch (Throwable e) { LOG.error("Unable to delete entities {} {} {} {} ", guids, entityType, attribute, value, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== EntityResource.deleteEntities({}, {}, {}, {})", guids, entityType, attribute, value); + } } } @@ -414,13 +525,24 @@ public class EntityResource { * * @param guid GUID for the entity */ - @Monitored @GET @Path("{guid}") @Produces(Servlets.JSON_MEDIA_TYPE) public Response getEntityDefinition(@PathParam("guid") String guid) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> EntityResource.getEntityDefinition({})", guid); + } + + AtlasPerfTracer perf = null; try { - LOG.debug("Fetching entity definition for guid={} ", guid); + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.getEntityDefinition(" + guid + ")"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Fetching entity definition for guid={} ", guid); + } + guid = ParamChecker.notEmpty(guid, "guid cannot be null"); final String entityDefinition = metadataService.getEntityDefinitionJson(guid); @@ -447,6 +569,12 @@ public class EntityResource { } catch (Throwable e) { LOG.error("Unable to get instance definition for GUID {}", guid, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== EntityResource.getEntityDefinition({})", guid); + } } } @@ -455,12 +583,14 @@ public class EntityResource { * * @param entityType name of a type which is unique */ - @Monitored public Response getEntityListByType(String entityType) { try { Preconditions.checkNotNull(entityType, "Entity type cannot be null"); - LOG.debug("Fetching entity list for type={} ", entityType); + if (LOG.isDebugEnabled()) { + LOG.debug("Fetching entity list for type={} ", entityType); + } + final List<String> entityList = metadataService.getEntityList(entityType); JSONObject response = new JSONObject(); @@ -488,12 +618,29 @@ public class EntityResource { public Response getEntity(@QueryParam("type") String entityType, @QueryParam("property") String attribute, @QueryParam("value") String value) { - if (StringUtils.isEmpty(attribute)) { - //List API - return getEntityListByType(entityType); - } else { - //Get entity by unique attribute - return getEntityDefinitionByAttribute(entityType, attribute, value); + if (LOG.isDebugEnabled()) { + LOG.debug("==> EntityResource.getEntity({}, {}, {})", entityType, attribute, value); + } + + AtlasPerfTracer perf = null; + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.getEntity(" + entityType + ", " + attribute + ", " + value + ")"); + } + + if (StringUtils.isEmpty(attribute)) { + //List API + return getEntityListByType(entityType); + } else { + //Get entity by unique attribute + return getEntityDefinitionByAttribute(entityType, attribute, value); + } + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== EntityResource.getEntity({}, {}, {})", entityType, attribute, value); + } } } @@ -504,10 +651,12 @@ public class EntityResource { * @param attribute * @param value */ - @Monitored public Response getEntityDefinitionByAttribute(String entityType, String attribute, String value) { try { - LOG.debug("Fetching entity definition for type={}, qualified name={}", entityType, value); + if (LOG.isDebugEnabled()) { + LOG.debug("Fetching entity definition for type={}, qualified name={}", entityType, value); + } + entityType = ParamChecker.notEmpty(entityType, "Entity type cannot be null"); attribute = ParamChecker.notEmpty(attribute, "attribute name cannot be null"); value = ParamChecker.notEmpty(value, "attribute value cannot be null"); @@ -549,13 +698,24 @@ public class EntityResource { * @param guid globally unique identifier for the entity * @return a list of trait names for the given entity guid */ - @Monitored @GET @Path("{guid}/traits") @Produces(Servlets.JSON_MEDIA_TYPE) public Response getTraitNames(@PathParam("guid") String guid) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> EntityResource.getTraitNames({})", guid); + } + + AtlasPerfTracer perf = null; try { - LOG.debug("Fetching trait names for entity={}", guid); + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.getTraitNames(" + guid + ")"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Fetching trait names for entity={}", guid); + } + final List<String> traitNames = metadataService.getTraitNames(guid); JSONObject response = new JSONObject(); @@ -573,6 +733,12 @@ public class EntityResource { } catch (Throwable e) { LOG.error("Unable to get trait names for entity {}", guid, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== EntityResource.getTraitNames({})", guid); + } } } @@ -580,13 +746,24 @@ public class EntityResource { * Fetches the trait definitions of all the traits associated to the given entity * @param guid globally unique identifier for the entity */ - @Monitored @GET @Path("{guid}/traitDefinitions") @Produces(Servlets.JSON_MEDIA_TYPE) public Response getTraitDefinitionsForEntity(@PathParam("guid") String guid){ + if (LOG.isDebugEnabled()) { + LOG.debug("==> EntityResource.getTraitDefinitionsForEntity({})", guid); + } + + AtlasPerfTracer perf = null; try { - LOG.debug("Fetching all trait definitions for entity={}", guid); + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.getTraitDefinitionsForEntity(" + guid + ")"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Fetching all trait definitions for entity={}", guid); + } + final String entityDefinition = metadataService.getEntityDefinitionJson(guid); Referenceable entity = InstanceSerialization.fromJsonReferenceable(entityDefinition, true); @@ -611,6 +788,12 @@ public class EntityResource { } catch (Throwable e) { LOG.error("Unable to get trait definitions for entity {}", guid, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== EntityResource.getTraitDefinitionsForEntity({})", guid); + } } } @@ -620,13 +803,24 @@ public class EntityResource { * @param guid globally unique identifier for the entity * @param traitName name of the trait */ - @Monitored @GET @Path("{guid}/traitDefinitions/{traitName}") @Produces(Servlets.JSON_MEDIA_TYPE) public Response getTraitDefinitionForEntity(@PathParam("guid") String guid, @PathParam("traitName") String traitName){ + if (LOG.isDebugEnabled()) { + LOG.debug("==> EntityResource.getTraitDefinitionForEntity({}, {})", guid, traitName); + } + + AtlasPerfTracer perf = null; try { - LOG.debug("Fetching trait definition for entity {} and trait name {}", guid, traitName); + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.getTraitDefinitionForEntity(" + guid + ", " + traitName + ")"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Fetching trait definition for entity {} and trait name {}", guid, traitName); + } + final IStruct traitDefinition = metadataService.getTraitDefinition(guid, traitName); JSONObject response = new JSONObject(); @@ -644,6 +838,12 @@ public class EntityResource { } catch (Throwable e) { LOG.error("Unable to get trait definition for entity {} and trait {}", guid, traitName, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== EntityResource.getTraitDefinitionForEntity({}, {})", guid, traitName); + } } } @@ -652,16 +852,28 @@ public class EntityResource { * * @param guid globally unique identifier for the entity */ - @Monitored @POST @Path("{guid}/traits") @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) @Produces(Servlets.JSON_MEDIA_TYPE) public Response addTrait(@Context HttpServletRequest request, @PathParam("guid") final String guid) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> EntityResource.addTrait({})", guid); + } + String traitDefinition = null; + AtlasPerfTracer perf = null; try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.addTrait(" + guid + ")"); + } + traitDefinition = Servlets.getRequestPayload(request); - LOG.info("Adding trait={} for entity={} ", traitDefinition, guid); + + if (LOG.isDebugEnabled()) { + LOG.debug("Adding trait={} for entity={} ", traitDefinition, guid); + } + metadataService.addTrait(guid, traitDefinition); URI locationURI = getLocationURI(new ArrayList<String>() {{ @@ -681,6 +893,12 @@ public class EntityResource { } catch (Throwable e) { LOG.error("Unable to add trait for entity={} traitDef={}", guid, traitDefinition, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== EntityResource.addTrait({})", guid); + } } } @@ -690,15 +908,27 @@ public class EntityResource { * @param guid globally unique identifier for the entity * @param traitName name of the trait */ - @Monitored @DELETE @Path("{guid}/traits/{traitName}") @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) @Produces(Servlets.JSON_MEDIA_TYPE) public Response deleteTrait(@Context HttpServletRequest request, @PathParam("guid") String guid, @PathParam(TRAIT_NAME) String traitName) { - LOG.info("Deleting trait={} from entity={} ", traitName, guid); + if (LOG.isDebugEnabled()) { + LOG.debug("==> EntityResource.deleteTrait({}, {})", guid, traitName); + } + + AtlasPerfTracer perf = null; + + if (LOG.isDebugEnabled()) { + LOG.debug("Deleting trait={} from entity={} ", traitName, guid); + } + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.deleteTrait(" + guid + ", " + traitName + ")"); + } + metadataService.deleteTrait(guid, traitName); JSONObject response = new JSONObject(); @@ -718,6 +948,12 @@ public class EntityResource { } catch (Throwable e) { LOG.error("Unable to delete trait name={} for entity={}", traitName, guid, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== EntityResource.deleteTrait({}, {})", guid, traitName); + } } } @@ -731,15 +967,26 @@ public class EntityResource { * @param count number of events required * @return */ - @Monitored @GET @Path("{guid}/audit") @Produces(Servlets.JSON_MEDIA_TYPE) public Response getAuditEvents(@PathParam("guid") String guid, @QueryParam("startKey") String startKey, @QueryParam("count") @DefaultValue("100") short count) { - LOG.debug("Audit events request for entity {}, start key {}, number of results required {}", guid, startKey, - count); + if (LOG.isDebugEnabled()) { + LOG.debug("==> EntityResource.getAuditEvents({}, {}, {})", guid, startKey, count); + } + + AtlasPerfTracer perf = null; + + if (LOG.isDebugEnabled()) { + LOG.debug("Audit events request for entity {}, start key {}, number of results required {}", guid, startKey, count); + } + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.getAuditEvents(" + guid + ", " + startKey + ", " + count + ")"); + } + List<EntityAuditEvent> events = metadataService.getAuditEvents(guid, startKey, count); JSONObject response = new JSONObject(); @@ -752,6 +999,12 @@ public class EntityResource { } catch (Throwable e) { LOG.error("Unable to get audit events for entity guid={} startKey={}", guid, startKey, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== EntityResource.getAuditEvents({}, {}, {})", guid, startKey, count); + } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/10e82ff4/webapp/src/main/java/org/apache/atlas/web/resources/LineageResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/LineageResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/LineageResource.java index c7aa372..e8d2d74 100644 --- a/webapp/src/main/java/org/apache/atlas/web/resources/LineageResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/LineageResource.java @@ -19,7 +19,6 @@ package org.apache.atlas.web.resources; import org.apache.atlas.AtlasClient; -import org.apache.atlas.aspect.Monitored; import org.apache.atlas.discovery.AtlasLineageService; import org.apache.atlas.discovery.DiscoveryException; import org.apache.atlas.discovery.LineageService; @@ -29,6 +28,7 @@ import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.typesystem.exception.EntityNotFoundException; import org.apache.atlas.typesystem.exception.SchemaNotFoundException; +import org.apache.atlas.utils.AtlasPerfTracer; import org.apache.atlas.web.util.LineageUtils; import org.apache.atlas.web.util.Servlets; import org.codehaus.jettison.json.JSONException; @@ -50,6 +50,7 @@ import javax.ws.rs.core.Response; @Singleton public class LineageResource { private static final Logger LOG = LoggerFactory.getLogger(DataSetLineageResource.class); + private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.LineageResource"); private final AtlasLineageService atlasLineageService; private final LineageService lineageService; @@ -73,15 +74,21 @@ public class LineageResource { * @param guid dataset entity id * @return */ - @Monitored @GET @Path("{guid}/inputs/graph") @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) public Response inputsGraph(@PathParam("guid") String guid) { - LOG.info("Fetching lineage inputs graph for guid={}", guid); + if (LOG.isDebugEnabled()) { + LOG.debug("==> LineageResource.inputsGraph({})", guid); + } + AtlasPerfTracer perf = null; try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "LineageResource.inputsGraph(" + guid + ")"); + } + AtlasLineageInfo lineageInfo = atlasLineageService.getAtlasLineageInfo(guid, LineageDirection.INPUT, -1); final String result = LineageUtils.toLineageStruct(lineageInfo, typeRegistry); @@ -96,6 +103,12 @@ public class LineageResource { } catch (JSONException e) { LOG.error("Unable to get lineage inputs graph for entity guid={}", guid, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== LineageResource.inputsGraph({})", guid); + } } } @@ -104,15 +117,22 @@ public class LineageResource { * * @param guid dataset entity id */ - @Monitored @GET @Path("{guid}/outputs/graph") @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) public Response outputsGraph(@PathParam("guid") String guid) { - LOG.info("Fetching lineage outputs graph for entity guid={}", guid); + if (LOG.isDebugEnabled()) { + LOG.debug("==> LineageResource.outputsGraph({})", guid); + } + + AtlasPerfTracer perf = null; try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "LineageResource.outputsGraph(" + guid + ")"); + } + AtlasLineageInfo lineageInfo = atlasLineageService.getAtlasLineageInfo(guid, LineageDirection.OUTPUT, -1); final String result = LineageUtils.toLineageStruct(lineageInfo, typeRegistry); @@ -127,6 +147,12 @@ public class LineageResource { } catch (JSONException e) { LOG.error("Unable to get lineage outputs graph for entity guid={}", guid, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== LineageResource.outputsGraph({})", guid); + } } } @@ -135,15 +161,21 @@ public class LineageResource { * * @param guid dataset entity id */ - @Monitored @GET @Path("{guid}/schema") @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) public Response schema(@PathParam("guid") String guid) { - LOG.info("Fetching schema for entity guid={}", guid); + if (LOG.isDebugEnabled()) { + LOG.debug("==> LineageResource.schema({})", guid); + } + AtlasPerfTracer perf = null; try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "LineageResource.schema(" + guid + ")"); + } + final String jsonResult = lineageService.getSchemaForEntity(guid); JSONObject response = new JSONObject(); @@ -163,6 +195,12 @@ public class LineageResource { } catch (Throwable e) { LOG.error("Unable to get schema for entity={}", guid, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== LineageResource.schema({})", guid); + } } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/10e82ff4/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java index a5ef57d..61e833f 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java @@ -21,7 +21,6 @@ package org.apache.atlas.web.resources; import com.google.common.base.Preconditions; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasConfiguration; -import org.apache.atlas.aspect.Monitored; import org.apache.atlas.classification.InterfaceAudience; import org.apache.atlas.discovery.DiscoveryException; import org.apache.atlas.discovery.DiscoveryService; @@ -83,7 +82,6 @@ public class MetadataDiscoveryResource { * @param offset offset to the results returned, used for pagination. offset >= 0. -1 maps to offset 0 * @return JSON representing the type and results. */ - @Monitored @GET @Path("search") @Consumes(Servlets.JSON_MEDIA_TYPE) @@ -99,13 +97,17 @@ public class MetadataDiscoveryResource { dslQueryFailed = true; } } catch (Exception e) { - LOG.debug("Error while running DSL. Switching to fulltext for query {}", query, e); + if (LOG.isDebugEnabled()) { + LOG.debug("Error while running DSL. Switching to fulltext for query {}", query, e); + } + dslQueryFailed = true; } if ( dslQueryFailed ) { response = searchUsingFullText(query, limit, offset); } + return response; } @@ -121,7 +123,6 @@ public class MetadataDiscoveryResource { * * @return JSON representing the type and results. */ - @Monitored @GET @Path("search/dsl") @Consumes(Servlets.JSON_MEDIA_TYPE) @@ -129,7 +130,16 @@ public class MetadataDiscoveryResource { public Response searchUsingQueryDSL(@QueryParam("query") String dslQuery, @DefaultValue(LIMIT_OFFSET_DEFAULT) @QueryParam("limit") int limit, @DefaultValue(LIMIT_OFFSET_DEFAULT) @QueryParam("offset") int offset) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> MetadataDiscoveryResource.searchUsingQueryDSL({}, {}, {})", dslQuery, limit, offset); + } + + AtlasPerfTracer perf = null; try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MetadataDiscoveryResource.searchUsingQueryDSL(" + dslQuery + ", " + limit + ", " + offset + ")"); + } + dslQuery = ParamChecker.notEmpty(dslQuery, "dslQuery cannot be null"); QueryParams queryParams = validateQueryParams(limit, offset); final String jsonResultStr = discoveryService.searchByDSL(dslQuery, queryParams); @@ -143,6 +153,12 @@ public class MetadataDiscoveryResource { } catch (Throwable e) { LOG.error("Unable to get entity list for dslQuery {}", dslQuery, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== MetadataDiscoveryResource.searchUsingQueryDSL({}, {}, {})", dslQuery, limit, offset); + } } } @@ -174,14 +190,22 @@ public class MetadataDiscoveryResource { * @param gremlinQuery search query in raw gremlin format. * @return JSON representing the type and results. */ - @Monitored @GET @Path("search/gremlin") @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) @InterfaceAudience.Private public Response searchUsingGremlinQuery(@QueryParam("query") String gremlinQuery) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> MetadataDiscoveryResource.searchUsingGremlinQuery({})", gremlinQuery); + } + + AtlasPerfTracer perf = null; try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MetadataDiscoveryResource.searchUsingGremlinQuery(" + gremlinQuery + ")"); + } + gremlinQuery = ParamChecker.notEmpty(gremlinQuery, "gremlinQuery cannot be null or empty"); final List<Map<String, String>> results = discoveryService.searchByGremlin(gremlinQuery); @@ -204,6 +228,12 @@ public class MetadataDiscoveryResource { } catch (Throwable e) { LOG.error("Unable to get entity list for gremlinQuery {}", gremlinQuery, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== MetadataDiscoveryResource.searchUsingGremlinQuery({})", gremlinQuery); + } } } @@ -215,7 +245,6 @@ public class MetadataDiscoveryResource { * @param offset offset to the results returned, used for pagination. offset >= 0. -1 maps to offset 0 * @return JSON representing the type and results. */ - @Monitored @GET @Path("search/fulltext") @Consumes(Servlets.JSON_MEDIA_TYPE) @@ -223,7 +252,16 @@ public class MetadataDiscoveryResource { public Response searchUsingFullText(@QueryParam("query") String query, @DefaultValue(LIMIT_OFFSET_DEFAULT) @QueryParam("limit") int limit, @DefaultValue(LIMIT_OFFSET_DEFAULT) @QueryParam("offset") int offset) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> MetadataDiscoveryResource.searchUsingFullText({}, {}, {})", query, limit, offset); + } + + AtlasPerfTracer perf = null; try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MetadataDiscoveryResource.searchUsingFullText(" + query + ", " + limit + ", " + offset + ")"); + } + query = ParamChecker.notEmpty(query, "query cannot be null or empty"); QueryParams queryParams = validateQueryParams(limit, offset); final String jsonResultStr = discoveryService.searchByFullText(query, queryParams); @@ -237,6 +275,12 @@ public class MetadataDiscoveryResource { } catch (Throwable e) { LOG.error("Unable to get entity list for query {}", query, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== MetadataDiscoveryResource.searchUsingFullText({}, {}, {})", query, limit, offset); + } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/10e82ff4/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java index bbe660e..c45096f 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java @@ -22,7 +22,6 @@ import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.core.ResourceContext; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; -import org.apache.atlas.aspect.Monitored; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.typedef.AtlasClassificationDef; @@ -88,17 +87,29 @@ public class TypesResource { * Submits a type definition corresponding to a given type representing a meta model of a * domain. Could represent things like Hive Database, Hive Table, etc. */ - @Monitored @POST @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) @Produces(Servlets.JSON_MEDIA_TYPE) public Response submit(@Context HttpServletRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TypesResource.submit()"); + } + + AtlasPerfTracer perf = null; + + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesResource.submit()"); + } + TypesREST typesRest = resourceContext.getResource(TypesREST.class); JSONArray typesResponse = new JSONArray(); try { final String typeDefinition = Servlets.getRequestPayload(request); - LOG.info("Creating type with definition {} ", typeDefinition); + + if (LOG.isDebugEnabled()) { + LOG.debug("Creating type with definition {} ", typeDefinition); + } AtlasTypesDef createTypesDef = RestUtils.toAtlasTypesDef(typeDefinition, typeRegistry); AtlasTypesDef createdTypesDef = typesRest.createAtlasTypeDefs(createTypesDef); @@ -127,6 +138,12 @@ public class TypesResource { } catch (Throwable e) { LOG.error("Unable to persist types", e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TypesResource.submit()"); + } } } @@ -139,16 +156,28 @@ public class TypesResource { * @param request * @return */ - @Monitored @PUT @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) @Produces(Servlets.JSON_MEDIA_TYPE) public Response update(@Context HttpServletRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TypesResource.update()"); + } + + AtlasPerfTracer perf = null; + + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesResource.update()"); + } + TypesREST typesRest = resourceContext.getResource(TypesREST.class); JSONArray typesResponse = new JSONArray(); try { final String typeDefinition = Servlets.getRequestPayload(request); - LOG.info("Updating type with definition {} ", typeDefinition); + + if (LOG.isDebugEnabled()) { + LOG.debug("Updating type with definition {} ", typeDefinition); + } AtlasTypesDef updateTypesDef = RestUtils.toAtlasTypesDef(typeDefinition, typeRegistry); AtlasTypesDef updatedTypesDef = typesRest.updateAtlasTypeDefs(updateTypesDef); @@ -174,6 +203,12 @@ public class TypesResource { } catch (Throwable e) { LOG.error("Unable to persist types", e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TypesResource.update()"); + } } } @@ -182,11 +217,20 @@ public class TypesResource { * * @param typeName name of a type which is unique. */ - @Monitored @GET @Path("{typeName}") @Produces(Servlets.JSON_MEDIA_TYPE) public Response getDefinition(@Context HttpServletRequest request, @PathParam("typeName") String typeName) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TypesResource.getDefinition({})", typeName); + } + + AtlasPerfTracer perf = null; + + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesResource.getDefinition(" + typeName + ")"); + } + TypesREST typesRest = resourceContext.getResource(TypesREST.class); JSONObject response = new JSONObject(); @@ -233,6 +277,12 @@ public class TypesResource { } catch (Throwable e) { LOG.error("Unable to get type definition for type {}", typeName, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TypesResource.getDefinition({})", typeName); + } } } @@ -248,12 +298,21 @@ public class TypesResource { * For example, typeCategory = TRAIT && supertype contains 'X' && supertype !contains 'Y' * If there is no filter, all the types are returned */ - @Monitored @GET @Produces(Servlets.JSON_MEDIA_TYPE) public Response getTypesByFilter(@Context HttpServletRequest request, @QueryParam("type") String typeCategory, @QueryParam("supertype") String supertype, @QueryParam("notsupertype") String notsupertype) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TypesResource.getTypesByFilter({}, {}, {})", typeCategory, supertype, notsupertype); + } + + AtlasPerfTracer perf = null; + + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesResource.getTypesByFilter(" + typeCategory + ", " + supertype + ", " + notsupertype + ")"); + } + TypesREST typesRest = resourceContext.getResource(TypesREST.class); JSONObject response = new JSONObject(); try { @@ -270,6 +329,12 @@ public class TypesResource { } catch (Throwable e) { LOG.error("Unable to get types list", e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TypesResource.getTypesByFilter({}, {}, {})", typeCategory, supertype, notsupertype); + } } } }
