AMBARI-15807. Initial commit for Logsearch REST API implementation (rnettleton via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/20e66fbb Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/20e66fbb Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/20e66fbb Branch: refs/heads/trunk Commit: 20e66fbb2b5c92b755b8d85e8009d64ccb1c5403 Parents: ea0f60b Author: Bob Nettleton <rnettle...@hortonworks.com> Authored: Wed Apr 13 00:30:17 2016 +0200 Committer: oleewere <oleew...@gmail.com> Committed: Wed Apr 13 00:33:47 2016 +0200 ---------------------------------------------------------------------- .../resources/LoggingResourceDefinition.java | 44 +++ .../resources/ResourceInstanceFactoryImpl.java | 4 + .../server/api/services/ClusterService.java | 14 + .../server/api/services/LoggingService.java | 232 ++++++++++++++ .../AbstractControllerResourceProvider.java | 2 + .../internal/AbstractProviderModule.java | 14 +- .../internal/LoggingResourceProvider.java | 145 +++++++++ .../logging/HostComponentLoggingInfo.java | 55 ++++ .../logging/LogFileDefinitionInfo.java | 84 +++++ .../server/controller/logging/LogFileType.java | 33 ++ .../controller/logging/LogLineResult.java | 315 +++++++++++++++++++ .../controller/logging/LogQueryResponse.java | 123 ++++++++ .../logging/LoggingRequestHelper.java | 29 ++ .../logging/LoggingRequestHelperFactory.java | 35 +++ .../LoggingRequestHelperFactoryImpl.java | 82 +++++ .../logging/LoggingRequestHelperImpl.java | 202 ++++++++++++ .../logging/LoggingSearchPropertyProvider.java | 187 +++++++++++ .../ambari/server/controller/spi/Resource.java | 5 +- .../src/main/resources/key_properties.json | 3 + .../controller/logging/LogLineResultTest.java | 135 ++++++++ .../logging/LogQueryResponseTest.java | 226 +++++++++++++ .../LoggingRequestHelperFactoryImplTest.java | 133 ++++++++ .../LoggingSearchPropertyProviderTest.java | 170 ++++++++++ 23 files changed, 2266 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/main/java/org/apache/ambari/server/api/resources/LoggingResourceDefinition.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/LoggingResourceDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/LoggingResourceDefinition.java new file mode 100644 index 0000000..b2c1a94 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/LoggingResourceDefinition.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.resources; + + +import org.apache.ambari.server.controller.spi.Resource; + +import java.util.Collections; +import java.util.Set; + +public class LoggingResourceDefinition extends BaseResourceDefinition { + + public LoggingResourceDefinition() { + super(Resource.Type.LoggingQuery); + } + + + @Override + public String getPluralName() { + return "logging"; + } + + @Override + public String getSingularName() { + return "logging"; + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java index c711bed..eed2703 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java @@ -429,6 +429,10 @@ public class ResourceInstanceFactoryImpl implements ResourceInstanceFactory { resourceDefinition = new SimpleResourceDefinition(Resource.Type.ClusterKerberosDescriptor, "kerberos_descriptor", "kerberos_descriptors"); break; + case LoggingQuery: + resourceDefinition = new LoggingResourceDefinition(); + break; + default: throw new IllegalArgumentException("Unsupported resource type: " + type); } http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/main/java/org/apache/ambari/server/api/services/ClusterService.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ClusterService.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ClusterService.java index 371411d..072c4a2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ClusterService.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ClusterService.java @@ -651,6 +651,20 @@ public class ClusterService extends BaseService { return new ClusterKerberosDescriptorService(clusterName); } + /** + * Gets the Logging Service + * + * @param request the request + * @param clusterName the cluster name + * + * @return a new instance of the LoggingService + */ + @Path("{clusterName}/logging") + public LoggingService getLogging(@Context javax.ws.rs.core.Request request, + @PathParam("clusterName") String clusterName) { + return new LoggingService(clusterName); + } + // ----- helper methods ---------------------------------------------------- /** http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/main/java/org/apache/ambari/server/api/services/LoggingService.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/LoggingService.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/LoggingService.java new file mode 100644 index 0000000..08ec06e --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/LoggingService.java @@ -0,0 +1,232 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.services; + +import org.apache.ambari.server.api.resources.ResourceInstance; +import org.apache.ambari.server.api.services.serializers.ResultSerializer; +import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.controller.AmbariServer; +import org.apache.ambari.server.controller.internal.ResourceImpl; +import org.apache.ambari.server.controller.logging.LogQueryResponse; +import org.apache.ambari.server.controller.logging.LoggingRequestHelper; +import org.apache.ambari.server.controller.logging.LoggingRequestHelperFactory; +import org.apache.ambari.server.controller.logging.LoggingRequestHelperFactoryImpl; +import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.utils.RetryHelper; +import org.apache.log4j.Logger; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * This Service provides access to the LogSearch query services, including: + * - Access to all service log files in a given cluster + * - Search query capability across the service logs + * - Metrics data regarding logging (log level counts, etc) + * + */ +public class LoggingService extends BaseService { + + private static final Logger LOG = Logger.getLogger(LoggingService.class); + + public static final String LOGSEARCH_SITE_CONFIG_TYPE_NAME = "logsearch-site"; + + public static final String LOGSEARCH_SERVICE_NAME = "LOGSEARCH"; + + public static final String LOGSEARCH_SERVER_COMPONENT_NAME = "LOGSEARCH_SERVER"; + + private final ControllerFactory controllerFactory; + + private final LoggingRequestHelperFactory helperFactory; + + + private final String clusterName; + + public LoggingService(String clusterName) { + this(clusterName, new DefaultControllerFactory(), new LoggingRequestHelperFactoryImpl()); + } + + public LoggingService(String clusterName, ControllerFactory controllerFactory, LoggingRequestHelperFactory helperFactory) { + this.clusterName = clusterName; + this.controllerFactory = controllerFactory; + this.helperFactory = helperFactory; + } + + @GET + @Produces("text/plain") + public Response getLogSearchResource(String body, @Context HttpHeaders headers, @Context UriInfo uri) { + return handleRequest(headers, body, uri, Request.Type.GET, createLoggingResource()); + } + + @GET + @Path("searchEngine") + @Produces("text/plain") + public Response getSearchEngine(String body, @Context HttpHeaders headers, @Context UriInfo uri) { + //TODO, fix this cast after testing,RWN + return handleDirectRequest(headers, body, uri, Request.Type.GET, (MediaType)null); + } + + @GET + @Path("levelCount") + @Produces("text/plain") + public Response getLevelCountForCluster(String body, @Context HttpHeaders headers, @Context UriInfo ui) { + throw new IllegalStateException("not implemented yet"); + } + + @GET + @Path("graphing") + @Produces("text/plain") + public Response getGraphData(String body, @Context HttpHeaders headers, @Context UriInfo ui) { + throw new IllegalStateException("not implemented yet"); + } + + + private ResourceInstance createLoggingResource() { + return createResource(Resource.Type.LoggingQuery, + Collections.singletonMap(Resource.Type.LoggingQuery, "logging")); + } + + + /** + * Handling method for REST services that don't require the QueryParameter and + * partial-response syntax support provided by the Ambari REST framework. + * + * In the case of the LoggingService, the query parameters passed to the search engine must + * be preserved, since they are passed to the LogSearch REST API. + * + * @param headers + * @param body + * @param uriInfo + * @param requestType + * @param mediaType + * @return + */ + protected Response handleDirectRequest(HttpHeaders headers, String body, UriInfo uriInfo, Request.Type requestType, MediaType mediaType) { + + MultivaluedMap<String, String> queryParameters = + uriInfo.getQueryParameters(); + + Map<String, String> enumeratedQueryParameters = + new HashMap<String, String>(); + + + for (String queryName : queryParameters.keySet()) { + List<String> queryValue = queryParameters.get(queryName); + for (String value : queryValue) { + enumeratedQueryParameters.put(queryName, value); + } + } + + AmbariManagementController controller = + controllerFactory.getController(); + + LoggingRequestHelper requestHelper = + helperFactory.getHelper(controller, clusterName); + + LogQueryResponse response = + requestHelper.sendQueryRequest(enumeratedQueryParameters); + + if (response != null) { + ResultSerializer serializer = mediaType == null ? getResultSerializer() : getResultSerializer(mediaType); + + Result result = new ResultImpl(new ResultStatus(ResultStatus.STATUS.OK)); + + Resource loggingResource = new ResourceImpl(Resource.Type.LoggingQuery); + // include the top-level query result properties + loggingResource.setProperty("startIndex", response.getStartIndex()); + loggingResource.setProperty("pageSize", response.getPageSize()); + loggingResource.setProperty("resultSize", response.getResultSize()); + loggingResource.setProperty("queryTimeMMS", response.getQueryTimeMS()); + loggingResource.setProperty("totalCount", response.getTotalCount()); + + // include the individual responses + loggingResource.setProperty("logList", response.getListOfResults()); + + result.getResultTree().addChild(loggingResource, "logging"); + + Response.ResponseBuilder builder = Response.status(result.getStatus().getStatusCode()).entity( + serializer.serialize(result)); + + + if (mediaType != null) { + builder.type(mediaType); + } + + RetryHelper.clearAffectedClusters(); + return builder.build(); + } + + + + //TODO, add error handling and logging here, RWN + return null; + } + + + /** + * Internal interface that defines an access factory for the + * AmbariManagementController. This facilitates simpler unit testing. + * + */ + interface ControllerFactory { + AmbariManagementController getController(); + } + + /** + * Default implementation of the internal ControllerFactory interface, + * which uses the AmbariServer static method to obtain the controller. + */ + private static class DefaultControllerFactory implements ControllerFactory { + @Override + public AmbariManagementController getController() { + return AmbariServer.getController(); + } + } + + private static class LogSearchConnectionInfo { + + private final String hostName; + private final String portNumber; + + public LogSearchConnectionInfo(String hostName, String portNumber) { + this.hostName = hostName; + this.portNumber = portNumber; + } + + public String getHostName() { + return this.hostName; + } + + public String getPortNumber() { + return this.portNumber; + } + + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java index f24da8d..3721113 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java @@ -223,6 +223,8 @@ public abstract class AbstractControllerResourceProvider extends AbstractAuthori return new VersionDefinitionResourceProvider(); case ClusterKerberosDescriptor: return new ClusterKerberosDescriptorResourceProvider(managementController); + case LoggingQuery: + return new LoggingResourceProvider(propertyIds, keyPropertyIds, managementController); default: throw new IllegalArgumentException("Unknown type " + type); } http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java index ca491f2..4ce974d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java @@ -30,6 +30,7 @@ import org.apache.ambari.server.controller.ServiceComponentHostRequest; import org.apache.ambari.server.controller.ServiceComponentHostResponse; import org.apache.ambari.server.controller.jmx.JMXHostProvider; import org.apache.ambari.server.controller.jmx.JMXPropertyProvider; +import org.apache.ambari.server.controller.logging.LoggingSearchPropertyProvider; import org.apache.ambari.server.controller.metrics.MetricHostProvider; import org.apache.ambari.server.controller.metrics.MetricsPropertyProvider; import org.apache.ambari.server.controller.metrics.MetricsReportPropertyProvider; @@ -800,11 +801,14 @@ public abstract class AbstractProviderModule implements ProviderModule, gpp)); providers.add(new HttpPropertyProvider(streamProvider, - managementController.getClusters(), - PropertyHelper.getPropertyId("HostRoles", "cluster_name"), - PropertyHelper.getPropertyId("HostRoles", "host_name"), - PropertyHelper.getPropertyId("HostRoles", "component_name"), - HTTP_PROPERTY_REQUESTS)); + managementController.getClusters(), + PropertyHelper.getPropertyId("HostRoles", "cluster_name"), + PropertyHelper.getPropertyId("HostRoles", "host_name"), + PropertyHelper.getPropertyId("HostRoles", "component_name"), + HTTP_PROPERTY_REQUESTS)); + + //TODO, this may need to be conditional based on the presence/absence of LogSearch + providers.add(new LoggingSearchPropertyProvider()); } break; case RootServiceComponent: http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/LoggingResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/LoggingResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/LoggingResourceProvider.java new file mode 100644 index 0000000..f2e07bd --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/LoggingResourceProvider.java @@ -0,0 +1,145 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.internal; + + +import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.controller.logging.LogQueryResponse; +import org.apache.ambari.server.controller.logging.LoggingRequestHelperImpl; +import org.apache.ambari.server.controller.spi.NoSuchParentResourceException; +import org.apache.ambari.server.controller.spi.NoSuchResourceException; +import org.apache.ambari.server.controller.spi.Predicate; +import org.apache.ambari.server.controller.spi.Request; +import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.controller.spi.SystemException; +import org.apache.ambari.server.controller.spi.UnsupportedPropertyException; +import org.apache.ambari.server.controller.utilities.PropertyHelper; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class LoggingResourceProvider extends AbstractControllerResourceProvider { + + private static final String LOGGING_SEARCH_SERVICE_PROPERTY_ID = PropertyHelper.getPropertyId("Logging", "search_service_name"); + private static final String LOGGING_SEARCH_TERM_PROPERTY_ID = PropertyHelper.getPropertyId("Logging", "searchTerm"); + private static final String LOGGING_COMPONENT_PROPERTY_ID = PropertyHelper.getPropertyId("Logging", "component"); + + private static final Set<String> PROPERTY_IDS; + + private static final Map<Resource.Type, String> KEY_PROPERTY_IDS; + + static { + Set<String> localSet = new HashSet<String>(); + localSet.add(LOGGING_SEARCH_SERVICE_PROPERTY_ID); + localSet.add(LOGGING_SEARCH_TERM_PROPERTY_ID); + localSet.add(LOGGING_COMPONENT_PROPERTY_ID); + + PROPERTY_IDS = Collections.unmodifiableSet(localSet); + + Map<Resource.Type, String> localMap = + new HashMap<Resource.Type, String>(); + + localMap.put(Resource.Type.LoggingQuery, LOGGING_SEARCH_SERVICE_PROPERTY_ID); + KEY_PROPERTY_IDS = Collections.unmodifiableMap(localMap); + + } + + + public LoggingResourceProvider(Set<String> propertyIds, + Map<Resource.Type, String> keyPropertyIds, + AmbariManagementController controller) { + + + + super(PROPERTY_IDS, KEY_PROPERTY_IDS, controller); + } + + @Override + protected Set<String> getPKPropertyIds() { + return Collections.emptySet(); + } + + @Override + public Set<Resource> getResources(Request request, Predicate predicate) throws SystemException, UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException { + // just a simple text resource for now, to describe the logging service + Resource resource = new ResourceImpl(Resource.Type.LoggingQuery); + setResourceProperty(resource, LOGGING_SEARCH_SERVICE_PROPERTY_ID, "logging", getRequestPropertyIds(request, predicate)); + + LoggingRequestHelperImpl requestHelper = + new LoggingRequestHelperImpl(); + + Map<String, String> queryParameters = + new HashMap<String, String>(); + + queryParameters.put("level", "ERROR"); + + LogQueryResponse response = + requestHelper.sendQueryRequest(queryParameters); + + // include the top-level query result properties + resource.setProperty("startIndex", response.getStartIndex()); + resource.setProperty("pageSize", response.getPageSize()); + resource.setProperty("resultSize", response.getResultSize()); + resource.setProperty("queryTimeMMS", response.getQueryTimeMS()); + resource.setProperty("totalCount", response.getTotalCount()); + + // include the individual responses + resource.setProperty("logList", response.getListOfResults()); + + return Collections.singleton(resource); + } + + private static List<Map<String, String>> createTestData(Resource resource) { + // just create some test data for verifying basic resource code, not an actual result + Map<String, String> levelCounts = new HashMap<String, String>(); + levelCounts.put("INFO", "100"); + levelCounts.put("WARN", "250"); + levelCounts.put("DEBUG", "300"); + + resource.setProperty("logLevels", levelCounts); + + List<Map <String, String>> listOfResults = new LinkedList<Map<String, String>>(); + Map<String, String> resultOne = new HashMap<String, String>(); + resultOne.put("data", "This is a test sentence."); + resultOne.put("score", "100"); + resultOne.put("level", "INFO"); + resultOne.put("type", "hdfs_namenode"); + resultOne.put("host", "c6401.ambari.apache.org"); + resultOne.put("LoggerName", "NameNodeLogger"); + + listOfResults.add(resultOne); + return listOfResults; + } + + @Override + public Set<String> checkPropertyIds(Set<String> propertyIds) { + Set<String> unSupportedProperties = + super.checkPropertyIds(propertyIds); + + unSupportedProperties.remove("searchTerm"); + + return unSupportedProperties; + + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/HostComponentLoggingInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/HostComponentLoggingInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/HostComponentLoggingInfo.java new file mode 100644 index 0000000..516c322 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/HostComponentLoggingInfo.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.logging; + + +import org.codehaus.jackson.annotate.JsonProperty; + +import java.util.List; + +public class HostComponentLoggingInfo { + + private String componentName; + + private List<LogFileDefinitionInfo> listOfLogFileDefinitions; + + + public HostComponentLoggingInfo() { + } + + @JsonProperty("name") + public String getComponentName() { + return componentName; + } + + @JsonProperty("name") + public void setComponentName(String componentName) { + this.componentName = componentName; + } + + @JsonProperty("logs") + public List<LogFileDefinitionInfo> getListOfLogFileDefinitions() { + return listOfLogFileDefinitions; + } + + @JsonProperty("logs") + public void setListOfLogFileDefinitions(List<LogFileDefinitionInfo> listOfLogFileDefinitions) { + this.listOfLogFileDefinitions = listOfLogFileDefinitions; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogFileDefinitionInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogFileDefinitionInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogFileDefinitionInfo.java new file mode 100644 index 0000000..12fa1ab --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogFileDefinitionInfo.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.logging; + + +import org.codehaus.jackson.annotate.JsonProperty; + +public class LogFileDefinitionInfo { + + private String logFileName; + + private LogFileType logFileType; + + private String searchEngineURL; + + private String logFileTailURL; + + // default, no args constructor, required for Jackson usage + public LogFileDefinitionInfo() { + } + + protected LogFileDefinitionInfo(String logFileName, LogFileType logFileType, String searchEngineURL, String logFileTailURL) { + this.logFileName = logFileName; + this.logFileType = logFileType; + this.searchEngineURL = searchEngineURL; + this.logFileTailURL = logFileTailURL; + } + + @JsonProperty("name") + public String getLogFileName() { + return logFileName; + } + + @JsonProperty("name") + public void setLogFileName(String logFileName) { + this.logFileName = logFileName; + } + + @JsonProperty("type") + public LogFileType getLogFileType() { + return logFileType; + } + + @JsonProperty("type") + public void setLogFileType(LogFileType logFileType) { + this.logFileType = logFileType; + } + + @JsonProperty("searchEngineURL") + public String getSearchEngineURL() { + return searchEngineURL; + } + + @JsonProperty("searchEngineURL") + public void setSearchEngineURL(String searchEngineURL) { + this.searchEngineURL = searchEngineURL; + } + + @JsonProperty("logFileTailURL") + public String getLogFileTailURL() { + return logFileTailURL; + } + + @JsonProperty("logFileTailURL") + public void setLogFileTailURL(String logFileTailURL) { + this.logFileTailURL = logFileTailURL; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogFileType.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogFileType.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogFileType.java new file mode 100644 index 0000000..3914f8d --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogFileType.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.logging; + + +/** + * Simple Enumerated type to describe the type of log file, as defined by LogSearch. + * + */ +public enum LogFileType { + + // service logs include most log files in Hadoop + SERVICE, + // audit logs include security access information, not normally enabled by default + AUDIT + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogLineResult.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogLineResult.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogLineResult.java new file mode 100644 index 0000000..b6ab6bd --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogLineResult.java @@ -0,0 +1,315 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.logging; + +import org.codehaus.jackson.annotate.JsonProperty; + +import java.util.HashMap; +import java.util.Map; + +/** + * This class represents a single entry from a LogSearch query. + * + */ +public class LogLineResult { + + private final Map<String, String> mapOfLogLineProperties = + new HashMap<String, String>(); + + private String clusterName; + + private String logMethod; + + private String logLevel; + + private String eventCount; + + private String ipAddress; + + private String componentType; + + private String sequenceNumber; + + private String logFilePath; + + private String sourceFile; + + private String sourceFileLineNumber; + + private String hostName; + + private String logMessage; + + private String loggerName; + + private String id; + + private String messageMD5; + + private String logTime; + + private String eventMD5; + + private String logFileLineNumber; + + private String ttl; + + private String expirationTime; + + private String version; + + private String thread_name; + + public LogLineResult() { + + } + + public LogLineResult(Map<String, String> propertiesMap) { + mapOfLogLineProperties.putAll(propertiesMap); + } + + public String getProperty(String propertyName) { + return mapOfLogLineProperties.get(propertyName); + } + + public boolean doesPropertyExist(String propertyName) { + return mapOfLogLineProperties.containsKey(propertyName); + } + + + @JsonProperty("cluster") + public String getClusterName() { + return clusterName; + } + + @JsonProperty("cluster") + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + @JsonProperty("method") + public String getLogMethod() { + return logMethod; + } + + @JsonProperty("method") + public void setLogMethod(String logMethod) { + this.logMethod = logMethod; + } + + @JsonProperty("level") + public String getLogLevel() { + return logLevel; + } + + @JsonProperty("level") + public void setLogLevel(String logLevel) { + this.logLevel = logLevel; + } + + @JsonProperty("event_count") + public String getEventCount() { + return eventCount; + } + + @JsonProperty("event_count") + public void setEventCount(String eventCount) { + this.eventCount = eventCount; + } + + @JsonProperty("ip") + public String getIpAddress() { + return ipAddress; + } + + @JsonProperty("ip") + public void setIpAddress(String ipAddress) { + this.ipAddress = ipAddress; + } + + @JsonProperty("type") + public String getComponentType() { + return componentType; + } + + @JsonProperty("type") + public void setComponentType(String componentType) { + this.componentType = componentType; + } + + @JsonProperty("seq_num") + public String getSequenceNumber() { + return sequenceNumber; + } + + @JsonProperty("seq_num") + public void setSequenceNumber(String sequenceNumber) { + this.sequenceNumber = sequenceNumber; + } + + @JsonProperty("path") + public String getLogFilePath() { + return logFilePath; + } + + @JsonProperty("path") + public void setLogFilePath(String logFilePath) { + this.logFilePath = logFilePath; + } + + @JsonProperty("file") + public String getSourceFile() { + return sourceFile; + } + + @JsonProperty("file") + public void setSourceFile(String sourceFile) { + this.sourceFile = sourceFile; + } + + @JsonProperty("line_number") + public String getSourceFileLineNumber() { + return sourceFileLineNumber; + } + + @JsonProperty("line_number") + public void setSourceFileLineNumber(String sourceFileLineNumber) { + this.sourceFileLineNumber = sourceFileLineNumber; + } + + @JsonProperty("host") + public String getHostName() { + return hostName; + } + + @JsonProperty("host") + public void setHostName(String hostName) { + this.hostName = hostName; + } + + @JsonProperty("log_message") + public String getLogMessage() { + return logMessage; + } + + @JsonProperty("log_message") + public void setLogMessage(String logMessage) { + this.logMessage = logMessage; + } + + @JsonProperty("logger_name") + public String getLoggerName() { + return loggerName; + } + + @JsonProperty("logger_name") + public void setLoggerName(String loggerName) { + this.loggerName = loggerName; + } + + @JsonProperty("id") + public String getId() { + return id; + } + + @JsonProperty("id") + public void setId(String id) { + this.id = id; + } + + @JsonProperty("message_md5") + public String getMessageMD5() { + return messageMD5; + } + + @JsonProperty("message_md5") + public void setMessageMD5(String messageMD5) { + this.messageMD5 = messageMD5; + } + + @JsonProperty("logtime") + public String getLogTime() { + return logTime; + } + + @JsonProperty("logtime") + public void setLogTime(String logTime) { + this.logTime = logTime; + } + + @JsonProperty("event_md5") + public String getEventMD5() { + return eventMD5; + } + + @JsonProperty("event_md5") + public void setEventMD5(String eventMD5) { + this.eventMD5 = eventMD5; + } + + @JsonProperty("logfile_line_number") + public String getLogFileLineNumber() { + return logFileLineNumber; + } + + @JsonProperty("logfile_line_number") + public void setLogFileLineNumber(String logFileLineNumber) { + this.logFileLineNumber = logFileLineNumber; + } + + @JsonProperty("_ttl_") + public String getTtl() { + return ttl; + } + + @JsonProperty("_ttl_") + public void setTtl(String ttl) { + this.ttl = ttl; + } + + @JsonProperty("_expire_at_") + public String getExpirationTime() { + return expirationTime; + } + + @JsonProperty("_expire_at_") + public void setExpirationTime(String expirationTime) { + this.expirationTime = expirationTime; + } + + @JsonProperty("_version_") + public String getVersion() { + return version; + } + + @JsonProperty("_version_") + public void setVersion(String version) { + this.version = version; + } + + @JsonProperty("thread_name") + public String getThreadName() { + return thread_name; + } + + @JsonProperty("thread_name") + public void setThreadName(String thread_name) { + this.thread_name = thread_name; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogQueryResponse.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogQueryResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogQueryResponse.java new file mode 100644 index 0000000..9bde7a5 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogQueryResponse.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.logging; + +import org.codehaus.jackson.annotate.JsonProperty; +import org.codehaus.jackson.map.util.JSONPObject; + +import java.util.LinkedList; +import java.util.List; + +/** + * This class respresents the results of a LogSearch query, as returned by + * a call to the LogSearch service. + * + */ +public class LogQueryResponse { + + + private String startIndex; + + private String pageSize; + + private String totalCount; + + private String resultSize; + + private String queryTimeMS; + + + private List<LogLineResult> listOfResults; + + private StringBuffer resultString; + + public LogQueryResponse() { + + } + + public LogQueryResponse(StringBuffer resultString) { + this.resultString = resultString; + } + + @JsonProperty("logList") + public List<LogLineResult> getListOfResults() { + return listOfResults; + } + + @JsonProperty("logList") + public void setLogList(List<LogLineResult> logList) { + this.listOfResults = logList; + } + + @JsonProperty("startIndex") + public String getStartIndex() { + return startIndex; + } + + @JsonProperty("startIndex") + public void setStartIndex(String startIndex) { + this.startIndex = startIndex; + } + + @JsonProperty("pageSize") + public String getPageSize() { + return pageSize; + } + + @JsonProperty("pageSize") + public void setPageSize(String pageSize) { + this.pageSize = pageSize; + } + + @JsonProperty("totalCount") + public String getTotalCount() { + return totalCount; + } + + @JsonProperty("totalCount") + public void setTotalCount(String totalCount) { + this.totalCount = totalCount; + } + + @JsonProperty("resultSize") + public String getResultSize() { + return resultSize; + } + + @JsonProperty("resultSize") + public void setResultSize(String resultSize) { + this.resultSize = resultSize; + } + + @JsonProperty("queryTimeMS") + public String getQueryTimeMS() { + return queryTimeMS; + } + + @JsonProperty("queryTimeMS") + public void setQueryTimeMS(String queryTimeMS) { + this.queryTimeMS = queryTimeMS; + } + + public StringBuffer getResultString() { + return resultString; + } + + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelper.java new file mode 100644 index 0000000..c1fe1d2 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelper.java @@ -0,0 +1,29 @@ +package org.apache.ambari.server.controller.logging; + +import java.util.Map; +import java.util.Set; + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public interface LoggingRequestHelper { + + public LogQueryResponse sendQueryRequest(Map<String, String> queryParameters); + + public Set<String> sendGetLogFileNamesRequest(String componentName, String hostName); + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperFactory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperFactory.java new file mode 100644 index 0000000..5ccab6f --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperFactory.java @@ -0,0 +1,35 @@ +package org.apache.ambari.server.controller.logging; + +import org.apache.ambari.server.controller.AmbariManagementController; + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public interface LoggingRequestHelperFactory { + + /** + * Obtain a new instance of a LoggingRequestHelper, which can + * be used to connect to the given cluster + * @param ambariManagementController + * @param clusterName name of cluster that includes a LogSearch deployment + * + * @return an instance of LoggingRequestHelper that can be used to + * connect to this cluster's LogSearch service + */ + LoggingRequestHelper getHelper(AmbariManagementController ambariManagementController, String clusterName); + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperFactoryImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperFactoryImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperFactoryImpl.java new file mode 100644 index 0000000..69d4f3c --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperFactoryImpl.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.controller.logging; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; +import org.apache.ambari.server.state.ServiceComponentHost; +import org.apache.log4j.Logger; + +import java.util.List; + +public class LoggingRequestHelperFactoryImpl implements LoggingRequestHelperFactory { + + private static final Logger LOG = Logger.getLogger(LoggingRequestHelperFactoryImpl.class); + + private static final String LOGSEARCH_SITE_CONFIG_TYPE_NAME = "logsearch-site"; + + private static final String LOGSEARCH_SERVICE_NAME = "LOGSEARCH"; + + private static final String LOGSEARCH_SERVER_COMPONENT_NAME = "LOGSEARCH_SERVER"; + + private static final String LOGSEARCH_UI_PORT_PROPERTY_NAME = "logsearch.ui.port"; + + + @Override + public LoggingRequestHelper getHelper(AmbariManagementController ambariManagementController, String clusterName) { + Clusters clusters = + ambariManagementController.getClusters(); + + try { + Cluster cluster = clusters.getCluster(clusterName); + if (cluster != null) { + Config logSearchSiteConfig = + cluster.getDesiredConfigByType(LOGSEARCH_SITE_CONFIG_TYPE_NAME); + + List<ServiceComponentHost> listOfMatchingHosts = + cluster.getServiceComponentHosts(LOGSEARCH_SERVICE_NAME, LOGSEARCH_SERVER_COMPONENT_NAME); + + if (listOfMatchingHosts.size() == 0) { + LOG.info("No matching LOGSEARCH_SERVER instances found, this may indicate a deployment problem. Please verify that LogSearch is deployed and running."); + return null; + } + + if (listOfMatchingHosts.size() > 1) { + LOG.warn("More than one LOGSEARCH_SERVER instance found, this may be a deployment error. Only the first LOGSEARCH_SERVER instance will be used."); + } + + ServiceComponentHost serviceComponentHost = + listOfMatchingHosts.get(0); + + final String logSearchHostName = serviceComponentHost.getHostName(); + final String logSearchPortNumber = + logSearchSiteConfig.getProperties().get(LOGSEARCH_UI_PORT_PROPERTY_NAME); + + return new LoggingRequestHelperImpl(logSearchHostName, logSearchPortNumber); + } + } catch (AmbariException ambariException) { + LOG.error("Error occurred while trying to obtain the cluster, cluster name = " + clusterName, ambariException); + } + + + return null; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java new file mode 100644 index 0000000..2c3d941 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java @@ -0,0 +1,202 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.logging; + + +import org.apache.commons.codec.binary.Base64; +import org.apache.http.client.utils.URIBuilder; +import org.apache.log4j.Logger; +import org.codehaus.jackson.map.AnnotationIntrospector; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.ObjectReader; +import org.codehaus.jackson.map.annotate.JsonSerialize; +import org.codehaus.jackson.map.introspect.JacksonAnnotationIntrospector; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Convenience class to handle the connection details of a LogSearch query request. + * + */ +public class LoggingRequestHelperImpl implements LoggingRequestHelper { + + private Logger LOG = Logger.getLogger(LoggingRequestHelperImpl.class); + + //TODO, hardcoded localhost for LogSearch service for dev purposes, will switch to config after POC finished + + + private static String DEFAULT_HOSTNAME = "localhost"; + + private static String DEFAULT_PORT = "61888"; + + private static String LOGSEARCH_QUERY_PATH = "/service/dashboard/solr/logs_search"; + + private static String DEFAULT_LOGSEARCH_USER = "admin"; + + private static String DEFAULT_LOGSEARCH_PWD = "admin"; + + private final String hostName; + + private final String portNumber; + + + public LoggingRequestHelperImpl() { + this(DEFAULT_HOSTNAME, DEFAULT_PORT); + } + + public LoggingRequestHelperImpl(String hostName, String portNumber) { + this.hostName = hostName; + this.portNumber = portNumber; + } + + public LogQueryResponse sendQueryRequest(Map<String, String> queryParameters) { + try { + // use the Apache builder to create the correct URI + URI logSearchURI = createLogSearchQueryURI(queryParameters); + LOG.info("Attempting to connect to LogSearch server at " + logSearchURI); + + HttpURLConnection httpURLConnection = (HttpURLConnection)logSearchURI.toURL().openConnection(); + httpURLConnection.setRequestMethod("GET"); + setupBasicAuthentication(httpURLConnection); + StringBuffer buffer = readQueryResponseFromServer(httpURLConnection); + + // setup a reader for the JSON response + StringReader stringReader = + new StringReader(buffer.toString()); + + // setup the Jackson mapper/reader to read in the data structure + ObjectMapper mapper = createJSONObjectMapper(); + + ObjectReader logQueryResponseReader = + mapper.reader(LogQueryResponse.class); + + LogQueryResponse queryResult = + logQueryResponseReader.readValue(stringReader); + + LOG.debug("DEBUG: response from LogSearch was: " + buffer); + + return queryResult; + + } catch (MalformedURLException e) { + // TODO, need better error handling here + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + } catch (IOException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + } catch (URISyntaxException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + } + + return null; + } + + public Set<String> sendGetLogFileNamesRequest(String componentName, String hostName) { + Map<String, String> queryParameters = + new HashMap<String, String>(); + + // TODO, this current method will be a temporary workaround + // TODO, until the new LogSearch API method is available to handle this request + + queryParameters.put("host", hostName); + queryParameters.put("components_name",componentName); + // ask for page size of 1, since we really only want a single entry to + // get the file path name + queryParameters.put("pageSize", "1"); + + LogQueryResponse response = sendQueryRequest(queryParameters); + if ((response != null) && (!response.getListOfResults().isEmpty())) { + LogLineResult lineOne = response.getListOfResults().get(0); + // this assumes that each component has only one associated log file, + // which may not always hold true + LOG.info("For componentName = " + componentName + ", log file name is = " + lineOne.getLogFilePath()); + return Collections.singleton(lineOne.getLogFilePath()); + + } + + return Collections.emptySet(); + } + + private URI createLogSearchQueryURI(Map<String, String> queryParameters) throws URISyntaxException { + URIBuilder uriBuilder = new URIBuilder(); + uriBuilder.setScheme("http"); + uriBuilder.setHost(hostName); + uriBuilder.setPort(Integer.valueOf(portNumber)); + uriBuilder.setPath(LOGSEARCH_QUERY_PATH); + + // add any query strings specified + for (String key : queryParameters.keySet()) { + uriBuilder.addParameter(key, queryParameters.get(key)); + } + + return uriBuilder.build(); + } + + protected ObjectMapper createJSONObjectMapper() { + ObjectMapper mapper = + new ObjectMapper(); + AnnotationIntrospector introspector = + new JacksonAnnotationIntrospector(); + mapper.setAnnotationIntrospector(introspector); + mapper.getSerializationConfig().setSerializationInclusion(JsonSerialize.Inclusion.NON_NULL); + return mapper; + } + + private StringBuffer readQueryResponseFromServer(HttpURLConnection httpURLConnection) throws IOException { + InputStream resultStream = null; + try { + // read in the response from LogSearch + resultStream = httpURLConnection.getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(resultStream)); + LOG.info("Response code from LogSearch Service is = " + httpURLConnection.getResponseCode()); + + String line = reader.readLine(); + StringBuffer buffer = new StringBuffer(); + while (line != null) { + buffer.append(line); + line = reader.readLine(); + } + return buffer; + } finally { + // make sure to close the stream after request is completed + if (resultStream != null) { + resultStream.close(); + } + } + } + + private static void setupBasicAuthentication(HttpURLConnection httpURLConnection) { + //TODO, using hard-coded Base64 auth for now, need to revisit this + String encodedCredentials = + Base64.encodeBase64String((DEFAULT_LOGSEARCH_USER + ":" + DEFAULT_LOGSEARCH_PWD).getBytes()); + httpURLConnection.setRequestProperty("Authorization", "Basic " + encodedCredentials); + } + + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingSearchPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingSearchPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingSearchPropertyProvider.java new file mode 100644 index 0000000..6b4a8a4 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingSearchPropertyProvider.java @@ -0,0 +1,187 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.controller.logging; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.controller.AmbariServer; +import org.apache.ambari.server.controller.spi.Predicate; +import org.apache.ambari.server.controller.spi.PropertyProvider; +import org.apache.ambari.server.controller.spi.Request; +import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.controller.spi.SystemException; +import org.apache.ambari.server.controller.utilities.PropertyHelper; +import org.apache.ambari.server.state.ComponentInfo; +import org.apache.ambari.server.state.LogDefinition; +import org.apache.ambari.server.state.StackId; +import org.apache.log4j.Logger; + +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +public class LoggingSearchPropertyProvider implements PropertyProvider { + + private static final Logger LOG = Logger.getLogger(LoggingSearchPropertyProvider.class); + + private static final String CLUSTERS_PATH = "/api/v1/clusters"; + + private static final String PATH_TO_SEARCH_ENGINE = "/logging/searchEngine"; + + private static final String DEFAULT_PAGE_SIZE = "50"; + + private static final String COMPONENT_QUERY_PARAMETER_NAME = "components_name"; + + private static final String HOST_QUERY_PARAMETER_NAME = "host"; + + private static final String PAGE_SIZE_QUERY_PARAMETER_NAME = "pageSize"; + + private final LoggingRequestHelperFactory requestHelperFactory; + + private final ControllerFactory controllerFactory; + + public LoggingSearchPropertyProvider() { + this(new LoggingRequestHelperFactoryImpl(), new DefaultControllerFactory()); + } + + protected LoggingSearchPropertyProvider(LoggingRequestHelperFactory requestHelperFactory, ControllerFactory controllerFactory) { + this.requestHelperFactory = requestHelperFactory; + this.controllerFactory = controllerFactory; + } + + + + @Override + public Set<Resource> populateResources(Set<Resource> resources, Request request, Predicate predicate) throws SystemException { + + AmbariManagementController controller = + controllerFactory.getAmbariManagementController(); + + for (Resource resource : resources) { + // obtain the required identifying properties on the host component resource + final String componentName = (String)resource.getPropertyValue(PropertyHelper.getPropertyId("HostRoles", "component_name")); + final String hostName = (String) resource.getPropertyValue(PropertyHelper.getPropertyId("HostRoles", "host_name")); + final String clusterName = (String) resource.getPropertyValue(PropertyHelper.getPropertyId("HostRoles", "cluster_name")); + + // query the stack definitions to find the correct component name (stack name mapped to LogSearch-defined name) + final String mappedComponentNameForLogSearch = + getMappedComponentNameForSearch(clusterName, componentName, controller); + + if (mappedComponentNameForLogSearch != null) { + HostComponentLoggingInfo loggingInfo = + new HostComponentLoggingInfo(); + + // make query to LogSearch server to find the associated file names + // create helper instance using factory + LoggingRequestHelper requestHelper = + requestHelperFactory.getHelper(controller, clusterName); + + // send query to obtain logging metadata + Set<String> logFileNames = + requestHelper.sendGetLogFileNamesRequest(mappedComponentNameForLogSearch, hostName); + + if ((logFileNames != null) && (!logFileNames.isEmpty())) { + loggingInfo.setComponentName(mappedComponentNameForLogSearch); + List<LogFileDefinitionInfo> listOfFileDefinitions = + new LinkedList<LogFileDefinitionInfo>(); + + for (String fileName : logFileNames) { + // generate the URIs that can be used by clients to obtain search results/tail log results/etc + final String searchEngineURI = controller.getAmbariServerURI(getFullPathToSearchEngine(clusterName)); + final String logFileTailURI = createLogFileTailURI(searchEngineURI, mappedComponentNameForLogSearch, hostName); + // all log files are assumed to be service types for now + listOfFileDefinitions.add(new LogFileDefinitionInfo(fileName, LogFileType.SERVICE, searchEngineURI, logFileTailURI)); + } + + loggingInfo.setListOfLogFileDefinitions(listOfFileDefinitions); + + LOG.info("Adding logging info for component name = " + componentName + " on host name = " + hostName); + // add the logging metadata for this host component + resource.setProperty("logging", loggingInfo); + } else { + LOG.error("Error occurred while making request to LogSearch service, unable to populate logging properties on this resource"); + } + } + + } + + return resources; + } + + private String getMappedComponentNameForSearch(String clusterName, String componentName, AmbariManagementController controller) { + try { + AmbariMetaInfo metaInfo = controller.getAmbariMetaInfo(); + StackId stackId = + controller.getClusters().getCluster(clusterName).getCurrentStackVersion(); + final String stackName = stackId.getStackName(); + final String stackVersion = stackId.getStackVersion(); + final String serviceName = + metaInfo.getComponentToService(stackName, stackVersion, componentName); + + ComponentInfo componentInfo = + metaInfo.getComponent(stackName, stackVersion, serviceName, componentName); + if (componentInfo != null) { + List<LogDefinition> listOfLogs = + componentInfo.getLogs(); + // for now, the assumption is that there is only one log file associated with each + // component in LogSearch, but this may change in the future + if ((listOfLogs != null) && (!listOfLogs.isEmpty())) { + LogDefinition definition = listOfLogs.get(0); + // return the first log id we find + return definition.getLogId(); + } + } + + } catch (AmbariException e) { + LOG.error("Error occurred while attempting to locate the log component name for component = " + componentName, e); + } + + return null; + } + + private String getFullPathToSearchEngine(String clusterName) { + return CLUSTERS_PATH + "/" + clusterName + PATH_TO_SEARCH_ENGINE; + } + + protected static String createLogFileTailURI(String searchEngineURI, String componentName, String hostName) { + return searchEngineURI + "?" + COMPONENT_QUERY_PARAMETER_NAME + "=" + componentName + "&" + HOST_QUERY_PARAMETER_NAME + "=" + hostName + + "&" + PAGE_SIZE_QUERY_PARAMETER_NAME + "=" + DEFAULT_PAGE_SIZE; + } + + @Override + public Set<String> checkPropertyIds(Set<String> propertyIds) { + return Collections.emptySet(); + } + + /** + * Internal interface used to control how the AmbariManagementController + * instance is obtained. This is useful for unit testing as well. + */ + interface ControllerFactory { + AmbariManagementController getAmbariManagementController(); + } + + private static class DefaultControllerFactory implements ControllerFactory { + @Override + public AmbariManagementController getAmbariManagementController() { + return AmbariServer.getController(); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/main/java/org/apache/ambari/server/controller/spi/Resource.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/spi/Resource.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/spi/Resource.java index 63af4c4..5a8476d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/spi/Resource.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/spi/Resource.java @@ -154,7 +154,8 @@ public interface Resource { RoleAuthorization, UserAuthorization, VersionDefinition, - ClusterKerberosDescriptor; + ClusterKerberosDescriptor, + LoggingQuery; /** * Get the {@link Type} that corresponds to this InternalType. @@ -270,6 +271,8 @@ public interface Resource { public static final Type UserAuthorization = InternalType.UserAuthorization.getType(); public static final Type VersionDefinition = InternalType.VersionDefinition.getType(); public static final Type ClusterKerberosDescriptor = InternalType.ClusterKerberosDescriptor.getType(); + public static final Type LoggingQuery = InternalType.LoggingQuery.getType(); + /** * The type name. */ http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/main/resources/key_properties.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/key_properties.json b/ambari-server/src/main/resources/key_properties.json index 46a6cf9..8069349 100644 --- a/ambari-server/src/main/resources/key_properties.json +++ b/ambari-server/src/main/resources/key_properties.json @@ -150,5 +150,8 @@ }, "KerberosDescriptor": { "KerberosDescriptor": "KerberosDescriptors/kerberos_descriptor_name" + }, + "LoggingQuery": { + "Cluster" : "logging/cluster_name" } } http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogLineResultTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogLineResultTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogLineResultTest.java new file mode 100644 index 0000000..069f448 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogLineResultTest.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.ambari.server.controller.logging; + + +import org.codehaus.jackson.map.AnnotationIntrospector; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.ObjectReader; +import org.codehaus.jackson.map.annotate.JsonSerialize; +import org.codehaus.jackson.map.introspect.JacksonAnnotationIntrospector; +import org.junit.Test; + +import java.io.StringReader; + +import static org.junit.Assert.*; + +public class LogLineResultTest { + + private static final String TEST_JSON_DATA_SINGLE_ENTRY = + "{" + + + " \"cluster\" : \"clusterone\"," + + " \"method\" : \"chooseUnderReplicatedBlocks\"," + + " \"level\" : \"INFO\"," + + " \"event_count\" : 1," + + " \"ip\" : \"192.168.1.1\"," + + " \"type\" : \"hdfs_namenode\"," + + " \"thread_name\" : \"thread-id-one\"," + + " \"seq_num\" : 10584," + + " \"path\" : \"/var/log/hadoop/hdfs/hadoop-hdfs-namenode-c6401.ambari.apache.org.log\"," + + " \"file\" : \"UnderReplicatedBlocks.java\"," + + " \"line_number\" : 394," + + " \"host\" : \"c6401.ambari.apache.org\"," + + " \"log_message\" : \"chooseUnderReplicatedBlocks selected 2 blocks at priority level 0; Total=2 Reset bookmarks? false\"," + + " \"logger_name\" : \"BlockStateChange\"," + + " \"id\" : \"9c5562fb-123f-47c8-aaf5-b5e407326c08\"," + + " \"message_md5\" : \"-3892769501348410581\"," + + " \"logtime\" : 1458148749036," + + " \"event_md5\" : \"1458148749036-2417481968206345035\"," + + " \"logfile_line_number\" : 2084," + + " \"_ttl_\" : \"+7DAYS\"," + + " \"_expire_at_\" : 1458753550322," + + " \"_version_\" : 1528979784023932928" + + " }"; + + + + @Test + public void testBasicParsing() throws Exception { + // setup a reader for the test JSON data + StringReader stringReader = + new StringReader(TEST_JSON_DATA_SINGLE_ENTRY); + + // setup the Jackson mapper/reader to read in the data structure + ObjectMapper mapper = + new ObjectMapper(); + AnnotationIntrospector introspector = + new JacksonAnnotationIntrospector(); + mapper.setAnnotationIntrospector(introspector); + mapper.getSerializationConfig().setSerializationInclusion(JsonSerialize.Inclusion.NON_NULL); + + + ObjectReader logLineResultReader = + mapper.reader(LogLineResult.class); + + LogLineResult result = + logLineResultReader.readValue(stringReader); + + // verify that all fields in this class are parsed as expected + assertEquals("Cluster name not parsed properly", + "clusterone", result.getClusterName()); + assertEquals("Method Name not parsed properly", + "chooseUnderReplicatedBlocks", result.getLogMethod()); + assertEquals("Log Level not parsed properly", + "INFO", result.getLogLevel()); + assertEquals("event_count not parsed properly", + "1", result.getEventCount()); + assertEquals("ip address not parsed properly", + "192.168.1.1", result.getIpAddress()); + assertEquals("component type not parsed properly", + "hdfs_namenode", result.getComponentType()); + assertEquals("sequence number not parsed properly", + "10584", result.getSequenceNumber()); + assertEquals("log file path not parsed properly", + "/var/log/hadoop/hdfs/hadoop-hdfs-namenode-c6401.ambari.apache.org.log", result.getLogFilePath()); + assertEquals("log src file name not parsed properly", + "UnderReplicatedBlocks.java", result.getSourceFile()); + assertEquals("log src line number not parsed properly", + "394", result.getSourceFileLineNumber()); + assertEquals("host name not parsed properly", + "c6401.ambari.apache.org", result.getHostName()); + assertEquals("log message not parsed properly", + "chooseUnderReplicatedBlocks selected 2 blocks at priority level 0; Total=2 Reset bookmarks? false", result.getLogMessage()); + assertEquals("logger name not parsed properly", + "BlockStateChange", result.getLoggerName()); + assertEquals("id not parsed properly", + "9c5562fb-123f-47c8-aaf5-b5e407326c08", result.getId()); + assertEquals("message MD5 not parsed properly", + "-3892769501348410581", result.getMessageMD5()); + assertEquals("log time not parsed properly", + "1458148749036", result.getLogTime()); + assertEquals("event MD5 not parsed properly", + "1458148749036-2417481968206345035", result.getEventMD5()); + assertEquals("logfile line number not parsed properly", + "2084", result.getLogFileLineNumber()); + assertEquals("ttl not parsed properly", + "+7DAYS", result.getTtl()); + assertEquals("expire at not parsed properly", + "1458753550322", result.getExpirationTime()); + assertEquals("version not parsed properly", + "1528979784023932928", result.getVersion()); + assertEquals("thread_name not parsed properly", + "thread-id-one", result.getThreadName()); + + } + + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/20e66fbb/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogQueryResponseTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogQueryResponseTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogQueryResponseTest.java new file mode 100644 index 0000000..d281d6e --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogQueryResponseTest.java @@ -0,0 +1,226 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.logging; + + +import org.codehaus.jackson.map.AnnotationIntrospector; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.ObjectReader; +import org.codehaus.jackson.map.annotate.JsonSerialize; +import org.codehaus.jackson.map.introspect.JacksonAnnotationIntrospector; +import org.junit.Test; + +import java.io.StringReader; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class LogQueryResponseTest { + + private static final String TEST_JSON_INPUT_TWO_LIST_ENTRIES = + "{" + + " \"startIndex\" : 0," + + " \"pageSize\" : 5," + + " \"totalCount\" : 10452," + + " \"resultSize\" : 5," + + " \"queryTimeMS\" : 1458148754113," + + " \"logList\" : [ {" + + " \"cluster\" : \"clusterone\"," + + " \"method\" : \"chooseUnderReplicatedBlocks\"," + + " \"level\" : \"INFO\"," + + " \"event_count\" : 1," + + " \"ip\" : \"192.168.1.1\"," + + " \"type\" : \"hdfs_namenode\"," + + " \"seq_num\" : 10584," + + " \"path\" : \"/var/log/hadoop/hdfs/hadoop-hdfs-namenode-c6401.ambari.apache.org.log\"," + + " \"file\" : \"UnderReplicatedBlocks.java\"," + + " \"line_number\" : 394," + + " \"host\" : \"c6401.ambari.apache.org\"," + + " \"log_message\" : \"chooseUnderReplicatedBlocks selected 2 blocks at priority level 0; Total=2 Reset bookmarks? false\"," + + " \"logger_name\" : \"BlockStateChange\"," + + " \"id\" : \"9c5562fb-123f-47c8-aaf5-b5e407326c08\"," + + " \"message_md5\" : \"-3892769501348410581\"," + + " \"logtime\" : 1458148749036," + + " \"event_md5\" : \"1458148749036-2417481968206345035\"," + + " \"logfile_line_number\" : 2084," + + " \"_ttl_\" : \"+7DAYS\"," + + " \"_expire_at_\" : 1458753550322," + + " \"_version_\" : 1528979784023932928" + + " }, {" + + " \"cluster\" : \"clusterone\"," + + " \"method\" : \"putMetrics\"," + + " \"level\" : \"WARN\"," + + " \"event_count\" : 1," + + " \"ip\" : \"192.168.1.1\"," + + " \"type\" : \"yarn_resourcemanager\"," + + " \"seq_num\" : 10583," + + " \"path\" : \"/var/log/hadoop-yarn/yarn/yarn-yarn-resourcemanager-c6401.ambari.apache.org.log\"," + + " \"file\" : \"HadoopTimelineMetricsSink.java\"," + + " \"line_number\" : 262," + + " \"host\" : \"c6401.ambari.apache.org\"," + + " \"log_message\" : \"Unable to send metrics to collector by address:http://c6401.ambari.apache.org:6188/ws/v1/timeline/metrics\"," + + " \"logger_name\" : \"timeline.HadoopTimelineMetricsSink\"," + + " \"id\" : \"8361c5a9-5b1c-4f44-bc8f-4c6f07d94228\"," + + " \"message_md5\" : \"5942185045779825717\"," + + " \"logtime\" : 1458148746937," + + " \"event_md5\" : \"14581487469371427138486123628676\"," + + " \"logfile_line_number\" : 549," + + " \"_ttl_\" : \"+7DAYS\"," + + " \"_expire_at_\" : 1458753550322," + + " \"_version_\" : 1528979784022884357" + + " }" + + "]" + + "}"; + + + @Test + public void testBasicParsing() throws Exception { + // setup a reader for the test JSON data + StringReader stringReader = + new StringReader(TEST_JSON_INPUT_TWO_LIST_ENTRIES); + + // setup the Jackson mapper/reader to read in the data structure + ObjectMapper mapper = + new ObjectMapper(); + AnnotationIntrospector introspector = + new JacksonAnnotationIntrospector(); + mapper.setAnnotationIntrospector(introspector); + mapper.getSerializationConfig().setSerializationInclusion(JsonSerialize.Inclusion.NON_NULL); + + + ObjectReader logQueryResponseReader = + mapper.reader(LogQueryResponse.class); + + LogQueryResponse result = + logQueryResponseReader.readValue(stringReader); + + assertEquals("startIndex not parsed properly", + "0", result.getStartIndex()); + assertEquals("pageSize not parsed properly", + "5", result.getPageSize()); + assertEquals("totalCount not parsed properly", + "10452", result.getTotalCount()); + assertEquals("resultSize not parsed properly", + "5", result.getResultSize()); + assertEquals("queryTimeMS not parsed properly", + "1458148754113", result.getQueryTimeMS()); + + assertEquals("incorrect number of LogLineResult items parsed", + 2, result.getListOfResults().size()); + + List<LogLineResult> listOfLineResults = + result.getListOfResults(); + + { + LogLineResult resultOne = listOfLineResults.get(0); + // verify that all fields in this class are parsed as expected + assertEquals("Cluster name not parsed properly", + "clusterone", resultOne.getClusterName()); + assertEquals("Method Name not parsed properly", + "chooseUnderReplicatedBlocks", resultOne.getLogMethod()); + assertEquals("Log Level not parsed properly", + "INFO", resultOne.getLogLevel()); + assertEquals("event_count not parsed properly", + "1", resultOne.getEventCount()); + assertEquals("ip address not parsed properly", + "192.168.1.1", resultOne.getIpAddress()); + assertEquals("component type not parsed properly", + "hdfs_namenode", resultOne.getComponentType()); + assertEquals("sequence number not parsed properly", + "10584", resultOne.getSequenceNumber()); + assertEquals("log file path not parsed properly", + "/var/log/hadoop/hdfs/hadoop-hdfs-namenode-c6401.ambari.apache.org.log", resultOne.getLogFilePath()); + assertEquals("log src file name not parsed properly", + "UnderReplicatedBlocks.java", resultOne.getSourceFile()); + assertEquals("log src line number not parsed properly", + "394", resultOne.getSourceFileLineNumber()); + assertEquals("host name not parsed properly", + "c6401.ambari.apache.org", resultOne.getHostName()); + assertEquals("log message not parsed properly", + "chooseUnderReplicatedBlocks selected 2 blocks at priority level 0; Total=2 Reset bookmarks? false", resultOne.getLogMessage()); + assertEquals("logger name not parsed properly", + "BlockStateChange", resultOne.getLoggerName()); + assertEquals("id not parsed properly", + "9c5562fb-123f-47c8-aaf5-b5e407326c08", resultOne.getId()); + assertEquals("message MD5 not parsed properly", + "-3892769501348410581", resultOne.getMessageMD5()); + assertEquals("log time not parsed properly", + "1458148749036", resultOne.getLogTime()); + assertEquals("event MD5 not parsed properly", + "1458148749036-2417481968206345035", resultOne.getEventMD5()); + assertEquals("logfile line number not parsed properly", + "2084", resultOne.getLogFileLineNumber()); + assertEquals("ttl not parsed properly", + "+7DAYS", resultOne.getTtl()); + assertEquals("expire at not parsed properly", + "1458753550322", resultOne.getExpirationTime()); + assertEquals("version not parsed properly", + "1528979784023932928", resultOne.getVersion()); + } + + { + LogLineResult resultTwo = listOfLineResults.get(1); + // verify second log line record's data is parsed correctly + assertEquals("Cluster name not parsed properly", + "clusterone", resultTwo.getClusterName()); + assertEquals("Method Name not parsed properly", + "putMetrics", resultTwo.getLogMethod()); + assertEquals("Log Level not parsed properly", + "WARN", resultTwo.getLogLevel()); + assertEquals("event_count not parsed properly", + "1", resultTwo.getEventCount()); + assertEquals("ip address not parsed properly", + "192.168.1.1", resultTwo.getIpAddress()); + assertEquals("component type not parsed properly", + "yarn_resourcemanager", resultTwo.getComponentType()); + assertEquals("sequence number not parsed properly", + "10583", resultTwo.getSequenceNumber()); + assertEquals("log file path not parsed properly", + "/var/log/hadoop-yarn/yarn/yarn-yarn-resourcemanager-c6401.ambari.apache.org.log", resultTwo.getLogFilePath()); + assertEquals("log src file name not parsed properly", + "HadoopTimelineMetricsSink.java", resultTwo.getSourceFile()); + assertEquals("log src line number not parsed properly", + "262", resultTwo.getSourceFileLineNumber()); + assertEquals("host name not parsed properly", + "c6401.ambari.apache.org", resultTwo.getHostName()); + assertEquals("log message not parsed properly", + "Unable to send metrics to collector by address:http://c6401.ambari.apache.org:6188/ws/v1/timeline/metrics", resultTwo.getLogMessage()); + assertEquals("logger name not parsed properly", + "timeline.HadoopTimelineMetricsSink", resultTwo.getLoggerName()); + assertEquals("id not parsed properly", + "8361c5a9-5b1c-4f44-bc8f-4c6f07d94228", resultTwo.getId()); + assertEquals("message MD5 not parsed properly", + "5942185045779825717", resultTwo.getMessageMD5()); + assertEquals("log time not parsed properly", + "1458148746937", resultTwo.getLogTime()); + assertEquals("event MD5 not parsed properly", + "14581487469371427138486123628676", resultTwo.getEventMD5()); + assertEquals("logfile line number not parsed properly", + "549", resultTwo.getLogFileLineNumber()); + assertEquals("ttl not parsed properly", + "+7DAYS", resultTwo.getTtl()); + assertEquals("expire at not parsed properly", + "1458753550322", resultTwo.getExpirationTime()); + assertEquals("version not parsed properly", + "1528979784022884357", resultTwo.getVersion()); + } + + } + +}