http://git-wip-us.apache.org/repos/asf/ambari/blob/1a225bd2/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java index 1d4f91f..6e319da 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java @@ -18,1246 +18,349 @@ */ package org.apache.ambari.logsearch.manager; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; -import java.text.ParseException; +import java.io.StringWriter; import java.util.ArrayList; -import java.util.Calendar; -import java.util.Collection; import java.util.Date; -import java.util.GregorianCalendar; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.TimeZone; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; import javax.inject.Inject; +import javax.inject.Named; +import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import com.google.common.collect.Lists; +import freemarker.template.Configuration; +import freemarker.template.Template; +import freemarker.template.TemplateException; import org.apache.ambari.logsearch.common.LogSearchConstants; import org.apache.ambari.logsearch.common.MessageEnums; -import org.apache.ambari.logsearch.conf.SolrServiceLogPropsConfig; import org.apache.ambari.logsearch.dao.ServiceLogsSolrDao; import org.apache.ambari.logsearch.graph.GraphDataGenerator; -import org.apache.ambari.logsearch.model.response.BarGraphData; +import org.apache.ambari.logsearch.model.request.impl.ServiceAnyGraphRequest; +import org.apache.ambari.logsearch.model.request.impl.ServiceGraphRequest; +import org.apache.ambari.logsearch.model.request.impl.ServiceLogAggregatedInfoRequest; +import org.apache.ambari.logsearch.model.request.impl.ServiceLogComponentHostRequest; +import org.apache.ambari.logsearch.model.request.impl.ServiceLogComponentLevelRequest; +import org.apache.ambari.logsearch.model.request.impl.ServiceLogExportRequest; +import org.apache.ambari.logsearch.model.request.impl.ServiceLogHostComponentRequest; +import org.apache.ambari.logsearch.model.request.impl.ServiceLogLevelCountRequest; +import org.apache.ambari.logsearch.model.request.impl.ServiceLogRequest; +import org.apache.ambari.logsearch.model.request.impl.ServiceLogTruncatedRequest; import org.apache.ambari.logsearch.model.response.BarGraphDataListResponse; -import org.apache.ambari.logsearch.model.response.CountData; import org.apache.ambari.logsearch.model.response.CountDataListResponse; -import org.apache.ambari.logsearch.model.response.GraphData; import org.apache.ambari.logsearch.model.response.GraphDataListResponse; import org.apache.ambari.logsearch.model.response.GroupListResponse; import org.apache.ambari.logsearch.model.response.LogData; import org.apache.ambari.logsearch.model.response.LogListResponse; -import org.apache.ambari.logsearch.model.response.LogSearchResponse; -import org.apache.ambari.logsearch.model.response.NameValueData; import org.apache.ambari.logsearch.model.response.NameValueDataListResponse; -import org.apache.ambari.logsearch.model.response.NodeData; import org.apache.ambari.logsearch.model.response.NodeListResponse; import org.apache.ambari.logsearch.model.response.ServiceLogData; import org.apache.ambari.logsearch.model.response.ServiceLogResponse; -import org.apache.ambari.logsearch.query.QueryGenerationBase; -import org.apache.ambari.logsearch.query.model.CommonServiceLogSearchCriteria; -import org.apache.ambari.logsearch.query.model.ServiceAnyGraphSearchCriteria; -import org.apache.ambari.logsearch.query.model.ServiceGraphSearchCriteria; -import org.apache.ambari.logsearch.query.model.ServiceLogExportSearchCriteria; -import org.apache.ambari.logsearch.query.model.ServiceLogSearchCriteria; -import org.apache.ambari.logsearch.query.model.ServiceLogTruncatedSearchCriteria; +import org.apache.ambari.logsearch.converter.BaseServiceLogRequestQueryConverter; +import org.apache.ambari.logsearch.converter.ServiceLogTruncatedRequestQueryConverter; import org.apache.ambari.logsearch.solr.model.SolrComponentTypeLogData; import org.apache.ambari.logsearch.solr.model.SolrHostLogData; import org.apache.ambari.logsearch.solr.model.SolrServiceLogData; -import org.apache.ambari.logsearch.util.BizUtil; +import org.apache.ambari.logsearch.util.DownloadUtil; import org.apache.ambari.logsearch.util.DateUtil; -import org.apache.ambari.logsearch.util.FileUtil; import org.apache.ambari.logsearch.util.RESTErrorUtil; import org.apache.ambari.logsearch.util.SolrUtil; -import org.apache.ambari.logsearch.view.VSummary; -import org.apache.ambari.logsearch.query.model.SearchCriteria; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.time.DateUtils; import org.apache.log4j.Logger; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.response.FacetField; import org.apache.solr.client.solrj.response.FacetField.Count; -import org.apache.solr.client.solrj.response.PivotField; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SimpleOrderedMap; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Component; - -import com.google.common.collect.Lists; - -@Component +import org.springframework.core.convert.ConversionService; +import org.springframework.data.solr.core.DefaultQueryParser; +import org.springframework.data.solr.core.query.Criteria; +import org.springframework.data.solr.core.query.SimpleFacetQuery; +import org.springframework.data.solr.core.query.SimpleFilterQuery; +import org.springframework.data.solr.core.query.SimpleQuery; +import org.springframework.data.solr.core.query.SimpleStringCriteria; + +import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.ID; +import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.SEQUENCE_ID; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.COMPONENT; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.HOST; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.KEY_LOG_MESSAGE; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LEVEL; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LOGTIME; + +@Named public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceLogResponse> { - private static final Logger logger = Logger.getLogger(ServiceLogsManager.class); - private static List<String> cancelByDate = new CopyOnWriteArrayList<String>(); - - private static Map<String, String> mapUniqueId = new ConcurrentHashMap<String, String>(); - - private enum CONDITION { - OR, AND - } + private static final String SERVICE_LOG_TEMPLATE = "service_log_txt.ftl"; @Inject private ServiceLogsSolrDao serviceLogsSolrDao; @Inject private GraphDataGenerator graphDataGenerator; @Inject - private SolrServiceLogPropsConfig solrServiceLogPropsConfig; - - public ServiceLogResponse searchLogs(ServiceLogSearchCriteria searchCriteria) { - String keyword = searchCriteria.getKeyword(); - String logId = searchCriteria.getSourceLogId(); - Boolean isLastPage = searchCriteria.isLastPage(); + private ConversionService conversionService; + @Inject + private Configuration freemarkerConfiguration; + public ServiceLogResponse searchLogs(ServiceLogRequest request) { + String event = "/service/logs"; + String keyword = request.getKeyWord(); + Boolean isLastPage = request.isLastPage(); + SimpleQuery solrQuery = conversionService.convert(request, SimpleQuery.class); if (!StringUtils.isBlank(keyword)) { try { - return (ServiceLogResponse) getPageByKeyword(searchCriteria); + return (ServiceLogResponse) getPageByKeyword(request, event); } catch (SolrException | SolrServerException e) { logger.error("Error while getting keyword=" + keyword, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR - .getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } - } else if (!StringUtils.isBlank(logId)) { - try { - return (ServiceLogResponse) getPageByLogId(searchCriteria); - } catch (SolrException e) { - logger.error("Error while getting keyword=" + keyword, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR - .getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); } } else if (isLastPage) { - SolrQuery lastPageQuery = queryGenerator.commonServiceFilterQuery(searchCriteria); - ServiceLogResponse logResponse = getLastPage(searchCriteria,LogSearchConstants.LOGTIME,serviceLogsSolrDao,lastPageQuery); + ServiceLogResponse logResponse = getLastPage(serviceLogsSolrDao, solrQuery, event); if(logResponse == null){ logResponse = new ServiceLogResponse(); } return logResponse; } else { - SolrQuery solrQuery = queryGenerator - .commonServiceFilterQuery(searchCriteria); - - solrQuery.setParam("event", "/service/logs"); - - return getLogAsPaginationProvided(solrQuery, - serviceLogsSolrDao); + return getLogAsPaginationProvided(solrQuery, serviceLogsSolrDao, event); } } public GroupListResponse getHosts() { - return getFields(LogSearchConstants.SOLR_HOST, SolrHostLogData.class); - } - - private <T extends LogData> GroupListResponse getFields(String field, Class<T> clazz) { - - SolrQuery solrQuery = new SolrQuery(); - GroupListResponse collection = new GroupListResponse(); - SolrUtil.setMainQuery(solrQuery, null); - SolrUtil.setFacetField(solrQuery, - field); - SolrUtil.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); - try { - QueryResponse response = serviceLogsSolrDao.process(solrQuery); - if(response == null){ - return collection; - } - FacetField facetField = response - .getFacetField(field); - if (facetField == null){ - return collection; - } - List<Count> fieldList = facetField.getValues(); - if (fieldList == null){ - return collection; - } - SolrDocumentList docList = response.getResults(); - if(docList == null){ - return collection; - } - List<LogData> groupList = getLogDataListByFieldType(clazz, response, fieldList); - - collection.setGroupList(groupList); - if(!docList.isEmpty()){ - collection.setStartIndex((int) docList.getStart()); - collection.setTotalCount(docList.getNumFound()); - } - return collection; - } catch (IOException | SolrServerException | SolrException e) { - logger.error(e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR - .getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } - + return getFields(HOST, SolrHostLogData.class); } public GroupListResponse getComponents() { - return getFields(LogSearchConstants.SOLR_COMPONENT, SolrComponentTypeLogData.class); - } - - public GraphDataListResponse getAggregatedInfo(CommonServiceLogSearchCriteria searchCriteria) { - SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria); - String hierarchy = "host,type,level"; - GraphDataListResponse graphInfo = new GraphDataListResponse(); - try { - SolrUtil.setMainQuery(solrQuery, null); - SolrUtil.setFacetPivot(solrQuery, 1, hierarchy); - QueryResponse response = serviceLogsSolrDao.process(solrQuery); - if (response == null) { - return graphInfo; - } - - List<List<PivotField>> hirarchicalPivotField = new ArrayList<List<PivotField>>(); - List<GraphData> dataList = new ArrayList<>(); - NamedList<List<PivotField>> namedList = response.getFacetPivot(); - if (namedList != null) { - hirarchicalPivotField = namedList.getAll(hierarchy); - } - if (!hirarchicalPivotField.isEmpty()) { - dataList = buidGraphData(hirarchicalPivotField.get(0)); - } - if (!dataList.isEmpty()) { - graphInfo.setGraphData(dataList); - } - - return graphInfo; - } catch (SolrException | SolrServerException | IOException e) { - logger.error("Error during solrQuery=" + solrQuery, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR - .getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } + return getFields(COMPONENT, SolrComponentTypeLogData.class); } - public List<GraphData> buidGraphData(List<PivotField> pivotFields) { - List<GraphData> logList = new ArrayList<>(); - if (pivotFields != null) { - for (PivotField pivotField : pivotFields) { - if (pivotField != null) { - GraphData logLevel = new GraphData(); - logLevel.setName("" + pivotField.getValue()); - logLevel.setCount(Long.valueOf(pivotField.getCount())); - if (pivotField.getPivot() != null) { - logLevel.setDataList(buidGraphData(pivotField.getPivot())); - } - logList.add(logLevel); - } - } - } - return logList; + public GraphDataListResponse getAggregatedInfo(ServiceLogAggregatedInfoRequest request) { + SimpleQuery solrDataQuery = new BaseServiceLogRequestQueryConverter().convert(request); + SolrQuery solrQuery = new DefaultQueryParser().doConstructSolrQuery(solrDataQuery); + String hierarchy = String.format("%s,%s,%s", HOST, COMPONENT, LEVEL); + solrQuery.setQuery("*:*"); + SolrUtil.setFacetPivot(solrQuery, 1, hierarchy); + QueryResponse response = serviceLogsSolrDao.process(solrQuery); + return graphDataGenerator.generateSimpleGraphResponse(response, hierarchy); } - public CountDataListResponse getFieldCount(String field){ - CountDataListResponse collection = new CountDataListResponse(); - List<CountData> vCounts = new ArrayList<>(); - SolrQuery solrQuery = new SolrQuery(); - SolrUtil.setMainQuery(solrQuery, null); - if(field == null){ - return collection; - } - SolrUtil.setFacetField(solrQuery, field); - try { - QueryResponse response = serviceLogsSolrDao.process(solrQuery); - if (response == null){ - return collection; - } - FacetField facetFields = response.getFacetField(field); - if (facetFields == null){ - return collection; - } - List<Count> fieldList = facetFields.getValues(); - - if(fieldList == null){ - return collection; - } - - for (Count cnt : fieldList) { - if (cnt != null) { - CountData vCount = new CountData(); - vCount.setName(cnt.getName()); - vCount.setCount(cnt.getCount()); - vCounts.add(vCount); - } - } - - } catch (SolrException | SolrServerException | IOException e) { - logger.error("Error during solrQuery=" + solrQuery, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR - .getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } - - collection.setvCounts(vCounts); - return collection; - } - - public CountDataListResponse getLogLevelCount() { - return getFieldCount(LogSearchConstants.SOLR_LEVEL); + public CountDataListResponse getFieldCount(String field) { + return graphDataGenerator.generateCountResponseByField(serviceLogsSolrDao.process(conversionService.convert(field, SimpleFacetQuery.class)), field); } public CountDataListResponse getComponentsCount() { - return getFieldCount(LogSearchConstants.SOLR_COMPONENT); + return getFieldCount(COMPONENT); } public CountDataListResponse getHostsCount() { - return getFieldCount(LogSearchConstants.SOLR_HOST); + return getFieldCount(HOST); } - public List<NodeData> buidTreeData(List<PivotField> pivotFields, - List<PivotField> pivotFieldHost, SolrQuery query, - String firstPriority, String secondPriority) { - List<NodeData> extensionTree = new ArrayList<>(); - String hostQuery = null; - if (pivotFields != null) { - // For Host - for (PivotField pivotHost : pivotFields) { - if (pivotHost != null) { - NodeData hostNode = new NodeData(); - String name = (pivotHost.getValue() == null ? "" : ""+ pivotHost.getValue()); - String value = "" + pivotHost.getCount(); - if(!StringUtils.isBlank(name)){ - hostNode.setName(name); - } - if(!StringUtils.isBlank(value)){ - hostNode.setValue(value); - } - if(!StringUtils.isBlank(firstPriority)){ - hostNode.setType(firstPriority); - } - - hostNode.setParent(true); - hostNode.setRoot(true); - PivotField hostPivot = null; - for (PivotField searchHost : pivotFieldHost) { - if (!StringUtils.isBlank(hostNode.getName()) - && hostNode.getName().equals(searchHost.getValue())) { - hostPivot = searchHost; - break; - } - } - List<PivotField> pivotLevelHost = hostPivot.getPivot(); - if (pivotLevelHost != null) { - Collection<NameValueData> logLevelCount = new ArrayList<>(); - for (PivotField pivotLevel : pivotLevelHost) { - if (pivotLevel != null) { - NameValueData vnameValue = new NameValueData(); - String levelName = (pivotLevel.getValue() == null ? "" : "" - + pivotLevel.getValue()); - vnameValue.setName(levelName.toUpperCase()); - vnameValue.setValue("" + pivotLevel.getCount()); - logLevelCount.add(vnameValue); - } - } - hostNode.setLogLevelCount(logLevelCount); - } - - query.addFilterQuery(hostQuery); - List<PivotField> pivotComponents = pivotHost.getPivot(); - // For Components - if (pivotComponents != null) { - Collection<NodeData> componentNodes = new ArrayList<>(); - for (PivotField pivotComp : pivotComponents) { - if (pivotComp != null) { - NodeData compNode = new NodeData(); - String compName = (pivotComp.getValue() == null ? "" : "" - + pivotComp.getValue()); - compNode.setName(compName); - if (!StringUtils.isBlank(secondPriority)) { - compNode.setType(secondPriority); - } - compNode.setValue("" + pivotComp.getCount()); - compNode.setParent(false); - compNode.setRoot(false); - List<PivotField> pivotLevels = pivotComp.getPivot(); - if (pivotLevels != null) { - Collection<NameValueData> logLevelCount = new ArrayList<>(); - for (PivotField pivotLevel : pivotLevels) { - if (pivotLevel != null) { - NameValueData vnameValue = new NameValueData(); - String compLevel = pivotLevel.getValue() == null ? "" - : "" + pivotLevel.getValue(); - vnameValue.setName((compLevel).toUpperCase()); - - vnameValue.setValue("" + pivotLevel.getCount()); - logLevelCount.add(vnameValue); - } - } - compNode.setLogLevelCount(logLevelCount); - } - componentNodes.add(compNode); - }} - hostNode.setChilds(componentNodes); - } - extensionTree.add(hostNode); - }} - } - - return extensionTree; - } - - public NodeListResponse getTreeExtension(ServiceLogSearchCriteria searchCriteria) { - SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria); - solrQuery.setParam("event", "/getTreeExtension"); - - if (searchCriteria.getSortBy() == null) { - searchCriteria.setSortBy(LogSearchConstants.SOLR_HOST); - searchCriteria.setSortType(SolrQuery.ORDER.asc.toString()); - } - queryGenerator.setFilterFacetSort(solrQuery, searchCriteria); - String hostName = "" - + ((searchCriteria.getParamValue("hostName") == null) ? "" - : searchCriteria.getParamValue("hostName")); + public NodeListResponse getTreeExtension(ServiceLogHostComponentRequest request) { + SimpleFacetQuery facetQuery = conversionService.convert(request, SimpleFacetQuery.class); + SolrQuery solrQuery = new DefaultQueryParser().doConstructSolrQuery(facetQuery); + String hostName = request.getHostName() == null ? "" : request.getHostName(); if (!StringUtils.isBlank(hostName)){ - solrQuery.addFilterQuery(LogSearchConstants.SOLR_HOST + ":*" - + hostName + "*"); + solrQuery.addFilterQuery(String.format("%s:*%s*", HOST, hostName)); } - String firstHirarchy = "host,type,level"; - String secondHirarchy = "host,level"; - NodeListResponse list = new NodeListResponse(); - try { - - SolrUtil.setFacetPivot(solrQuery, 1, firstHirarchy, - secondHirarchy); - - QueryResponse response = serviceLogsSolrDao.process(solrQuery); - List<List<PivotField>> listFirstHirarchicalPivotFields = new ArrayList<List<PivotField>>(); - NamedList<List<PivotField>> firstNamedList = response - .getFacetPivot(); - if (firstNamedList != null) { - listFirstHirarchicalPivotFields = firstNamedList - .getAll(firstHirarchy); - } - List<List<PivotField>> listSecondHirarchicalPivotFields = new ArrayList<List<PivotField>>(); - NamedList<List<PivotField>> secondNamedList = response - .getFacetPivot(); - if (secondNamedList != null) { - listSecondHirarchicalPivotFields = secondNamedList - .getAll(secondHirarchy); - } - List<PivotField> firstHirarchicalPivotFields = new ArrayList<PivotField>(); - List<PivotField> secondHirarchicalPivotFields = new ArrayList<PivotField>(); - if (!listFirstHirarchicalPivotFields.isEmpty()) { - firstHirarchicalPivotFields = listFirstHirarchicalPivotFields - .get(0); - } - if (!listSecondHirarchicalPivotFields.isEmpty()) { - secondHirarchicalPivotFields = listSecondHirarchicalPivotFields - .get(0); - } - List<NodeData> dataList = buidTreeData(firstHirarchicalPivotFields, - secondHirarchicalPivotFields, solrQuery, - LogSearchConstants.HOST, LogSearchConstants.COMPONENT); - - list.setvNodeList(dataList); - } catch (SolrException | SolrServerException | IOException e) { - logger.error("Error during solrQuery=" + solrQuery, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR - .getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } - - return list; + QueryResponse response = serviceLogsSolrDao.process(solrQuery, "/service/logs/tree"); + String firstHierarchy = String.format("%s,%s,%s", HOST, COMPONENT, LEVEL); + String secondHierarchy = String.format("%s,%s", HOST, LEVEL); + return graphDataGenerator.generateServiceNodeTreeFromFacetResponse(response, firstHierarchy, secondHierarchy, + LogSearchConstants.HOST, LogSearchConstants.COMPONENT); } - public NodeListResponse getHostListByComponent(ServiceLogSearchCriteria searchCriteria) { - SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria); - solrQuery.setParam("event", "/service/hosts/components"); + public NodeListResponse getHostListByComponent(ServiceLogComponentHostRequest request) { + SimpleFacetQuery facetQuery = conversionService.convert(request, SimpleFacetQuery.class); + SolrQuery solrQuery = new DefaultQueryParser().doConstructSolrQuery(facetQuery); + solrQuery.setFacetSort(request.getSortBy() == null ? HOST: request.getSortBy()); NodeListResponse list = new NodeListResponse(); - if (searchCriteria.getSortBy() == null) { - searchCriteria.setSortBy(LogSearchConstants.SOLR_HOST); - searchCriteria.setSortType(SolrQuery.ORDER.asc.toString()); - } - queryGenerator.setFilterFacetSort(solrQuery, searchCriteria); - String componentName = "" - + ((searchCriteria.getParamValue("componentName") == null) ? "" - : searchCriteria.getParamValue("componentName")); + String componentName = request.getComponentName() == null ? "" : request.getComponentName(); if (!StringUtils.isBlank(componentName)){ - solrQuery.addFilterQuery(LogSearchConstants.SOLR_COMPONENT + ":" + solrQuery.addFilterQuery(COMPONENT + ":" + componentName); - } else { - return list; - } - - String firstHirarchy = "type,host,level"; - String secondHirarchy = "type,level"; - - try { - SolrUtil.setFacetPivot(solrQuery, 1, firstHirarchy, - secondHirarchy); - QueryResponse response = serviceLogsSolrDao.process(solrQuery); - List<List<PivotField>> firstHirarchicalPivotFields = null; - List<List<PivotField>> secondHirarchicalPivotFields = null; - NamedList<List<PivotField>> firstNamedList = response - .getFacetPivot(); - if (firstNamedList != null) { - firstHirarchicalPivotFields = firstNamedList - .getAll(firstHirarchy); - secondHirarchicalPivotFields = firstNamedList - .getAll(secondHirarchy); - } - - if (firstHirarchicalPivotFields == null - || secondHirarchicalPivotFields == null) { - return list; - } - - List<NodeData> dataList = buidTreeData( - firstHirarchicalPivotFields.get(0), - secondHirarchicalPivotFields.get(0), solrQuery, + QueryResponse response = serviceLogsSolrDao.process(solrQuery, "/service/logs/hosts/components"); + String firstHierarchy = String.format("%s,%s,%s", COMPONENT, HOST, LEVEL); + String secondHierarchy = String.format("%s,%s", COMPONENT, LEVEL); + return graphDataGenerator.generateServiceNodeTreeFromFacetResponse(response, firstHierarchy, secondHierarchy, LogSearchConstants.COMPONENT, LogSearchConstants.HOST); - if(dataList == null){ - return list; - } - - list.setvNodeList(dataList); + } else { return list; - } catch (SolrException | SolrServerException | IOException e) { - logger.error("Error during solrQuery=" + solrQuery, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR - .getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); } } - public NameValueDataListResponse getLogsLevelCount(ServiceLogSearchCriteria sc) { - NameValueDataListResponse nameValueList = new NameValueDataListResponse(); - SolrQuery query = queryGenerator.commonServiceFilterQuery(sc); - query.setParam("event", "/service/logs/levels/counts/namevalues"); - List<NameValueData> logsCounts = getLogLevelFacets(query); - nameValueList.setvNameValues(logsCounts); - - return nameValueList; + public NameValueDataListResponse getLogsLevelCount(ServiceLogLevelCountRequest request) { + SimpleFacetQuery facetQuery = conversionService.convert(request, SimpleFacetQuery.class); + QueryResponse response = serviceLogsSolrDao.process(facetQuery, "/service/logs/levels/counts"); + return graphDataGenerator.getNameValueDataListResponseWithDefaults(response, LogSearchConstants.SUPPORTED_LOG_LEVELS); } - public List<NameValueData> getLogLevelFacets(SolrQuery query) { - String defalutValue = "0"; - HashMap<String, String> map = new HashMap<String, String>(); - List<NameValueData> logsCounts = new ArrayList<>(); - try { - SolrUtil.setFacetField(query, LogSearchConstants.SOLR_LEVEL); - List<Count> logLevelCounts = getFacetCounts(query, - LogSearchConstants.SOLR_LEVEL); - if (logLevelCounts == null) { - return logsCounts; - } - for (Count count : logLevelCounts) { - map.put(count.getName().toUpperCase(), "" + count.getCount()); - } - for (String level : LogSearchConstants.SUPPORTED_LOG_LEVEL) { - NameValueData nameValue = new NameValueData(); - String value = map.get(level); - if (StringUtils.isBlank(value)) { - value = defalutValue; - } - nameValue.setName(level); - nameValue.setValue(value); - logsCounts.add(nameValue); - } - } catch (SolrException | SolrServerException | IOException e) { - logger.error("Error during solrQuery=" + query, e); - } - return logsCounts; + public BarGraphDataListResponse getHistogramData(ServiceGraphRequest request) { + SolrQuery solrQuery = conversionService.convert(request, SolrQuery.class); + QueryResponse response = serviceLogsSolrDao.process(solrQuery, "/service/logs/histogram"); + return graphDataGenerator.generateBarGraphDataResponseWithRanges(response, LEVEL, true); } - // Get Facet Count According to FacetFeild - public List<Count> getFacetCounts(SolrQuery solrQuery, String facetField) - throws SolrServerException, IOException, SolrException { - List<Count> list = new ArrayList<FacetField.Count>(); - - QueryResponse response = serviceLogsSolrDao.process(solrQuery); - if(response == null){ - return list; - } - - FacetField field = response.getFacetField(facetField); - if (field == null) { - return list; - } - list = field.getValues(); - - - return list; - } - public LogListResponse getPageByKeyword(ServiceLogSearchCriteria searchCriteria) + public LogListResponse getPageByKeyword(ServiceLogRequest request, String event) throws SolrServerException { String defaultChoice = "0"; - - String key = (String) searchCriteria.getKeyword(); - if(StringUtils.isBlank(key)){ - throw RESTErrorUtil.createRESTException("Keyword was not given", - MessageEnums.DATA_NOT_FOUND); + String key = request.getKeyWord(); + if (StringUtils.isBlank(key)) { + throw RESTErrorUtil.createRESTException("Keyword was not given", MessageEnums.DATA_NOT_FOUND); } - String keyword = SolrUtil.escapeForStandardTokenizer(key); - if(keyword.startsWith("\"") && keyword.endsWith("\"")){ + if (keyword.startsWith("\"") && keyword.endsWith("\"")) { keyword = keyword.substring(1); - keyword = keyword.substring(0, keyword.length()-1); + keyword = keyword.substring(0, keyword.length() - 1); } - keyword = "*" + keyword + "*"; - - - String keyType = (String) searchCriteria.getKeywordType(); - QueryResponse queryResponse = null; - - if (!defaultChoice.equals(keyType)) { - try { - int currentPageNumber = searchCriteria.getPage(); - int maxRows = searchCriteria.getMaxRows(); - String nextPageLogID = ""; - - int lastLogIndexNumber = ((currentPageNumber + 1) - * maxRows); - String nextPageLogTime = ""; - - - // Next Page Start Time Calculation - SolrQuery nextPageLogTimeQuery = queryGenerator - .commonServiceFilterQuery(searchCriteria); - nextPageLogTimeQuery.remove("start"); - nextPageLogTimeQuery.remove("rows"); - nextPageLogTimeQuery.setStart(lastLogIndexNumber); - nextPageLogTimeQuery.setRows(1); - - queryResponse = serviceLogsSolrDao.process( - nextPageLogTimeQuery); - if(queryResponse == null){ - throw RESTErrorUtil.createRESTException("The keyword "+"\""+key+"\""+" was not found", - MessageEnums.ERROR_SYSTEM); - } - - SolrDocumentList docList = queryResponse.getResults(); - if(docList ==null){ - throw RESTErrorUtil.createRESTException("The keyword "+"\""+key+"\""+" was not found", - MessageEnums.ERROR_SYSTEM); - } - - SolrDocument solrDoc = docList.get(0); - - Date logDate = (Date) solrDoc.get(LogSearchConstants.LOGTIME); - if(logDate == null){ - throw RESTErrorUtil.createRESTException("The keyword "+"\""+key+"\""+" was not found", - MessageEnums.ERROR_SYSTEM); - } - nextPageLogTime = DateUtil - .convertDateWithMillisecondsToSolrDate(logDate); - nextPageLogID = "" - + solrDoc.get(LogSearchConstants.ID); - - if (StringUtils.isBlank(nextPageLogID)){ - nextPageLogID = "0"; - } - - String filterQueryListIds = ""; - // Remove the same Time Ids - SolrQuery listRemoveIds = queryGenerator - .commonServiceFilterQuery(searchCriteria); - listRemoveIds.remove("start"); - listRemoveIds.remove("rows"); - queryGenerator.setSingleIncludeFilter(listRemoveIds, - LogSearchConstants.LOGTIME, "\"" + nextPageLogTime + "\""); - queryGenerator.setSingleExcludeFilter(listRemoveIds, - LogSearchConstants.ID, nextPageLogID); - SolrUtil.setFl(listRemoveIds, LogSearchConstants.ID); - queryResponse = serviceLogsSolrDao.process( - listRemoveIds); - if(queryResponse == null){ - throw RESTErrorUtil.createRESTException("The keyword "+"\""+key+"\""+" was not found", - MessageEnums.ERROR_SYSTEM); - } - - SolrDocumentList docListIds = queryResponse.getResults(); - if(docListIds ==null){ - throw RESTErrorUtil.createRESTException("The keyword "+"\""+key+"\""+" was not found", - MessageEnums.ERROR_SYSTEM); - } - boolean isFirst = true; - for (SolrDocument solrDocId : docListIds ) { - String id = "" + solrDocId.get(LogSearchConstants.ID); - if (isFirst) { - filterQueryListIds += LogSearchConstants.MINUS_OPERATOR + LogSearchConstants.ID + ":" + id; - isFirst = false; - } else { - filterQueryListIds += " "+CONDITION.AND+" " + LogSearchConstants.MINUS_OPERATOR + LogSearchConstants.ID + ":" + id; - } - } - - // Keyword Sequence Number Calculation - String endTime = searchCriteria.getTo(); - String startTime = searchCriteria.getFrom(); - SolrQuery logTimeThroughRangeQuery = queryGenerator - .commonServiceFilterQuery(searchCriteria); - logTimeThroughRangeQuery.remove("start"); - logTimeThroughRangeQuery.remove("rows"); - logTimeThroughRangeQuery.setRows(1); - if (!StringUtils.isBlank(filterQueryListIds)){ - logTimeThroughRangeQuery.setFilterQueries(filterQueryListIds); - } - - String sortByType = searchCriteria.getSortType(); - - if (!StringUtils.isBlank(sortByType) && sortByType - .equalsIgnoreCase(LogSearchConstants.ASCENDING_ORDER)) { - - queryGenerator.setSingleRangeFilter(logTimeThroughRangeQuery, - LogSearchConstants.LOGTIME, nextPageLogTime, - endTime); - logTimeThroughRangeQuery.set(LogSearchConstants.SORT, - LogSearchConstants.LOGTIME + " " - + LogSearchConstants.ASCENDING_ORDER); - - } else { - - queryGenerator.setSingleRangeFilter(logTimeThroughRangeQuery, - LogSearchConstants.LOGTIME, startTime, - nextPageLogTime); - logTimeThroughRangeQuery.set(LogSearchConstants.SORT, - LogSearchConstants.LOGTIME + " " - + LogSearchConstants.DESCENDING_ORDER); - } - queryGenerator.setSingleIncludeFilter(logTimeThroughRangeQuery, - LogSearchConstants.SOLR_KEY_LOG_MESSAGE, keyword); - - - queryResponse = serviceLogsSolrDao.process( - logTimeThroughRangeQuery); - if(queryResponse == null){ - throw RESTErrorUtil.createRESTException("The keyword "+"\""+key+"\""+" was not found", - MessageEnums.ERROR_SYSTEM); - } - - SolrDocumentList documentList = queryResponse.getResults(); - if(documentList ==null){ - throw RESTErrorUtil.createRESTException("The keyword "+"\""+key+"\""+" was not found", - MessageEnums.ERROR_SYSTEM); - } - - SolrDocument solrDocument = new SolrDocument(); - if (!documentList.isEmpty()){ - solrDocument = documentList.get(0); - } - - Date keywordLogDate = (Date) solrDocument.get(LogSearchConstants.LOGTIME); - if(keywordLogDate == null){ - throw RESTErrorUtil.createRESTException("The keyword "+"\""+key+"\""+" was not found", - MessageEnums.ERROR_SYSTEM); - } - String originalKeywordDate = DateUtil - .convertDateWithMillisecondsToSolrDate(keywordLogDate); - String keywordId = "" + solrDocument.get(LogSearchConstants.ID); - - // Getting Range Count from StartTime To Keyword Log Time - SolrQuery rangeLogQuery = nextPageLogTimeQuery.getCopy(); - rangeLogQuery.remove("start"); - rangeLogQuery.remove("rows"); - - if (!StringUtils.isBlank(sortByType) && sortByType - .equalsIgnoreCase(LogSearchConstants.ASCENDING_ORDER)) { - keywordLogDate = DateUtils.addMilliseconds(keywordLogDate, 1); - String keywordDateTime = DateUtil - .convertDateWithMillisecondsToSolrDate(keywordLogDate); - queryGenerator.setSingleRangeFilter(rangeLogQuery, - LogSearchConstants.LOGTIME, startTime, - keywordDateTime); - } else { - keywordLogDate = DateUtils.addMilliseconds(keywordLogDate, -1); - String keywordDateTime = DateUtil - .convertDateWithMillisecondsToSolrDate(keywordLogDate); - queryGenerator.setSingleRangeFilter(rangeLogQuery, - LogSearchConstants.LOGTIME, keywordDateTime, - endTime); - } + boolean isNext = !defaultChoice.equals(request.getKeywordType()); // 1 is next, 0 is previous + return getPageForKeywordByType(request, keyword, isNext, event); + } - long countNumberLogs = countQuery(rangeLogQuery,serviceLogsSolrDao) - 1; + private LogListResponse getPageForKeywordByType(ServiceLogRequest request, String keyword, boolean isNext, String event) { + String fromDate = request.getFrom(); // store start & end dates + String toDate = request.getTo(); + boolean timeAscending = LogSearchConstants.ASCENDING_ORDER.equals(request.getSortType()); + int currentPageNumber = Integer.parseInt(request.getPage()); + int maxRows = Integer.parseInt(request.getPageSize()); + Date logDate = getDocDateFromNextOrLastPage(request, keyword, isNext, currentPageNumber, maxRows); + if (logDate == null) { + throw RESTErrorUtil.createRESTException("The keyword " + "\"" + keyword + "\"" + " was not found", MessageEnums.ERROR_SYSTEM); + } - //Adding numbers on + String nextOrPreviousPageDate = DateUtil.convertDateWithMillisecondsToSolrDate(logDate); + SolrServiceLogData firstKeywordLog = getNextHitForKeyword(request, keyword, isNext, event, timeAscending, nextOrPreviousPageDate); + long keywordSeqNum = firstKeywordLog.getSeqNum(); + String keywordLogtime = DateUtil.convertDateWithMillisecondsToSolrDate(firstKeywordLog.getLogTime()); - try { - SolrQuery sameIdQuery = queryGenerator - .commonServiceFilterQuery(searchCriteria); - queryGenerator.setSingleIncludeFilter(sameIdQuery, - LogSearchConstants.LOGTIME, "\"" + originalKeywordDate + "\""); - SolrUtil.setFl(sameIdQuery, LogSearchConstants.ID); - SolrDocumentList sameQueryDocList = serviceLogsSolrDao.process(sameIdQuery) - .getResults(); - for (SolrDocument solrDocumenent : sameQueryDocList) { - String id = (String) solrDocumenent - .getFieldValue(LogSearchConstants.ID); - countNumberLogs++; - - if (StringUtils.isBlank(id) && id.equals(keywordId)){ - break; - } - } - } catch (SolrException | SolrServerException | IOException e) { - logger.error(e); - } + long numberOfDateDuplicates = countNumberOfDuplicates(request, isNext, keywordSeqNum, keywordLogtime); - int start = (int) ((countNumberLogs / maxRows) * maxRows); - SolrQuery logIdQuery = nextPageLogTimeQuery.getCopy(); - rangeLogQuery.remove("start"); - rangeLogQuery.remove("rows"); - logIdQuery.setStart(start); - logIdQuery.setRows(searchCriteria.getMaxRows()); - return getLogAsPaginationProvided(logIdQuery, serviceLogsSolrDao); + long numberOfLogsUntilFound = getNumberOfLogsUntilFound(request, fromDate, toDate, timeAscending, keywordLogtime, numberOfDateDuplicates); + int start = (int) ((numberOfLogsUntilFound / maxRows)); - } catch (Exception e) { - //do nothing - } + request.setFrom(fromDate); + request.setTo(toDate); + request.setPage(String.valueOf(start)); + SolrQuery keywordNextPageQuery = new DefaultQueryParser().doConstructSolrQuery(conversionService.convert(request, SimpleQuery.class)); + return getLogAsPaginationProvided(keywordNextPageQuery, serviceLogsSolrDao, event); + } + private Long getNumberOfLogsUntilFound(ServiceLogRequest request, String fromDate, String toDate, boolean timeAscending, + String keywordLogtime, long numberOfDateDuplicates) { + if (!timeAscending) { + request.setTo(toDate); + request.setFrom(keywordLogtime); } else { - try { - int currentPageNumber = searchCriteria.getPage(); - int maxRows = searchCriteria.getMaxRows(); - - if (currentPageNumber == 0) { - throw RESTErrorUtil.createRESTException("This is first Page Not", - MessageEnums.DATA_NOT_FOUND); - } - - int firstLogCurrentPage = (currentPageNumber * maxRows); - String lastLogsLogTime = ""; - - // Next Page Start Time Calculation - SolrQuery lastLogTime = queryGenerator - .commonServiceFilterQuery(searchCriteria); - lastLogTime.remove("start"); - lastLogTime.remove("rows"); - - lastLogTime.setStart(firstLogCurrentPage); - lastLogTime.setRows(1); - - queryResponse = serviceLogsSolrDao.process( - lastLogTime); - if(queryResponse == null){ - throw RESTErrorUtil.createRESTException("The keyword "+"\""+key+"\""+" was not found", - MessageEnums.ERROR_SYSTEM); - } - - SolrDocumentList docList = queryResponse.getResults(); - if(docList ==null){ - throw RESTErrorUtil.createRESTException("The keyword "+"\""+key+"\""+" was not found", - MessageEnums.ERROR_SYSTEM); - } - SolrDocument solrDoc = docList.get(0); - - Date logDate = (Date) solrDoc.get(LogSearchConstants.LOGTIME); - String sortByType = searchCriteria.getSortType(); - lastLogsLogTime = DateUtil - .convertDateWithMillisecondsToSolrDate(logDate); - String lastLogsLogId = "" - + solrDoc.get(LogSearchConstants.ID); - - - String filterQueryListIds = ""; - // Remove the same Time Ids - SolrQuery listRemoveIds = queryGenerator - .commonServiceFilterQuery(searchCriteria); - listRemoveIds.remove("start"); - listRemoveIds.remove("rows"); - queryGenerator.setSingleIncludeFilter(listRemoveIds, - LogSearchConstants.LOGTIME, "\"" + lastLogsLogTime + "\""); - queryGenerator.setSingleExcludeFilter(listRemoveIds, - LogSearchConstants.ID, lastLogsLogId); - SolrUtil.setFl(listRemoveIds, LogSearchConstants.ID); - queryResponse = serviceLogsSolrDao.process( - lastLogTime); - if(queryResponse == null){ - throw RESTErrorUtil.createRESTException("The keyword "+"\""+key+"\""+" was not found", - MessageEnums.ERROR_SYSTEM); - } - - SolrDocumentList docListIds = queryResponse.getResults(); - if(docListIds == null){ - throw RESTErrorUtil.createRESTException("The keyword "+"\""+key+"\""+" was not found", - MessageEnums.ERROR_SYSTEM); - } - boolean isFirst = true; - for (SolrDocument solrDocId : docListIds) { - if (solrDocId != null) { - String id = "" + solrDocId.get(LogSearchConstants.ID); - if (isFirst) { - filterQueryListIds += LogSearchConstants.MINUS_OPERATOR + LogSearchConstants.ID + ":" + id; - isFirst = false; - } else { - filterQueryListIds += " "+CONDITION.AND+" " + LogSearchConstants.MINUS_OPERATOR + LogSearchConstants.ID + ":" - + id; - } - } - } - - - // Keyword LogTime Calculation - String endTime = (String) searchCriteria.getTo(); - String startTime = searchCriteria.getFrom(); - SolrQuery logTimeThroughRangeQuery = queryGenerator - .commonServiceFilterQuery(searchCriteria); - logTimeThroughRangeQuery.remove("start"); - logTimeThroughRangeQuery.remove("rows"); - logTimeThroughRangeQuery.setRows(1); - queryGenerator.setSingleExcludeFilter(logTimeThroughRangeQuery, - LogSearchConstants.ID, lastLogsLogId); - if (!StringUtils.isBlank(filterQueryListIds)){ - logTimeThroughRangeQuery.setFilterQueries(filterQueryListIds); - } - - if (!StringUtils.isBlank(sortByType) && sortByType - .equalsIgnoreCase(LogSearchConstants.ASCENDING_ORDER)) { - - logTimeThroughRangeQuery.remove(LogSearchConstants.SORT); - logTimeThroughRangeQuery.set(LogSearchConstants.SORT, - LogSearchConstants.LOGTIME + " " - + LogSearchConstants.DESCENDING_ORDER); - - - queryGenerator.setSingleRangeFilter( - logTimeThroughRangeQuery, - LogSearchConstants.LOGTIME, startTime, - lastLogsLogTime); - - } else { - - logTimeThroughRangeQuery.remove(LogSearchConstants.SORT); - logTimeThroughRangeQuery.set(LogSearchConstants.SORT, - LogSearchConstants.LOGTIME + " " - + LogSearchConstants.ASCENDING_ORDER); - - - queryGenerator.setSingleRangeFilter(logTimeThroughRangeQuery, - LogSearchConstants.LOGTIME, lastLogsLogTime, endTime); - } - queryGenerator.setSingleIncludeFilter(logTimeThroughRangeQuery, - LogSearchConstants.SOLR_KEY_LOG_MESSAGE, keyword); - - - queryResponse = serviceLogsSolrDao.process( - logTimeThroughRangeQuery); - if(queryResponse == null){ - throw RESTErrorUtil.createRESTException("The keyword "+"\""+key+"\""+" was not found", - MessageEnums.ERROR_SYSTEM); - } - - SolrDocumentList documentList = queryResponse.getResults(); - if(documentList == null){ - throw RESTErrorUtil.createRESTException("The keyword "+"\""+key+"\""+" was not found", - MessageEnums.ERROR_SYSTEM); - } - SolrDocument solrDocument = new SolrDocument(); - if (!documentList.isEmpty()){ - solrDocument = documentList.get(0); - } - - Date keywordLogDate = (Date) solrDocument.get(LogSearchConstants.LOGTIME); - if(keywordLogDate == null){ - throw RESTErrorUtil.createRESTException("The keyword "+"\""+key+"\""+" was not found", - MessageEnums.ERROR_SYSTEM); - } - String originalKeywordDate = DateUtil - .convertDateWithMillisecondsToSolrDate(keywordLogDate); - String keywordId = "" + solrDocument.get(LogSearchConstants.ID); - - // Getting Range Count from StartTime To Keyword Log Time - SolrQuery rangeLogQuery = lastLogTime.getCopy(); - rangeLogQuery.remove("start"); - rangeLogQuery.remove("rows"); - - if (!StringUtils.isBlank(sortByType) && sortByType - .equalsIgnoreCase(LogSearchConstants.ASCENDING_ORDER)) { - // keywordLogDate = DateUtil.addMilliSecondsToDate(keywordLogDate, 1); - String keywordDateTime = DateUtil - .convertDateWithMillisecondsToSolrDate(keywordLogDate); - queryGenerator.setSingleRangeFilter(rangeLogQuery, - LogSearchConstants.LOGTIME, startTime, - keywordDateTime); - - - } else { - // keywordLogDate = DateUtil.addMilliSecondsToDate(keywordLogDate, -1); - String keywordDateTime = DateUtil - .convertDateWithMillisecondsToSolrDate(keywordLogDate); - queryGenerator.setSingleRangeFilter(rangeLogQuery, - LogSearchConstants.LOGTIME, keywordDateTime, - endTime); - } - - - long countNumberLogs = countQuery(rangeLogQuery,serviceLogsSolrDao) - 1; - - //Adding numbers on - try { - SolrQuery sameIdQuery = queryGenerator - .commonServiceFilterQuery(searchCriteria); - queryGenerator.setSingleIncludeFilter(sameIdQuery, - LogSearchConstants.LOGTIME, "\"" + originalKeywordDate + "\""); - SolrUtil.setFl(sameIdQuery, LogSearchConstants.ID); - SolrDocumentList sameQueryDocList = serviceLogsSolrDao.process(sameIdQuery) - .getResults(); - for (SolrDocument solrDocumenent : sameQueryDocList) { - if (solrDocumenent != null) { - String id = (String) solrDocumenent - .getFieldValue(LogSearchConstants.ID); - countNumberLogs++; - if ( StringUtils.isBlank(id) && id.equals(keywordId)) { - break; - } - } - } - } catch (SolrException | SolrServerException | IOException e) { - logger.error(e); - } - int start = (int) ((countNumberLogs / maxRows) * maxRows); - - SolrQuery logIdQuery = lastLogTime.getCopy(); - rangeLogQuery.remove("start"); - rangeLogQuery.remove("rows"); - logIdQuery.setStart(start); - logIdQuery.setRows(searchCriteria.getMaxRows()); - return getLogAsPaginationProvided(logIdQuery, serviceLogsSolrDao); - } catch (Exception e) { - //do nothing - } - + request.setTo(keywordLogtime); + request.setFrom(fromDate); } - throw RESTErrorUtil.createRESTException("The keyword "+"\""+key+"\""+" was not found", - MessageEnums.ERROR_SYSTEM); + SimpleQuery rangeQuery = conversionService.convert(request, SimpleQuery.class); + return serviceLogsSolrDao.count(rangeQuery) - numberOfDateDuplicates; } - private LogSearchResponse getPageByLogId(ServiceLogSearchCriteria searchCriteria) { - LogSearchResponse logResponse = new ServiceLogResponse(); - String endLogTime = (String) searchCriteria.getParamValue("to"); - if(StringUtils.isBlank(endLogTime)){ - return logResponse; - } - long startIndex = 0l; - - String logId = (String) searchCriteria.getParamValue("sourceLogId"); - if(StringUtils.isBlank(logId)){ - return logResponse; + private long countNumberOfDuplicates(ServiceLogRequest request, boolean isNext, long keywordSeqNum, String keywordLogtime) { + request.setFrom(keywordLogtime); + request.setTo(keywordLogtime); + SimpleQuery duplicationsQuery = conversionService.convert(request, SimpleQuery.class); + if (isNext) { + duplicationsQuery.addFilterQuery(new SimpleFilterQuery(new SimpleStringCriteria(String.format("%s:[* TO %d]", SEQUENCE_ID, keywordSeqNum - 1)))); + } else { + duplicationsQuery.addFilterQuery(new SimpleFilterQuery(new SimpleStringCriteria(String.format("%s:[%d TO *]", SEQUENCE_ID, keywordSeqNum + 1)))); } - SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria); - - String endTimeMinusOneMilli = ""; - String logTime = ""; - try { - - SolrQuery logTimeByIdQuery = new SolrQuery(); - SolrUtil.setMainQuery(logTimeByIdQuery, null); - queryGenerator.setSingleIncludeFilter(logTimeByIdQuery, - LogSearchConstants.ID, logId); - SolrUtil.setRowCount(solrQuery, 1); - - QueryResponse queryResponse = serviceLogsSolrDao - .process(logTimeByIdQuery); - - if(queryResponse == null){ - return new ServiceLogResponse(); - } - - SolrDocumentList docList = queryResponse.getResults(); - Date dateOfLogId = null; - if (docList != null && !docList.isEmpty()) { - SolrDocument dateLogIdDoc = docList.get(0); - if(dateLogIdDoc != null){ - dateOfLogId = (Date) dateLogIdDoc.get(LogSearchConstants.LOGTIME); - } - } - - if (dateOfLogId != null) { - logTime = DateUtil.convertDateWithMillisecondsToSolrDate(dateOfLogId); - Date endDate = DateUtils.addMilliseconds(dateOfLogId, 1); - endTimeMinusOneMilli = (String) DateUtil - .convertDateWithMillisecondsToSolrDate(endDate); - } + return serviceLogsSolrDao.count(duplicationsQuery); + } - } catch (SolrException | SolrServerException | IOException e) { - logger.error(e); + private SolrServiceLogData getNextHitForKeyword(ServiceLogRequest request, String keyword, boolean isNext, String event, boolean timeAscending, String nextOrPreviousPageDate) { + if (hasNextOrAscOrder(isNext, timeAscending)) { + request.setTo(nextOrPreviousPageDate); + } else { + request.setFrom(nextOrPreviousPageDate); + } + SimpleQuery keywordNextQuery = conversionService.convert(request, SimpleQuery.class); + keywordNextQuery.addFilterQuery(new SimpleFilterQuery(new Criteria(KEY_LOG_MESSAGE).contains(keyword))); + keywordNextQuery.setRows(1); + SolrQuery kewordNextSolrQuery = new DefaultQueryParser().doConstructSolrQuery(keywordNextQuery); + kewordNextSolrQuery.setStart(0); + if (hasNextOrAscOrder(isNext, timeAscending)) { + kewordNextSolrQuery.setSort(LOGTIME, SolrQuery.ORDER.desc); + } else { + kewordNextSolrQuery.setSort(LOGTIME, SolrQuery.ORDER.asc); } - - try { - solrQuery.remove(LogSearchConstants.ID); - solrQuery.remove(LogSearchConstants.LOGTIME); - queryGenerator.setSingleRangeFilter(solrQuery, - LogSearchConstants.LOGTIME, endTimeMinusOneMilli, endLogTime); - SolrUtil.setRowCount(solrQuery, 0); - startIndex = countQuery(solrQuery,serviceLogsSolrDao); - } catch (SolrException | SolrServerException | IOException e) { - logger.error(e); + kewordNextSolrQuery.addSort(SEQUENCE_ID, SolrQuery.ORDER.desc); + QueryResponse queryResponse = serviceLogsSolrDao.process(kewordNextSolrQuery, event); + if (queryResponse == null) { + throw RESTErrorUtil.createRESTException("The keyword " + "\"" + keyword + "\"" + " was not found", MessageEnums.ERROR_SYSTEM); } - - try { - SolrQuery sameIdQuery = queryGenerator.commonServiceFilterQuery(searchCriteria); - queryGenerator.setSingleIncludeFilter(sameIdQuery, - LogSearchConstants.LOGTIME, "\"" + logTime + "\""); - sameIdQuery.set("fl", LogSearchConstants.ID); - - QueryResponse sameIdResponse = serviceLogsSolrDao.process(sameIdQuery); - SolrDocumentList docList = sameIdResponse.getResults(); - - for (SolrDocument solrDocumenent : docList) { - String id = (String) solrDocumenent - .getFieldValue(LogSearchConstants.ID); - startIndex++; - if (!StringUtils.isBlank(id)) { - if (id.equals(logId)) { - break; - } - } - } - - SolrQuery logIdQuery = queryGenerator.commonServiceFilterQuery(searchCriteria); - logIdQuery.remove("rows"); - logIdQuery.remove("start"); - int start = (int) ((startIndex / searchCriteria.getMaxRows()) * searchCriteria - .getMaxRows()); - logIdQuery.setStart(start); - logIdQuery.setRows(searchCriteria.getMaxRows()); - logResponse = getLogAsPaginationProvided(logIdQuery, - serviceLogsSolrDao); - return logResponse; - } catch (SolrException | SolrServerException | IOException e) { - logger.error(e); + List<SolrServiceLogData> solrServiceLogDataList = queryResponse.getBeans(SolrServiceLogData.class); + if (!CollectionUtils.isNotEmpty(solrServiceLogDataList)) { + throw RESTErrorUtil.createRESTException("The keyword " + "\"" + keyword + "\"" + " was not found", MessageEnums.ERROR_SYSTEM); } - - throw RESTErrorUtil.createRESTException("LogId not Found", - MessageEnums.ERROR_SYSTEM); + return solrServiceLogDataList.get(0); } - @SuppressWarnings("unchecked") - public BarGraphDataListResponse getHistogramData(ServiceGraphSearchCriteria searchCriteria) { - String deafalutValue = "0"; - BarGraphDataListResponse dataList = new BarGraphDataListResponse(); - SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria); - solrQuery.set("event", "/audit/logs/histogram"); - String from = getFrom(searchCriteria.getFrom()); - String to = getTo(searchCriteria.getTo()); - String unit = getUnit(searchCriteria.getUnit()); - - List<BarGraphData> histogramData = new ArrayList<>(); - - String jsonHistogramQuery = queryGenerator - .buildJSONFacetTermTimeRangeQuery( - LogSearchConstants.SOLR_LEVEL, - LogSearchConstants.LOGTIME, from, to, unit).replace( - "\\", ""); - - try { - SolrUtil.setJSONFacet(solrQuery, jsonHistogramQuery); - SolrUtil.setRowCount(solrQuery,Integer.parseInt(deafalutValue)); - QueryResponse response = serviceLogsSolrDao.process(solrQuery); - if (response == null){ - return dataList; - } - SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response - .getResponse().get("facets"); - - if (jsonFacetResponse == null - || jsonFacetResponse.toString().equals("{count=0}")){ - return dataList; - } - - serviceLogsSolrDao.extractValuesFromBuckets(jsonFacetResponse, "x", "y", histogramData); - - Collection<NameValueData> vNameValues = new ArrayList<NameValueData>(); - List<BarGraphData> graphDatas = new ArrayList<BarGraphData>(); - for (String level : LogSearchConstants.SUPPORTED_LOG_LEVEL) { - boolean isLevelPresent = false; - BarGraphData vData1 = null; - for (BarGraphData vData2 : histogramData) { - String name = vData2.getName(); - if (level.contains(name)) { - isLevelPresent = true; - vData1 = vData2; - break; - } - if (vNameValues.isEmpty()) { - Collection<NameValueData> vNameValues2 = vData2 - .getDataCount(); - for (NameValueData value : vNameValues2) { - NameValueData value2 = new NameValueData(); - value2.setValue(deafalutValue); - value2.setName(value.getName()); - vNameValues.add(value2); - } - } - } - if (!isLevelPresent) { - BarGraphData vBarGraphData = new BarGraphData(); - vBarGraphData.setName(level); - vBarGraphData.setDataCount(vNameValues); - graphDatas.add(vBarGraphData); - } else { - graphDatas.add(vData1); - } + private Date getDocDateFromNextOrLastPage(ServiceLogRequest request, String keyword, boolean isNext, int currentPageNumber, int maxRows) { + int lastOrFirstLogIndex; + if (isNext) { + lastOrFirstLogIndex = ((currentPageNumber + 1) * maxRows); + } else { + if (currentPageNumber == 0) { + throw RESTErrorUtil.createRESTException("This is the first Page", MessageEnums.DATA_NOT_FOUND); } - - dataList.setGraphData(graphDatas); - return dataList; - - } catch (SolrServerException | SolrException | IOException e) { - logger.error(e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR - .getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - + lastOrFirstLogIndex = (currentPageNumber * maxRows) - 1; } - } + SimpleQuery sq = conversionService.convert(request, SimpleQuery.class); + SolrQuery nextPageLogTimeQuery = new DefaultQueryParser().doConstructSolrQuery(sq); + nextPageLogTimeQuery.remove("start"); + nextPageLogTimeQuery.remove("rows"); + nextPageLogTimeQuery.setStart(lastOrFirstLogIndex); + nextPageLogTimeQuery.setRows(1); - public String cancelFindRequestByDate(String uniqueId) { - if (StringUtils.isEmpty(uniqueId)) { - logger.error("Unique id is Empty"); - throw RESTErrorUtil.createRESTException("Unique id is Empty", - MessageEnums.DATA_NOT_FOUND); + QueryResponse queryResponse = serviceLogsSolrDao.process(nextPageLogTimeQuery); + if (queryResponse == null) { + throw RESTErrorUtil.createRESTException(String.format("Cannot process next page query for \"%s\" ", keyword), MessageEnums.ERROR_SYSTEM); } - - if (cancelByDate.remove(uniqueId)) { - mapUniqueId.remove(uniqueId); - return "Cancel Request Successfully Procssed "; + SolrDocumentList docList = queryResponse.getResults(); + if (docList == null || docList.isEmpty()) { + throw RESTErrorUtil.createRESTException(String.format("Next page element for \"%s\" is not found", keyword), MessageEnums.ERROR_SYSTEM); } - return "Cancel Request Unable to Process"; + + SolrDocument solrDoc = docList.get(0); + return (Date) solrDoc.get(LOGTIME); } - public boolean cancelRequest(String uniqueId) { - if (StringUtils.isBlank(uniqueId)) { - logger.error("Unique id is Empty"); - throw RESTErrorUtil.createRESTException("Unique id is Empty", - MessageEnums.DATA_NOT_FOUND); - } - for (String date : cancelByDate) { - if (uniqueId.equalsIgnoreCase(date)){ - return false; - } - } - return true; + private boolean hasNextOrAscOrder(boolean isNext, boolean timeAscending) { + return isNext && !timeAscending || !isNext && timeAscending; } - public Response exportToTextFile(ServiceLogExportSearchCriteria searchCriteria) { + public Response export(ServiceLogExportRequest request) { String defaultFormat = "text"; - SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria); - String from = searchCriteria.getFrom(); - String to = searchCriteria.getTo(); - String utcOffset = searchCriteria.getUtcOffset(); - String format = searchCriteria.getFormat(); - - format = defaultFormat.equalsIgnoreCase(format) && format != null ? ".txt" - : ".json"; - - if(StringUtils.isBlank(utcOffset)){ - utcOffset = "0"; - } + SimpleQuery solrQuery = conversionService.convert(request, SimpleQuery.class); + String from = request.getFrom(); + String to = request.getTo(); + String utcOffset = StringUtils.isBlank(request.getUtcOffset()) ? "0" : request.getUtcOffset(); + String format = request.getFormat() != null && defaultFormat.equalsIgnoreCase(request.getFormat()) ? ".txt" : ".json"; + String fileName = "Component_Logs_" + DateUtil.getCurrentDateInString(); if (!DateUtil.isDateValid(from) || !DateUtil.isDateValid(to)) { - logger.error("Not valid date format. Valid format should be" - + LogSearchConstants.SOLR_DATE_FORMAT_PREFIX_Z); + logger.error("Not valid date format. Valid format should be" + LogSearchConstants.SOLR_DATE_FORMAT_PREFIX_Z); throw RESTErrorUtil.createRESTException("Not valid date format. Valid format should be" - + LogSearchConstants.SOLR_DATE_FORMAT_PREFIX_Z, - MessageEnums.INVALID_INPUT_DATA); + + LogSearchConstants.SOLR_DATE_FORMAT_PREFIX_Z, MessageEnums.INVALID_INPUT_DATA); } else { from = from.replace("T", " "); @@ -1266,91 +369,28 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL to = to.replace("T", " "); to = to.replace(".", ","); - to = DateUtil.addOffsetToDate(to, Long.parseLong(utcOffset), - "yyyy-MM-dd HH:mm:ss,SSS"); - from = DateUtil.addOffsetToDate(from, Long.parseLong(utcOffset), - "yyyy-MM-dd HH:mm:ss,SSS"); - } - - String fileName = DateUtil.getCurrentDateInString(); - if (searchCriteria.getParamValue("hostLogFile") != null - && searchCriteria.getParamValue("compLogFile") != null) { - fileName = searchCriteria.getParamValue("hostLogFile") + "_" - + searchCriteria.getParamValue("compLogFile"); + to = DateUtil.addOffsetToDate(to, Long.parseLong(utcOffset), "yyyy-MM-dd HH:mm:ss,SSS"); + from = DateUtil.addOffsetToDate(from, Long.parseLong(utcOffset), "yyyy-MM-dd HH:mm:ss,SSS"); } String textToSave = ""; try { QueryResponse response = serviceLogsSolrDao.process(solrQuery); if (response == null) { - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR - .getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); } SolrDocumentList docList = response.getResults(); if (docList == null) { - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR - .getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } - - VSummary vsummary = BizUtil.buildSummaryForLogFile(docList); - vsummary.setFormat(format); - vsummary.setFrom(from); - vsummary.setTo(to); - - String includeString = (String) searchCriteria.getParamValue("iMessage"); - if (StringUtils.isBlank(includeString)) { - includeString = ""; - } - - String include[] = includeString.split(LogSearchConstants.I_E_SEPRATOR); - - for (String inc : include) { - includeString = includeString + ",\"" + inc + "\""; - } - includeString = includeString.replaceFirst(",", ""); - if (!StringUtils.isBlank(includeString)) { - vsummary.setIncludeString(includeString); - } - - String excludeString = null; - boolean isNormalExcluded = false; - - excludeString = (String) searchCriteria.getParamValue("eMessage"); - if (StringUtils.isBlank(excludeString)) { - excludeString = ""; - } - - String exclude[] = excludeString.split(LogSearchConstants.I_E_SEPRATOR); - for (String exc : exclude) { - excludeString = excludeString + ",\"" + exc + "\""; - } - - excludeString = excludeString.replaceFirst(",", ""); - if (!StringUtils.isBlank(excludeString)) { - vsummary.setExcludeString(excludeString); - isNormalExcluded = true; - } - - if (!StringUtils.isBlank(excludeString)) { - if (!isNormalExcluded) { - excludeString = excludeString.replaceFirst(",", ""); - } - vsummary.setExcludeString(excludeString); - } - - for (SolrDocument solrDoc : docList) { - - Date logTimeDateObj = (Date) solrDoc.get(LogSearchConstants.LOGTIME); - if(logTimeDateObj != null){ - String logTime = DateUtil.convertSolrDateToNormalDateFormat( - logTimeDateObj.getTime(), Long.parseLong(utcOffset)); - solrDoc.remove(LogSearchConstants.LOGTIME); - solrDoc.addField(LogSearchConstants.LOGTIME, logTime); - } + throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); } if (format.toLowerCase(Locale.ENGLISH).equals(".txt")) { - textToSave = BizUtil.convertObjectToNormalText(docList); + Template template = freemarkerConfiguration.getTemplate(SERVICE_LOG_TEMPLATE); + Map<String, Object> models = new HashMap<>(); + DownloadUtil.fillModelsForLogFile(docList, models, request, format, from, to); + StringWriter stringWriter = new StringWriter(); + template.process(models, stringWriter); + textToSave = stringWriter.toString(); } else if (format.toLowerCase(Locale.ENGLISH).equals(".json")) { textToSave = convertObjToString(docList); } else { @@ -1358,222 +398,68 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL "unsoported format either should be json or text", MessageEnums.ERROR_SYSTEM); } - return FileUtil.saveToFile(textToSave, fileName, vsummary); - - } catch (SolrException | SolrServerException | IOException - | ParseException e) { + File file = File.createTempFile(fileName, format); + FileOutputStream fis = new FileOutputStream(file); + fis.write(textToSave.getBytes()); + return Response + .ok(file, MediaType.APPLICATION_OCTET_STREAM) + .header("Content-Disposition", "attachment;filename=" + fileName + format) + .build(); + } catch (SolrException | TemplateException | IOException e) { logger.error("Error during solrQuery=" + solrQuery, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR - .getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } - } - - public NodeListResponse getComponentListWithLevelCounts(ServiceLogSearchCriteria searchCriteria) { - SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria); - solrQuery.setParam("event", "/service/logs/components/levels/counts"); - - if (searchCriteria.getSortBy() == null) { - searchCriteria.setSortBy(LogSearchConstants.SOLR_COMPONENT); - searchCriteria.setSortType(SolrQuery.ORDER.asc.toString()); - } - queryGenerator.setFilterFacetSort(solrQuery, searchCriteria); - String componentLevelHirachy = "type,level"; - NodeListResponse list = new NodeListResponse(); - try { - - SolrUtil.setFacetPivot(solrQuery, 1, componentLevelHirachy); - - QueryResponse response = serviceLogsSolrDao.process(solrQuery); - - List<List<PivotField>> listPivotField = new ArrayList<List<PivotField>>(); - NamedList<List<PivotField>> namedList = response.getFacetPivot(); - if (namedList != null) { - listPivotField = namedList.getAll(componentLevelHirachy); - } - List<PivotField> secondHirarchicalPivotFields = null; - if (listPivotField == null || listPivotField.isEmpty()) { - return list; - } else { - secondHirarchicalPivotFields = listPivotField.get(0); - } - List<NodeData> datatList = new ArrayList<>(); - for (PivotField singlePivotField : secondHirarchicalPivotFields) { - if (singlePivotField != null) { - NodeData comp = new NodeData(); - comp.setName("" + singlePivotField.getValue()); - List<PivotField> levelList = singlePivotField.getPivot(); - List<NameValueData> levelCountList = new ArrayList<>(); - comp.setLogLevelCount(levelCountList); - if(levelList != null){ - for (PivotField levelPivot : levelList) { - NameValueData level = new NameValueData(); - level.setName(("" + levelPivot.getValue()).toUpperCase()); - level.setValue("" + levelPivot.getCount()); - levelCountList.add(level); - } - } - datatList.add(comp); - } - } - list.setvNodeList(datatList); - return list; - } catch (SolrException | SolrServerException | IOException e) { - logger.error(e.getMessage() + "SolrQuery"+solrQuery); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR - .getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); } } - public NameValueDataListResponse getExtremeDatesForBundelId(SearchCriteria searchCriteria) { - SolrQuery solrQuery = new SolrQuery(); - NameValueDataListResponse nameValueList = new NameValueDataListResponse(); - try { - String bundelId = (String) searchCriteria - .getParamValue(LogSearchConstants.BUNDLE_ID); - if(StringUtils.isBlank(bundelId)){ - bundelId = ""; - } - - queryGenerator.setSingleIncludeFilter(solrQuery, - LogSearchConstants.BUNDLE_ID, bundelId); - - SolrUtil.setMainQuery(solrQuery, null); - solrQuery.setSort(LogSearchConstants.LOGTIME, SolrQuery.ORDER.asc); - SolrUtil.setRowCount(solrQuery, 1); - - List<NameValueData> vNameValues = new ArrayList<>(); - QueryResponse response = serviceLogsSolrDao.process(solrQuery); - - if(response == null){ - return nameValueList; - } - - SolrDocumentList solrDocList = response.getResults(); - if(solrDocList == null){ - return nameValueList; - } - for (SolrDocument solrDoc : solrDocList) { - - Date logTimeAsc = (Date) solrDoc - .getFieldValue(LogSearchConstants.LOGTIME); - if (logTimeAsc != null) { - NameValueData nameValue = new NameValueData(); - nameValue.setName("From"); - nameValue.setValue("" + logTimeAsc.getTime()); - vNameValues.add(nameValue); - } - } - - solrQuery.clear(); - SolrUtil.setMainQuery(solrQuery, null); - queryGenerator.setSingleIncludeFilter(solrQuery, - LogSearchConstants.BUNDLE_ID, bundelId); - solrQuery.setSort(LogSearchConstants.LOGTIME, SolrQuery.ORDER.desc); - SolrUtil.setRowCount(solrQuery, 1); - - solrDocList.clear(); - response = serviceLogsSolrDao.process(solrQuery); - - solrDocList = response.getResults(); - for (SolrDocument solrDoc : solrDocList) { - if (solrDoc != null) { - Date logTimeDesc = (Date) solrDoc - .getFieldValue(LogSearchConstants.LOGTIME); - - if (logTimeDesc != null) { - NameValueData nameValue = new NameValueData(); - nameValue.setName("To"); - nameValue.setValue("" + logTimeDesc.getTime()); - vNameValues.add(nameValue); - } - } - } - nameValueList.setvNameValues(vNameValues); - - - } catch (SolrServerException | SolrException | IOException e) { - logger.error(e.getMessage() + "SolrQuery"+solrQuery); - nameValueList = new NameValueDataListResponse(); - } - return nameValueList; + public NodeListResponse getComponentListWithLevelCounts(ServiceLogComponentLevelRequest request) { + SimpleFacetQuery facetQuery = conversionService.convert(request, SimpleFacetQuery.class); + SolrQuery solrQuery = new DefaultQueryParser().doConstructSolrQuery(facetQuery); + solrQuery.setFacetSort(StringUtils.isEmpty(request.getSortBy()) ? COMPONENT: request.getSortBy()); + QueryResponse response = serviceLogsSolrDao.process(facetQuery, "/service/logs/components/levels/counts"); + return graphDataGenerator.generateOneLevelServiceNodeTree(response, String.format("%s,%s", COMPONENT, LEVEL)); } public String getServiceLogsSchemaFieldsName() { - return convertObjToString(serviceLogsSolrDao.schemaFieldNameMap); + return convertObjToString(serviceLogsSolrDao.getSolrSchemaFieldDao().getSchemaFieldNameMap()); } - public BarGraphDataListResponse getAnyGraphData(ServiceAnyGraphSearchCriteria searchCriteria) { - searchCriteria.addParam("fieldTime", LogSearchConstants.LOGTIME); - SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria); - BarGraphDataListResponse result = graphDataGenerator.getAnyGraphData(searchCriteria, serviceLogsSolrDao, solrQuery); - if (result == null) { - result = new BarGraphDataListResponse(); - } - return result; - + public BarGraphDataListResponse getAnyGraphCountData(ServiceAnyGraphRequest request) { + SimpleFacetQuery solrDataQuery = conversionService.convert(request, SimpleFacetQuery.class); + QueryResponse queryResponse = serviceLogsSolrDao.process(solrDataQuery); + return graphDataGenerator.getGraphDataWithDefaults(queryResponse, LEVEL, LogSearchConstants.SUPPORTED_LOG_LEVELS); } - public ServiceLogResponse getAfterBeforeLogs(ServiceLogTruncatedSearchCriteria searchCriteria) { + public ServiceLogResponse getAfterBeforeLogs(ServiceLogTruncatedRequest request) { ServiceLogResponse logResponse = new ServiceLogResponse(); List<SolrServiceLogData> docList = null; - String id = (String) searchCriteria - .getParamValue(LogSearchConstants.ID); - if (StringUtils.isBlank(id)) { - return logResponse; - - } - String maxRows = ""; - - maxRows = searchCriteria.getNumberRows(); - if (StringUtils.isBlank(maxRows)){ - maxRows = ""+maxRows; - } - String scrollType = searchCriteria.getScrollType(); - if(StringUtils.isBlank(scrollType)){ - scrollType = ""; - } + String scrollType = request.getScrollType() != null ? request.getScrollType() : ""; String logTime = null; String sequenceId = null; - try { - SolrQuery solrQuery = new SolrQuery(); - SolrUtil.setMainQuery(solrQuery, - queryGenerator.buildFilterQuery(LogSearchConstants.ID, id)); - SolrUtil.setRowCount(solrQuery, 1); - QueryResponse response = serviceLogsSolrDao.process(solrQuery); - if(response == null){ - return logResponse; - } - docList = convertToSolrBeans(response); - if (docList != null && !docList.isEmpty()) { - Date date = docList.get(0).getLogTime(); - logTime = DateUtil.convertDateWithMillisecondsToSolrDate(date); - sequenceId = "" - + docList.get(0).getSeqNum(); - } - if (StringUtils.isBlank(logTime)) { - return logResponse; - } - } catch (SolrServerException | SolrException | IOException e) { - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR - .getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + SolrQuery solrQuery = new SolrQuery(); + solrQuery.setQuery("*:*"); + solrQuery.setRows(1); + solrQuery.addFilterQuery(String.format("%s:%s", ID, request.getId())); + QueryResponse response = serviceLogsSolrDao.process(solrQuery); + if (response == null) { + return logResponse; } - if (LogSearchConstants.SCROLL_TYPE_BEFORE.equals(scrollType)) { - logResponse = whenScrollUp(searchCriteria, logTime, - sequenceId, maxRows); - + docList = convertToSolrBeans(response); + if (docList != null && !docList.isEmpty()) { + Date date = docList.get(0).getLogTime(); + logTime = DateUtil.convertDateWithMillisecondsToSolrDate(date); + sequenceId = docList.get(0).getSeqNum().toString(); + } + if (StringUtils.isBlank(logTime)) { + return logResponse; + } + if (LogSearchConstants.SCROLL_TYPE_BEFORE.equals(scrollType) || LogSearchConstants.SCROLL_TYPE_AFTER.equals(scrollType)) { List<ServiceLogData> solrDocList = new ArrayList<>(); - for (ServiceLogData solrDoc : logResponse.getLogList()) { - solrDocList.add(solrDoc); - } - logResponse.setLogList(solrDocList); + ServiceLogResponse beforeAfterResponse = whenScroll(request, logTime, sequenceId, scrollType); + if (beforeAfterResponse.getLogList() == null) { return logResponse; - - } else if (LogSearchConstants.SCROLL_TYPE_AFTER.equals(scrollType)) { - List<ServiceLogData> solrDocList = new ArrayList<>(); - logResponse = new ServiceLogResponse(); - for (ServiceLogData solrDoc : whenScrollDown(searchCriteria, logTime, - sequenceId, maxRows).getLogList()) { + } + for (ServiceLogData solrDoc : beforeAfterResponse.getLogList()) { solrDocList.add(solrDoc); } logResponse.setLogList(solrDocList); @@ -1582,122 +468,30 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL } else { logResponse = new ServiceLogResponse(); List<ServiceLogData> initial = new ArrayList<>(); - List<ServiceLogData> before = whenScrollUp(searchCriteria, logTime, - sequenceId, maxRows).getLogList(); - List<ServiceLogData> after = whenScrollDown(searchCriteria, logTime, - sequenceId, maxRows).getLogList(); + List<ServiceLogData> before = whenScroll(request, logTime, sequenceId, LogSearchConstants.SCROLL_TYPE_BEFORE).getLogList(); + List<ServiceLogData> after = whenScroll(request, logTime, sequenceId, LogSearchConstants.SCROLL_TYPE_AFTER).getLogList(); if (before != null && !before.isEmpty()) { for (ServiceLogData solrDoc : Lists.reverse(before)) { initial.add(solrDoc); } } - initial.add(docList.get(0)); if (after != null && !after.isEmpty()) { for (ServiceLogData solrDoc : after) { initial.add(solrDoc); } } - logResponse.setLogList(initial); - return logResponse; - - } - } - - private ServiceLogResponse whenScrollUp(SearchCriteria searchCriteria, - String logTime, String sequenceId, String maxRows) { - SolrQuery solrQuery = new SolrQuery(); - SolrUtil.setMainQuery(solrQuery, null); - try { - int seq_num = Integer.parseInt(sequenceId) - 1; - sequenceId = "" + seq_num; - } catch (Exception e) { - } - queryGenerator.setSingleRangeFilter( - solrQuery, - LogSearchConstants.SEQUNCE_ID, "*", sequenceId); - - queryGenerator.applyLogFileFilter(solrQuery, searchCriteria); - - queryGenerator.setSingleRangeFilter(solrQuery, - LogSearchConstants.LOGTIME, "*", logTime); - SolrUtil.setRowCount(solrQuery, Integer.parseInt(maxRows)); - String order1 = LogSearchConstants.LOGTIME + " " - + LogSearchConstants.DESCENDING_ORDER; - String order2 = LogSearchConstants.SEQUNCE_ID + " " - + LogSearchConstants.DESCENDING_ORDER; - List<String> sortOrder = new ArrayList<String>(); - sortOrder.add(order1); - sortOrder.add(order2); - searchCriteria.addParam(LogSearchConstants.SORT, sortOrder); - queryGenerator.setMultipleSortOrder(solrQuery, searchCriteria); - - return getLogAsPaginationProvided(solrQuery, serviceLogsSolrDao); } - private ServiceLogResponse whenScrollDown(SearchCriteria searchCriteria, - String logTime, String sequenceId, String maxRows) { - SolrQuery solrQuery = new SolrQuery(); - SolrUtil.setMainQuery(solrQuery, null); - queryGenerator.applyLogFileFilter(solrQuery, searchCriteria); - - try { - int seq_num = Integer.parseInt(sequenceId) + 1; - sequenceId = "" + seq_num; - } catch (Exception e) { - - } - queryGenerator.setSingleRangeFilter( - solrQuery, - LogSearchConstants.SEQUNCE_ID, sequenceId, "*"); - queryGenerator.setSingleRangeFilter(solrQuery, - LogSearchConstants.LOGTIME, logTime, "*"); - SolrUtil.setRowCount(solrQuery, Integer.parseInt(maxRows)); - - String order1 = LogSearchConstants.LOGTIME + " " - + LogSearchConstants.ASCENDING_ORDER; - String order2 = LogSearchConstants.SEQUNCE_ID + " " - + LogSearchConstants.ASCENDING_ORDER; - List<String> sortOrder = new ArrayList<String>(); - sortOrder.add(order1); - sortOrder.add(order2); - searchCriteria.addParam(LogSearchConstants.SORT, sortOrder); - queryGenerator.setMultipleSortOrder(solrQuery, searchCriteria); - - return getLogAsPaginationProvided(solrQuery, serviceLogsSolrDao); - } - - @Scheduled(cron = "${logsearch.solr.warming.cron}") - public void warmingSolrServer(){ - logger.info("solr warming triggered."); - SolrQuery solrQuery = new SolrQuery(); - TimeZone gmtTimeZone = TimeZone.getTimeZone("GMT"); - GregorianCalendar utc = new GregorianCalendar(gmtTimeZone); - utc.setTimeInMillis(new Date().getTime()); - utc.set(Calendar.HOUR, 0); - utc.set(Calendar.MINUTE, 0); - utc.set(Calendar.MILLISECOND, 001); - utc.set(Calendar.SECOND, 0); - DateUtil.convertDateWithMillisecondsToSolrDate(utc.getTime()); - String from = DateUtil.convertDateWithMillisecondsToSolrDate(utc.getTime()); - utc.set(Calendar.MILLISECOND, 999); - utc.set(Calendar.SECOND, 59); - utc.set(Calendar.MINUTE, 59); - utc.set(Calendar.HOUR, 23); - String to = DateUtil.convertDateWithMillisecondsToSolrDate(utc.getTime()); - queryGenerator.setSingleRangeFilter(solrQuery, - LogSearchConstants.LOGTIME, from,to); - String level = LogSearchConstants.FATAL+","+LogSearchConstants.ERROR+","+LogSearchConstants.WARN; - queryGenerator.setFilterClauseWithFieldName(solrQuery, level, - LogSearchConstants.SOLR_LEVEL, "", QueryGenerationBase.Condition.OR); - try { - serviceLogsSolrDao.process(solrQuery); - } catch (SolrServerException | IOException e) { - logger.error("Error while warming solr server",e); - } + private ServiceLogResponse whenScroll(ServiceLogTruncatedRequest request, String logTime, String sequenceId, String afterOrBefore) { + request.setScrollType(afterOrBefore); + ServiceLogTruncatedRequestQueryConverter converter = new ServiceLogTruncatedRequestQueryConverter(); + converter.setLogTime(logTime); + converter.setSequenceId(sequenceId); + return getLogAsPaginationProvided(converter.convert(request), serviceLogsSolrDao, "service/logs/truncated"); } @Override @@ -1728,6 +522,40 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL } } + private <T extends LogData> GroupListResponse getFields(String field, Class<T> clazz) { + SolrQuery solrQuery = new SolrQuery(); + solrQuery.setQuery("*:*"); + GroupListResponse collection = new GroupListResponse(); + SolrUtil.setFacetField(solrQuery, + field); + SolrUtil.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); + QueryResponse response = serviceLogsSolrDao.process(solrQuery); + if (response == null) { + return collection; + } + FacetField facetField = response + .getFacetField(field); + if (facetField == null) { + return collection; + } + List<Count> fieldList = facetField.getValues(); + if (fieldList == null) { + return collection; + } + SolrDocumentList docList = response.getResults(); + if (docList == null) { + return collection; + } + List<LogData> groupList = getLogDataListByFieldType(clazz, response, fieldList); + + collection.setGroupList(groupList); + if (!docList.isEmpty()) { + collection.setStartIndex((int) docList.getStart()); + collection.setTotalCount(docList.getNumFound()); + } + return collection; + } + private <T extends LogData> LogData createNewFieldByType(Class<T> clazz, Count count, String temp) { temp = count.getName(); LogData result = null; @@ -1745,5 +573,4 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL } throw new UnsupportedOperationException(); } - }
http://git-wip-us.apache.org/repos/asf/ambari/blob/1a225bd2/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/SessionManager.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/SessionManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/SessionManager.java index 206636a..e8b699e 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/SessionManager.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/SessionManager.java @@ -24,9 +24,10 @@ import org.apache.log4j.Logger; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.security.web.authentication.WebAuthenticationDetails; -import org.springframework.stereotype.Component; -@Component +import javax.inject.Named; + +@Named public class SessionManager { private static final Logger logger = Logger.getLogger(SessionManager.class);