http://git-wip-us.apache.org/repos/asf/ambari/blob/b0f1e340/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverter.java new file mode 100644 index 0000000..a1f6e75 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverter.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.converter; + +import org.apache.ambari.logsearch.common.LogSearchConstants; +import org.apache.ambari.logsearch.common.LogType; +import org.apache.ambari.logsearch.model.request.impl.BaseServiceLogRequest; +import org.apache.ambari.logsearch.util.SolrUtil; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.data.domain.Sort; +import org.springframework.data.solr.core.query.SimpleQuery; +import javax.inject.Named; +import java.util.List; + +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LOGTIME; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LEVEL; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.KEY_LOG_MESSAGE; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.HOST; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.PATH; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.COMPONENT; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.BUNDLE_ID; +import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.SEQUENCE_ID; + +@Named +public class BaseServiceLogRequestQueryConverter extends AbstractServiceLogRequestQueryConverter<BaseServiceLogRequest, SimpleQuery> { + + @Override + public SimpleQuery extendLogQuery(BaseServiceLogRequest request, SimpleQuery query) { + List<String> levels = splitValueAsList(request.getLevel(), ","); + addContainsFilterQuery(query, KEY_LOG_MESSAGE, SolrUtil.escapeForStandardTokenizer(request.getiMessage())); + addContainsFilterQuery(query, KEY_LOG_MESSAGE, SolrUtil.escapeForStandardTokenizer(request.geteMessage()), true); + addEqualsFilterQuery(query, HOST, SolrUtil.escapeQueryChars(request.getHostName())); + addEqualsFilterQuery(query, PATH, SolrUtil.escapeQueryChars(request.getFileName())); + addEqualsFilterQuery(query, COMPONENT, SolrUtil.escapeQueryChars(request.getComponentName())); + addEqualsFilterQuery(query, BUNDLE_ID, request.getBundleId()); + if (CollectionUtils.isNotEmpty(levels)){ + addInFilterQuery(query, LEVEL, levels); + } + addInFiltersIfNotNullAndEnabled(query, request.getHostList(), HOST, org.apache.commons.lang.StringUtils.isEmpty(request.getHostName())); + addRangeFilter(query, LOGTIME, request.getFrom(), request.getTo()); + return query; + } + + @Override + public Sort sort(BaseServiceLogRequest request) { + String sortBy = request.getSortBy(); + String sortType = request.getSortType(); + Sort.Order defaultSortOrder; + if (StringUtils.isNotBlank(sortBy)) { + Sort.Direction direction = StringUtils.equals(sortType, LogSearchConstants.ASCENDING_ORDER) ? Sort.Direction.ASC : Sort.Direction.DESC; + defaultSortOrder = new Sort.Order(direction, sortBy); + } else { + defaultSortOrder = new Sort.Order(Sort.Direction.DESC, LOGTIME); + } + Sort.Order sequenceIdOrder = new Sort.Order(Sort.Direction.DESC, SEQUENCE_ID); + return new Sort(defaultSortOrder, sequenceIdOrder); + } + + @Override + public SimpleQuery createQuery() { + return new SimpleQuery(); + } + + @Override + public LogType getLogType() { + return LogType.SERVICE; + } +}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b0f1e340/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverter.java new file mode 100644 index 0000000..a4e613b --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverter.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.converter; + +import org.apache.ambari.logsearch.common.LogType; +import org.apache.ambari.logsearch.model.request.impl.FieldAuditLogRequest; +import org.springframework.data.solr.core.query.FacetOptions; + +import javax.inject.Named; + +import static org.apache.ambari.logsearch.solr.SolrConstants.AuditLogConstants.AUDIT_EVTTIME; +import static org.apache.ambari.logsearch.solr.SolrConstants.AuditLogConstants.AUDIT_COMPONENT; + +@Named +public class FieldAuditLogRequestQueryConverter extends AbstractLogRequestFacetQueryConverter<FieldAuditLogRequest> { + + @Override + public void appendFacetOptions(FacetOptions facetOptions, FieldAuditLogRequest request) { + facetOptions.addFacetOnPivot(request.getField(), AUDIT_COMPONENT); + facetOptions.setFacetLimit(request.getTop()); + } + + @Override + public FacetOptions.FacetSort getFacetSort() { + return FacetOptions.FacetSort.COUNT; + } + + @Override + public String getDateTimeField() { + return AUDIT_EVTTIME; + } + + @Override + public LogType getLogType() { + return LogType.AUDIT; + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0f1e340/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/HostLogFilesRequestQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/HostLogFilesRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/HostLogFilesRequestQueryConverter.java new file mode 100644 index 0000000..e1e7418 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/HostLogFilesRequestQueryConverter.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.converter; + +import org.apache.ambari.logsearch.common.LogType; +import org.apache.ambari.logsearch.model.request.impl.HostLogFilesRequest; +import org.apache.commons.lang3.StringUtils; +import org.springframework.data.solr.core.query.FacetOptions; +import org.springframework.data.solr.core.query.FacetOptions.FacetSort; +import org.springframework.data.solr.core.query.SimpleFacetQuery; +import org.springframework.data.solr.core.query.SimpleStringCriteria; + +import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.CLUSTER; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.HOST; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.COMPONENT; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.PATH; + +import javax.inject.Named; + +@Named +public class HostLogFilesRequestQueryConverter extends AbstractOperationHolderConverter<HostLogFilesRequest, SimpleFacetQuery>{ + + @Override + public SimpleFacetQuery convert(HostLogFilesRequest request) { + SimpleFacetQuery facetQuery = new SimpleFacetQuery(); + facetQuery.addCriteria(new SimpleStringCriteria(String.format("%s:(%s)", HOST, request.getHostName()))); + if (StringUtils.isNotEmpty(request.getComponentName())) { + facetQuery.addCriteria(new SimpleStringCriteria(String.format("%s:(%s)", COMPONENT, request.getComponentName()))); + } + FacetOptions facetOptions = new FacetOptions(); + facetOptions.setFacetMinCount(1); + facetOptions.setFacetLimit(-1); + facetOptions.setFacetSort(FacetSort.COUNT); + facetOptions.addFacetOnPivot(COMPONENT, PATH); + facetQuery.setFacetOptions(facetOptions); + addInFilterQuery(facetQuery, CLUSTER, splitValueAsList(request.getClusters(), ",")); + facetQuery.setRows(0); + return facetQuery; + } + + @Override + public LogType getLogType() { + return LogType.SERVICE; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0f1e340/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestQueryConverter.java new file mode 100644 index 0000000..b194df3 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestQueryConverter.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.converter; + +import org.apache.ambari.logsearch.common.LogType; +import org.apache.ambari.logsearch.model.request.impl.ServiceAnyGraphRequest; +import org.apache.commons.lang.StringUtils; +import org.springframework.data.solr.core.query.FacetOptions; +import org.springframework.data.solr.core.query.SimpleFacetQuery; + +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.HOST; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LOGTIME; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LEVEL; + +import javax.inject.Named; + +@Named +public class ServiceLogAnyGraphRequestQueryConverter extends AbstractLogRequestFacetQueryConverter<ServiceAnyGraphRequest>{ + + @Override + public void appendFacetOptions(FacetOptions facetOptions, ServiceAnyGraphRequest request) { + facetOptions.addFacetOnField(LEVEL); + } + + @Override + public FacetOptions.FacetSort getFacetSort() { + return FacetOptions.FacetSort.COUNT; + } + + @Override + public String getDateTimeField() { + return LOGTIME; + } + + @Override + public LogType getLogType() { + return LogType.SERVICE; + } + + @Override + public void appendFacetQuery(SimpleFacetQuery facetQuery, ServiceAnyGraphRequest request) { + addInFiltersIfNotNullAndEnabled(facetQuery, request.getHostList(), HOST, StringUtils.isEmpty(request.getHostName())); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0f1e340/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverter.java new file mode 100644 index 0000000..3b4bb78 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverter.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.converter; + +import org.apache.ambari.logsearch.common.LogType; +import org.apache.ambari.logsearch.model.request.impl.ServiceLogComponentLevelRequest; +import org.springframework.data.solr.core.query.FacetOptions; + +import javax.inject.Named; + +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.COMPONENT; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LEVEL; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LOGTIME; + +@Named +public class ServiceLogComponentLevelRequestQueryConverter extends AbstractServiceLogRequestFacetQueryConverter<ServiceLogComponentLevelRequest> { + + @Override + public FacetOptions.FacetSort getFacetSort() { + return FacetOptions.FacetSort.INDEX; + } + + @Override + public String getDateTimeField() { + return LOGTIME; + } + + @Override + public void appendFacetOptions(FacetOptions facetOptions, ServiceLogComponentLevelRequest request) { + facetOptions.addFacetOnPivot(COMPONENT, LEVEL); + } + + @Override + public LogType getLogType() { + return LogType.SERVICE; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0f1e340/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverter.java new file mode 100644 index 0000000..6271ca8 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverter.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.converter; + +import org.apache.ambari.logsearch.common.LogType; +import org.apache.ambari.logsearch.model.request.impl.ServiceLogComponentHostRequest; +import org.springframework.data.solr.core.query.FacetOptions; + +import javax.inject.Named; + +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.COMPONENT; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.HOST; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LEVEL; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LOGTIME; + +@Named +public class ServiceLogComponentRequestFacetQueryConverter extends AbstractServiceLogRequestFacetQueryConverter<ServiceLogComponentHostRequest> { + + @Override + public FacetOptions.FacetSort getFacetSort() { + return FacetOptions.FacetSort.INDEX; + } + + @Override + public String getDateTimeField() { + return LOGTIME; + } + + @Override + public LogType getLogType() { + return LogType.SERVICE; + } + + @Override + public void appendFacetOptions(FacetOptions facetOptions, ServiceLogComponentHostRequest request) { + facetOptions.addFacetOnPivot(COMPONENT, HOST, LEVEL); + facetOptions.addFacetOnPivot(COMPONENT, LEVEL); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0f1e340/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverter.java new file mode 100644 index 0000000..982d2a1 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverter.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.converter; + +import org.apache.ambari.logsearch.common.LogType; +import org.apache.ambari.logsearch.model.request.impl.ServiceLogLevelCountRequest; +import org.apache.ambari.logsearch.util.SolrUtil; +import org.apache.commons.lang.StringUtils; +import org.springframework.data.solr.core.query.FacetOptions; +import org.springframework.data.solr.core.query.SimpleFacetQuery; + +import javax.inject.Named; + +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.BUNDLE_ID; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.COMPONENT; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.HOST; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LOGTIME; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LEVEL; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.PATH; + +@Named +public class ServiceLogLevelCountRequestQueryConverter extends AbstractLogRequestFacetQueryConverter<ServiceLogLevelCountRequest> { + + @Override + public FacetOptions.FacetSort getFacetSort() { + return FacetOptions.FacetSort.COUNT; + } + + @Override + public String getDateTimeField() { + return LOGTIME; + } + + @Override + public void appendFacetOptions(FacetOptions facetOptions, ServiceLogLevelCountRequest request) { + facetOptions.addFacetOnField(LEVEL); + } + + @Override + public LogType getLogType() { + return LogType.SERVICE; + } + + @Override + public void appendFacetQuery(SimpleFacetQuery facetQuery, ServiceLogLevelCountRequest request) { + addEqualsFilterQuery(facetQuery, HOST, SolrUtil.escapeQueryChars(request.getHostName())); + addEqualsFilterQuery(facetQuery, PATH, SolrUtil.escapeQueryChars(request.getFileName())); + addEqualsFilterQuery(facetQuery, COMPONENT, SolrUtil.escapeQueryChars(request.getComponentName())); + addEqualsFilterQuery(facetQuery, BUNDLE_ID, request.getBundleId()); + addInFiltersIfNotNullAndEnabled(facetQuery, request.getHostList(), HOST, StringUtils.isEmpty(request.getHostName())); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0f1e340/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverter.java new file mode 100644 index 0000000..46f8f85 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverter.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.converter; + +import org.apache.ambari.logsearch.common.LogType; +import org.apache.ambari.logsearch.model.request.impl.ServiceGraphRequest; +import org.apache.commons.lang.StringUtils; +import org.apache.solr.client.solrj.SolrQuery; + +import javax.inject.Named; + +import java.util.Arrays; +import java.util.List; + +import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.CLUSTER; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.HOST; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LEVEL; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LOGTIME; + +@Named +public class ServiceLogLevelDateRangeRequestQueryConverter extends AbstractDateRangeFacetQueryConverter<ServiceGraphRequest>{ + + @Override + public String getDateFieldName() { + return LOGTIME; + } + + @Override + public String getTypeFieldName() { + return LEVEL; + } + + @Override + public SolrQuery convert(ServiceGraphRequest request) { + SolrQuery solrQuery = super.convert(request); + addListFilterToSolrQuery(solrQuery, LEVEL, request.getLevel()); + if (request.getHostList() != null && StringUtils.isEmpty(request.getHostName())) { + List<String> hosts = request.getHostList().length() == 0 ? Arrays.asList("\\-1") : splitValueAsList(request.getHostList(), ","); + if (hosts.size() > 1) { + solrQuery.addFilterQuery(String.format("%s:(%s)", HOST, StringUtils.join(hosts, " OR "))); + } else { + solrQuery.addFilterQuery(String.format("%s:%s", HOST, hosts.get(0))); + } + } + addListFilterToSolrQuery(solrQuery, CLUSTER, request.getClusters()); + return solrQuery; + } + + @Override + public LogType getLogType() { + return LogType.SERVICE; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0f1e340/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverter.java new file mode 100644 index 0000000..7cb8f91 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverter.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.converter; + +import org.apache.ambari.logsearch.common.LogType; +import org.apache.ambari.logsearch.model.request.impl.ServiceLogHostComponentRequest; +import org.springframework.data.solr.core.query.FacetOptions; +import javax.inject.Named; + +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.COMPONENT; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.HOST; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LEVEL; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LOGTIME; + +@Named +public class ServiceLogTreeRequestFacetQueryConverter extends AbstractServiceLogRequestFacetQueryConverter<ServiceLogHostComponentRequest>{ + + @Override + public FacetOptions.FacetSort getFacetSort() { + return FacetOptions.FacetSort.INDEX; + } + + @Override + public String getDateTimeField() { + return LOGTIME; + } + + @Override + public LogType getLogType() { + return LogType.SERVICE; + } + + @Override + public void appendFacetOptions(FacetOptions facetOptions, ServiceLogHostComponentRequest request) { + facetOptions.addFacetOnPivot(HOST, COMPONENT, LEVEL); + facetOptions.addFacetOnPivot(HOST, LEVEL); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0f1e340/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverter.java new file mode 100644 index 0000000..d0273ac --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverter.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.converter; + +import org.apache.ambari.logsearch.common.LogSearchConstants; +import org.apache.ambari.logsearch.common.LogType; +import org.apache.ambari.logsearch.model.request.impl.ServiceLogTruncatedRequest; +import org.springframework.data.domain.Sort; +import org.springframework.data.solr.core.query.SimpleQuery; + +import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.SEQUENCE_ID; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.COMPONENT; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.HOST; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LOGTIME; + +public class ServiceLogTruncatedRequestQueryConverter extends AbstractServiceLogRequestQueryConverter<ServiceLogTruncatedRequest, SimpleQuery>{ + + private String sequenceId; + + private String logTime; + + @Override + public SimpleQuery extendLogQuery(ServiceLogTruncatedRequest request, SimpleQuery query) { + addEqualsFilterQuery(query, COMPONENT, request.getComponentName()); + addEqualsFilterQuery(query, HOST, request.getHostName()); + String scrollType = request.getScrollType(); + if (LogSearchConstants.SCROLL_TYPE_BEFORE.equals(scrollType)) { + Integer secuenceIdNum = Integer.parseInt(getSequenceId()) - 1; + addRangeFilter(query, LOGTIME, null, getLogTime()); + addRangeFilter(query, SEQUENCE_ID, null, secuenceIdNum.toString()); + } else if (LogSearchConstants.SCROLL_TYPE_AFTER.equals(scrollType)) { + Integer secuenceIdNum = Integer.parseInt(getSequenceId()) + 1; + addRangeFilter(query, LOGTIME, getLogTime(), null); + addRangeFilter(query, SEQUENCE_ID, secuenceIdNum.toString(), null); + } + query.setRows(request.getNumberRows()); + return query; + } + + @Override + public Sort sort(ServiceLogTruncatedRequest request) { + String scrollType = request.getScrollType(); + Sort.Direction direction; + if (LogSearchConstants.SCROLL_TYPE_AFTER.equals(scrollType)) { + direction = Sort.Direction.ASC; + } else { + direction = Sort.Direction.DESC; + } + Sort.Order logtimeSortOrder = new Sort.Order(direction, LOGTIME); + Sort.Order secuqnceIdSortOrder = new Sort.Order(direction, SEQUENCE_ID); + return new Sort(logtimeSortOrder, secuqnceIdSortOrder); + } + + @Override + public SimpleQuery createQuery() { + return new SimpleQuery(); + } + + @Override + public LogType getLogType() { + return LogType.SERVICE; + } + + public String getSequenceId() { + return sequenceId; + } + + public void setSequenceId(String sequenceId) { + this.sequenceId = sequenceId; + } + + public String getLogTime() { + return logTime; + } + + public void setLogTime(String logTime) { + this.logTime = logTime; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0f1e340/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/StringFieldFacetQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/StringFieldFacetQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/StringFieldFacetQueryConverter.java new file mode 100644 index 0000000..faff0be --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/StringFieldFacetQueryConverter.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.converter; + +import org.springframework.data.solr.core.query.Criteria; +import org.springframework.data.solr.core.query.FacetOptions; +import org.springframework.data.solr.core.query.SimpleFacetQuery; +import org.springframework.data.solr.core.query.SimpleStringCriteria; + +import javax.inject.Named; + +@Named +public class StringFieldFacetQueryConverter extends AbstractConverterAware<String, SimpleFacetQuery> { + + @Override + public SimpleFacetQuery convert(String fieldName) { + Criteria criteria = new SimpleStringCriteria("*:*"); + SimpleFacetQuery facetQuery = new SimpleFacetQuery(); + facetQuery.addCriteria(criteria); + facetQuery.setRows(0); + FacetOptions facetOptions = new FacetOptions(); + facetOptions.setFacetMinCount(1); + facetOptions.addFacetOnField(fieldName); + facetOptions.setFacetLimit(-1); + facetQuery.setFacetOptions(facetOptions); + return facetQuery; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0f1e340/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/UserConfigRequestQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/UserConfigRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/UserConfigRequestQueryConverter.java new file mode 100644 index 0000000..594f23b --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/UserConfigRequestQueryConverter.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.converter; + +import org.apache.ambari.logsearch.model.request.impl.UserConfigRequest; +import org.apache.ambari.logsearch.util.SolrUtil; +import org.apache.commons.lang.StringUtils; +import org.apache.solr.client.solrj.SolrQuery; + +import javax.inject.Named; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.CLUSTER; +import static org.apache.ambari.logsearch.solr.SolrConstants.UserConfigConstants.FILTER_NAME; +import static org.apache.ambari.logsearch.solr.SolrConstants.UserConfigConstants.ROW_TYPE; + +@Named +public class UserConfigRequestQueryConverter extends AbstractConverterAware<UserConfigRequest, SolrQuery> { + + @Override + public SolrQuery convert(UserConfigRequest userConfigRequest) { + SolrQuery userConfigQuery = new SolrQuery(); + userConfigQuery.setQuery("*:*"); + + int startIndex = StringUtils.isNotEmpty(userConfigRequest.getStartIndex()) && StringUtils.isNumeric(userConfigRequest.getStartIndex()) + ? Integer.parseInt(userConfigRequest.getStartIndex()) : 0; + int maxRows = StringUtils.isNotEmpty(userConfigRequest.getPageSize()) && StringUtils.isNumeric(userConfigRequest.getPageSize()) + ? Integer.parseInt(userConfigRequest.getPageSize()) : 10; + + SolrQuery.ORDER order = userConfigRequest.getSortType() != null && SolrQuery.ORDER.desc.equals(SolrQuery.ORDER.valueOf(userConfigRequest.getSortType())) + ? SolrQuery.ORDER.desc : SolrQuery.ORDER.asc; + String sortBy = StringUtils.isNotEmpty(userConfigRequest.getSortBy()) ? userConfigRequest.getSortBy() : FILTER_NAME; + String filterName = StringUtils.isBlank(userConfigRequest.getFilterName()) ? "*" : "*" + userConfigRequest.getFilterName() + "*"; + + userConfigQuery.addFilterQuery(String.format("%s:%s", ROW_TYPE, userConfigRequest.getRowType())); + userConfigQuery.addFilterQuery(String.format("%s:%s", FILTER_NAME, SolrUtil.makeSearcableString(filterName))); + userConfigQuery.setStart(startIndex); + userConfigQuery.setRows(maxRows); + + SolrQuery.SortClause sortOrder = SolrQuery.SortClause.create(sortBy, order); + List<SolrQuery.SortClause> sort = new ArrayList<>(); + sort.add(sortOrder); + userConfigQuery.setSorts(sort); + + SolrUtil.addListFilterToSolrQuery(userConfigQuery, CLUSTER, userConfigRequest.getClusters()); + + return userConfigQuery; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0f1e340/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverter.java new file mode 100644 index 0000000..6b7c00e --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverter.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.converter; + +import org.apache.ambari.logsearch.common.LogType; +import org.apache.ambari.logsearch.model.request.impl.UserExportRequest; +import org.springframework.data.solr.core.query.FacetOptions; + +import javax.inject.Named; + +import static org.apache.ambari.logsearch.solr.SolrConstants.AuditLogConstants.AUDIT_COMPONENT; +import static org.apache.ambari.logsearch.solr.SolrConstants.AuditLogConstants.AUDIT_EVTTIME; +import static org.apache.ambari.logsearch.solr.SolrConstants.AuditLogConstants.AUDIT_REQUEST_USER; +import static org.apache.ambari.logsearch.solr.SolrConstants.AuditLogConstants.AUDIT_RESOURCE; + +@Named +public class UserExportRequestQueryConverter extends AbstractLogRequestFacetQueryConverter<UserExportRequest> { + + @Override + public void appendFacetOptions(FacetOptions facetOptions, UserExportRequest request) { + facetOptions.addFacetOnPivot(AUDIT_REQUEST_USER, AUDIT_COMPONENT); + facetOptions.addFacetOnPivot(AUDIT_RESOURCE, AUDIT_COMPONENT); + } + + @Override + public FacetOptions.FacetSort getFacetSort() { + return FacetOptions.FacetSort.COUNT; + } + + @Override + public String getDateTimeField() { + return AUDIT_EVTTIME; + } + + @Override + public LogType getLogType() { + return LogType.AUDIT; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0f1e340/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java new file mode 100644 index 0000000..d058383 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.dao; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import javax.inject.Named; + +import org.apache.ambari.logsearch.common.LogType; +import org.apache.ambari.logsearch.conf.SolrAuditLogPropsConfig; +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.ambari.logsearch.conf.global.SolrCollectionState; +import org.apache.ambari.logsearch.configurer.SolrAuditAliasConfigurer; +import org.apache.ambari.logsearch.configurer.SolrCollectionConfigurer; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import org.springframework.data.solr.core.SolrTemplate; + +@Named +public class AuditSolrDao extends SolrDaoBase { + + private static final Logger LOG = Logger.getLogger(AuditSolrDao.class); + + @Inject + private SolrAuditLogPropsConfig solrAuditLogPropsConfig; + + @Inject + @Named("auditSolrTemplate") + private SolrTemplate auditSolrTemplate; + + @Inject + @Named("solrAuditLogsState") + private SolrCollectionState solrAuditLogsState; + + public AuditSolrDao() { + super(LogType.AUDIT); + } + + @Override + public SolrTemplate getSolrTemplate() { + return auditSolrTemplate; + } + + @Override + public void setSolrTemplate(SolrTemplate solrTemplate) { + this.auditSolrTemplate = solrTemplate; + } + + @PostConstruct + public void postConstructor() { + String aliasNameIn = solrAuditLogPropsConfig.getAliasNameIn(); + String rangerAuditCollection = solrAuditLogPropsConfig.getRangerCollection(); + + try { + new SolrCollectionConfigurer(this).start(); + boolean createAlias = (aliasNameIn != null && StringUtils.isNotBlank(rangerAuditCollection)); + if (createAlias) { + new SolrAuditAliasConfigurer(this).start(); + } + } catch (Exception e) { + LOG.error("Error while connecting to Solr for audit logs : solrUrl=" + solrAuditLogPropsConfig.getSolrUrl() + ", zkConnectString=" + + solrAuditLogPropsConfig.getZkConnectString() + ", collection=" + solrAuditLogPropsConfig.getCollection(), e); + } + } + + @Override + public SolrCollectionState getSolrCollectionState() { + return this.solrAuditLogsState; + } + + @Override + public SolrPropsConfig getSolrPropsConfig() { + return this.solrAuditLogPropsConfig; + } + + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0f1e340/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java new file mode 100644 index 0000000..37375dc --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.dao; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import javax.inject.Named; + +import org.apache.ambari.logsearch.common.LogType; +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.ambari.logsearch.conf.SolrServiceLogPropsConfig; +import org.apache.ambari.logsearch.conf.global.SolrCollectionState; +import org.apache.ambari.logsearch.configurer.SolrCollectionConfigurer; +import org.apache.log4j.Logger; +import org.springframework.data.solr.core.SolrTemplate; + +@Named +public class ServiceLogsSolrDao extends SolrDaoBase { + + private static final Logger LOG = Logger.getLogger(ServiceLogsSolrDao.class); + + @Inject + private SolrServiceLogPropsConfig solrServiceLogPropsConfig; + + @Inject + @Named("serviceSolrTemplate") + private volatile SolrTemplate serviceSolrTemplate; + + @Inject + @Named("solrServiceLogsState") + private SolrCollectionState solrServiceLogsState; + + public ServiceLogsSolrDao() { + super(LogType.SERVICE); + } + + @Override + public SolrTemplate getSolrTemplate() { + return serviceSolrTemplate; + } + + @Override + public void setSolrTemplate(SolrTemplate solrTemplate) { + this.serviceSolrTemplate = solrTemplate; + } + + @PostConstruct + public void postConstructor() { + LOG.info("postConstructor() called."); + try { + new SolrCollectionConfigurer(this).start(); + } catch (Exception e) { + LOG.error("error while connecting to Solr for service logs : solrUrl=" + solrServiceLogPropsConfig.getSolrUrl() + + ", zkConnectString=" + solrServiceLogPropsConfig.getZkConnectString() + + ", collection=" + solrServiceLogPropsConfig.getCollection(), e); + } + } + + @Override + public SolrCollectionState getSolrCollectionState() { + return solrServiceLogsState; + } + + @Override + public SolrPropsConfig getSolrPropsConfig() { + return solrServiceLogPropsConfig; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0f1e340/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java new file mode 100644 index 0000000..b30b6ef --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.dao; + +import org.apache.ambari.logsearch.common.LogSearchContext; +import org.apache.ambari.logsearch.common.LogType; +import org.apache.ambari.logsearch.common.MessageEnums; +import org.apache.ambari.logsearch.conf.SolrKerberosConfig; +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.ambari.logsearch.conf.global.SolrCollectionState; +import org.apache.ambari.logsearch.util.RESTErrorUtil; +import org.apache.ambari.logsearch.util.SolrUtil; +import org.apache.log4j.Logger; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrRequest.METHOD; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.SolrResponseBase; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.springframework.data.solr.core.DefaultQueryParser; +import org.springframework.data.solr.core.SolrCallback; +import org.springframework.data.solr.core.SolrTemplate; +import org.springframework.data.solr.core.query.SolrDataQuery; + +import javax.inject.Inject; +import java.io.IOException; + +public abstract class SolrDaoBase { + + private static final Logger LOG = Logger.getLogger(SolrDaoBase.class); + private static final Logger LOG_PERFORMANCE = Logger.getLogger("org.apache.ambari.logsearch.performance"); + + private LogType logType; + + @Inject + private SolrKerberosConfig solrKerberosConfig; + + protected SolrDaoBase(LogType logType) { + this.logType = logType; + } + + public QueryResponse process(SolrQuery solrQuery, String event) { + SolrUtil.removeDoubleOrTripleEscapeFromFilters(solrQuery); + LOG.info("Solr query will be processed: " + solrQuery); + if (getSolrClient() != null) { + event = event == null ? solrQuery.get("event") : event; + solrQuery.remove("event"); + try { + QueryResponse queryResponse = getSolrClient().query(solrQuery, METHOD.POST); + logSolrEvent(event, solrQuery, queryResponse); + return queryResponse; + } catch (Exception e){ + LOG.error("Error during solrQuery=" + e); + throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + } + } else { + throw RESTErrorUtil.createRESTException("Solr configuration improper for " + logType.getLabel() +" logs", + MessageEnums.ERROR_SYSTEM); + } + } + + public UpdateResponse deleteByQuery(SolrQuery solrQuery, String event) { + SolrUtil.removeDoubleOrTripleEscapeFromFilters(solrQuery); + LOG.info("Solr delete query will be processed: " + solrQuery); + if (getSolrClient() != null) { + try { + UpdateResponse updateResponse = getSolrClient().deleteByQuery(solrQuery.getQuery()); + logSolrEvent(event, solrQuery, updateResponse); + return updateResponse; + } catch (Exception e) { + LOG.error("Error during delete solrQuery=" + e); + throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + } + } else { + throw RESTErrorUtil.createRESTException("Solr configuration improper for " + logType.getLabel() + " logs", + MessageEnums.ERROR_SYSTEM); + } + } + + public UpdateResponse deleteByQuery(SolrDataQuery solrDataQuery, String event) { + return deleteByQuery(new DefaultQueryParser().doConstructSolrQuery(solrDataQuery), event); + } + + public QueryResponse process(SolrQuery solrQuery) { + return process(solrQuery, null); + } + + public QueryResponse process(SolrDataQuery solrDataQuery) { + return process(new DefaultQueryParser().doConstructSolrQuery(solrDataQuery)); + } + + public long count(final SolrDataQuery solrDataQuery) { + return getSolrTemplate().execute(new SolrCallback<Long>() { + @Override + public Long doInSolr(SolrClient solrClient) throws SolrServerException, IOException { + SolrQuery solrQuery = new DefaultQueryParser().doConstructSolrQuery(solrDataQuery); + solrQuery.setStart(0); + solrQuery.setRows(0); + QueryResponse queryResponse = solrClient.query(solrQuery); + long count = solrClient.query(solrQuery).getResults().getNumFound(); + LOG_PERFORMANCE.info("\n Username :- " + LogSearchContext.getCurrentUsername() + " Count SolrQuery :- " + + solrQuery + "\nQuery Time Execution :- " + queryResponse.getQTime() + " Total Time Elapsed is :- " + + queryResponse.getElapsedTime() + " Count result :- " + count); + return count; + } + }); + } + + public QueryResponse process(SolrDataQuery solrDataQuery, String event) { + return process(new DefaultQueryParser().doConstructSolrQuery(solrDataQuery), event); + } + + private void logSolrEvent(String event, SolrQuery solrQuery, SolrResponseBase solrResponseBase) { + if (event != null) { + LOG_PERFORMANCE.info("\n Username :- " + LogSearchContext.getCurrentUsername() + " Event :- " + event + " SolrQuery :- " + + solrQuery + "\nQuery Time Execution :- " + solrResponseBase.getQTime() + " Total Time Elapsed is :- " + + solrResponseBase.getElapsedTime()); + } + } + + public CloudSolrClient getSolrClient() { + return (CloudSolrClient) getSolrTemplate().getSolrClient(); + } + + public abstract SolrTemplate getSolrTemplate(); + + public abstract void setSolrTemplate(SolrTemplate solrTemplate); + + public abstract SolrCollectionState getSolrCollectionState(); + + public abstract SolrPropsConfig getSolrPropsConfig(); + + public SolrKerberosConfig getSolrKerberosConfig() { + return this.solrKerberosConfig; + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0f1e340/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java new file mode 100644 index 0000000..01378c1 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.dao; + +import org.apache.ambari.logsearch.common.LogSearchConstants; +import org.apache.ambari.logsearch.common.LogType; +import org.apache.ambari.logsearch.common.MessageEnums; +import org.apache.ambari.logsearch.conf.SolrUserPropsConfig; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition; +import org.apache.solr.client.solrj.request.schema.SchemaRequest; +import org.apache.solr.client.solrj.response.LukeResponse; +import org.apache.solr.client.solrj.response.LukeResponse.FieldInfo; +import org.apache.solr.client.solrj.response.schema.SchemaResponse; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.JavaBinCodec; +import org.apache.solr.common.util.NamedList; +import org.codehaus.jettison.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import javax.inject.Inject; + +public class SolrSchemaFieldDao { + + private static final Logger LOG = LoggerFactory.getLogger(SolrSchemaFieldDao.class); + + private static final int RETRY_SECOND = 30; + + @Inject + private ServiceLogsSolrDao serviceLogsSolrDao; + + @Inject + private AuditSolrDao auditSolrDao; + + @Inject + private SolrUserPropsConfig solrUserConfigPropsConfig; + + private int retryCount; + private int skipCount; + + private Map<String, String> serviceSchemaFieldNameMap = new HashMap<>(); + private Map<String, String> serviceSchemaFieldTypeMap = new HashMap<>(); + private Map<String, String> auditSchemaFieldNameMap = new HashMap<>(); + private Map<String, String> auditSchemaFieldTypeMap = new HashMap<>(); + + @Scheduled(fixedDelay = RETRY_SECOND * 1000) + public void populateAllSchemaFields() { + if (skipCount > 0) { + skipCount--; + return; + } + if (serviceLogsSolrDao.getSolrCollectionState().isSolrCollectionReady()) { + CloudSolrClient serviceSolrClient = (CloudSolrClient) serviceLogsSolrDao.getSolrTemplate().getSolrClient(); + populateSchemaFields(serviceSolrClient, serviceSchemaFieldNameMap, serviceSchemaFieldTypeMap); + } + if (auditSolrDao.getSolrCollectionState().isSolrCollectionReady()) { + CloudSolrClient auditSolrClient = (CloudSolrClient) auditSolrDao.getSolrTemplate().getSolrClient(); + populateSchemaFields(auditSolrClient, auditSchemaFieldNameMap, auditSchemaFieldTypeMap); + } + } + + private void populateSchemaFields(CloudSolrClient solrClient, Map<String, String> schemaFieldNameMap, + Map<String, String> schemaFieldTypeMap) { + if (solrClient != null) { + LOG.debug("Started thread to get fields for collection=" + solrClient.getDefaultCollection()); + List<LukeResponse> lukeResponses = null; + SchemaResponse schemaResponse = null; + try { + lukeResponses = getLukeResponsesForCores(solrClient); + + SolrRequest<SchemaResponse> schemaRequest = new SchemaRequest(); + schemaRequest.setMethod(SolrRequest.METHOD.GET); + schemaRequest.setPath("/schema"); + schemaResponse = schemaRequest.process(solrClient); + + LOG.debug("populateSchemaFields() collection=" + solrClient.getDefaultCollection() + ", luke=" + lukeResponses + + ", schema= " + schemaResponse); + } catch (SolrException | SolrServerException | IOException e) { + LOG.error("Error occured while popuplating field. collection=" + solrClient.getDefaultCollection(), e); + } + + if (schemaResponse != null) { + extractSchemaFieldsName(lukeResponses, schemaResponse, schemaFieldNameMap, schemaFieldTypeMap); + LOG.debug("Populate fields for collection " + solrClient.getDefaultCollection()+ " was successful, next update it after " + + solrUserConfigPropsConfig.getPopulateIntervalMins() + " minutes"); + retryCount = 0; + skipCount = (solrUserConfigPropsConfig.getPopulateIntervalMins() * 60) / RETRY_SECOND - 1; + } + else { + retryCount++; + LOG.error("Error while populating fields for collection " + solrClient.getDefaultCollection() + ", retryCount=" + retryCount); + } + } + } + + private static final String LUKE_REQUEST_URL_SUFFIX = "admin/luke?numTerms=0&wt=javabin&version=2"; + + @SuppressWarnings("unchecked") + private List<LukeResponse> getLukeResponsesForCores(CloudSolrClient solrClient) { + ZkStateReader zkStateReader = solrClient.getZkStateReader(); + Collection<Slice> activeSlices = zkStateReader.getClusterState().getActiveSlices(solrClient.getDefaultCollection()); + + List<LukeResponse> lukeResponses = new ArrayList<>(); + for (Slice slice : activeSlices) { + for (Replica replica : slice.getReplicas()) { + try (CloseableHttpClient httpClient = HttpClientUtil.createClient(null)) { + HttpGet request = new HttpGet(replica.getCoreUrl() + LUKE_REQUEST_URL_SUFFIX); + HttpResponse response = httpClient.execute(request); + NamedList<Object> lukeData = (NamedList<Object>) new JavaBinCodec(null, null).unmarshal(response.getEntity().getContent()); + LukeResponse lukeResponse = new LukeResponse(); + lukeResponse.setResponse(lukeData); + lukeResponses.add(lukeResponse); + } catch (IOException e) { + LOG.error("Exception during getting luke responses", e); + } + } + } + return lukeResponses; + } + + private void extractSchemaFieldsName(List<LukeResponse> lukeResponses, SchemaResponse schemaResponse, + Map<String, String> schemaFieldNameMap, Map<String, String> schemaFieldTypeMap) { + try { + HashMap<String, String> _schemaFieldNameMap = new HashMap<>(); + HashMap<String, String> _schemaFieldTypeMap = new HashMap<>(); + + for (LukeResponse lukeResponse : lukeResponses) { + for (Entry<String, FieldInfo> e : lukeResponse.getFieldInfo().entrySet()) { + String name = e.getKey(); + String type = e.getValue().getType(); + if (!name.contains("@") && !name.startsWith("_") && !name.contains("_md5") && !name.contains("_ms") && + !name.contains(LogSearchConstants.NGRAM_PREFIX) && !name.contains("tags") && !name.contains("_str")) { + _schemaFieldNameMap.put(name, type); + } + } + } + + List<FieldTypeDefinition> fieldTypes = schemaResponse.getSchemaRepresentation().getFieldTypes(); + for (FieldTypeDefinition fieldType : fieldTypes) { + Map<String, Object> fieldAttributes = fieldType.getAttributes(); + String name = (String) fieldAttributes.get("name"); + String fieldTypeJson = new JSONObject(fieldAttributes).toString(); + _schemaFieldTypeMap.put(name, fieldTypeJson); + } + + List<Map<String, Object>> fields = schemaResponse.getSchemaRepresentation().getFields(); + for (Map<String, Object> field : fields) { + String name = (String) field.get("name"); + String type = (String) field.get("type"); + if (!name.contains("@") && !name.startsWith("_") && !name.contains("_md5") && !name.contains("_ms") && + !name.contains(LogSearchConstants.NGRAM_PREFIX) && !name.contains("tags") && !name.contains("_str")) { + _schemaFieldNameMap.put(name, type); + } + } + + if (_schemaFieldNameMap.isEmpty() || _schemaFieldTypeMap.isEmpty()) { + return; + } + + synchronized (this) { + schemaFieldNameMap.clear(); + schemaFieldNameMap.putAll(_schemaFieldNameMap); + schemaFieldTypeMap.clear(); + schemaFieldTypeMap.putAll(_schemaFieldTypeMap); + } + } catch (Exception e) { + LOG.error(e + "Credentials not specified in logsearch.properties " + MessageEnums.ERROR_SYSTEM); + } + } + + public Map<String, String> getSchemaFieldNameMap(LogType logType) { + return LogType.AUDIT == logType ? auditSchemaFieldNameMap : serviceSchemaFieldNameMap; + } + + public Map<String, String> getSchemaFieldTypeMap(LogType logType) { + return LogType.AUDIT == logType ? auditSchemaFieldTypeMap : serviceSchemaFieldTypeMap; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0f1e340/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java new file mode 100644 index 0000000..256ddae --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.dao; + +import java.io.IOException; +import java.util.List; +import java.util.Set; +import java.util.TreeMap; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import javax.inject.Named; + +import org.apache.ambari.logsearch.common.HadoopServiceConfigHelper; +import org.apache.ambari.logsearch.common.LogSearchConstants; +import org.apache.ambari.logsearch.common.LogSearchContext; +import org.apache.ambari.logsearch.common.LogType; +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.ambari.logsearch.conf.SolrUserPropsConfig; +import org.apache.ambari.logsearch.conf.global.SolrCollectionState; +import org.apache.ambari.logsearch.configurer.LogfeederFilterConfigurer; +import org.apache.ambari.logsearch.configurer.SolrCollectionConfigurer; +import org.apache.ambari.logsearch.model.common.LogFeederDataMap; +import org.apache.ambari.logsearch.model.common.LogfeederFilterData; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; + +import org.apache.ambari.logsearch.util.JSONUtil; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Logger; +import org.springframework.data.solr.core.SolrTemplate; + +import static org.apache.ambari.logsearch.solr.SolrConstants.UserConfigConstants.ID; +import static org.apache.ambari.logsearch.solr.SolrConstants.UserConfigConstants.USER_NAME; +import static org.apache.ambari.logsearch.solr.SolrConstants.UserConfigConstants.VALUES; +import static org.apache.ambari.logsearch.solr.SolrConstants.UserConfigConstants.FILTER_NAME; +import static org.apache.ambari.logsearch.solr.SolrConstants.UserConfigConstants.ROW_TYPE; + +@Named +public class UserConfigSolrDao extends SolrDaoBase { + + private static final Logger LOG = Logger.getLogger(UserConfigSolrDao.class); + + private static final Logger LOG_PERFORMANCE = Logger.getLogger("org.apache.ambari.logsearch.performance"); + + @Inject + private SolrUserPropsConfig solrUserConfig; + + @Inject + @Named("userConfigSolrTemplate") + private SolrTemplate userConfigSolrTemplate; + + @Inject + @Named("solrUserConfigState") + private SolrCollectionState solrUserConfigState; + + public UserConfigSolrDao() { + super(LogType.SERVICE); + } + + @Override + public SolrTemplate getSolrTemplate() { + return userConfigSolrTemplate; + } + + @Override + public void setSolrTemplate(SolrTemplate solrTemplate) { + this.userConfigSolrTemplate = solrTemplate; + } + + @PostConstruct + public void postConstructor() { + String solrUrl = solrUserConfig.getSolrUrl(); + String zkConnectString = solrUserConfig.getZkConnectString(); + String collection = solrUserConfig.getCollection(); + + try { + new SolrCollectionConfigurer(this).start(); + new LogfeederFilterConfigurer(this).start(); + } catch (Exception e) { + LOG.error("error while connecting to Solr for history logs : solrUrl=" + solrUrl + ", zkConnectString=" + zkConnectString + + ", collection=" + collection, e); + } + } + + public void saveUserFilter(LogFeederDataMap logfeederFilterWrapper) throws SolrException, SolrServerException, IOException { + String filterName = LogSearchConstants.LOGFEEDER_FILTER_NAME; + String json = JSONUtil.objToJson(logfeederFilterWrapper); + SolrInputDocument configDocument = new SolrInputDocument(); + configDocument.addField(ID, logfeederFilterWrapper.getId()); + configDocument.addField(ROW_TYPE, filterName); + configDocument.addField(VALUES, json); + configDocument.addField(USER_NAME, filterName); + configDocument.addField(FILTER_NAME, filterName); + addDocs(configDocument); + } + + public void deleteUserConfig(String id) throws SolrException, SolrServerException, IOException { + removeDoc("id:" + id); + } + + public UpdateResponse addDocs(SolrInputDocument doc) throws SolrServerException, IOException, SolrException { + UpdateResponse updateResoponse = getSolrClient().add(doc); + LOG_PERFORMANCE.info("\n Username :- " + LogSearchContext.getCurrentUsername() + + " Update Time Execution :- " + updateResoponse.getQTime() + " Total Time Elapsed is :- " + updateResoponse.getElapsedTime()); + getSolrClient().commit(); + return updateResoponse; + } + + public UpdateResponse removeDoc(String query) throws SolrServerException, IOException, SolrException { + UpdateResponse updateResoponse = getSolrClient().deleteByQuery(query); + getSolrClient().commit(); + LOG_PERFORMANCE.info("\n Username :- " + LogSearchContext.getCurrentUsername() + + " Remove Time Execution :- " + updateResoponse.getQTime() + " Total Time Elapsed is :- " + updateResoponse.getElapsedTime()); + return updateResoponse; + } + + public LogFeederDataMap getUserFilter() throws SolrServerException, IOException { + SolrQuery solrQuery = new SolrQuery(); + solrQuery.setQuery("*:*"); + solrQuery.setFilterQueries(ROW_TYPE + ":" + LogSearchConstants.LOGFEEDER_FILTER_NAME); + + QueryResponse response = process(solrQuery); + SolrDocumentList documentList = response.getResults(); + LogFeederDataMap logfeederDataMap = null; + if (CollectionUtils.isNotEmpty(documentList)) { + SolrDocument configDoc = documentList.get(0); + String json = (String) configDoc.get(VALUES); + logfeederDataMap = (LogFeederDataMap) JSONUtil.jsonToObj(json, LogFeederDataMap.class); + logfeederDataMap.setId("" + configDoc.get(ID)); + } else { + logfeederDataMap = new LogFeederDataMap(); + logfeederDataMap.setFilter(new TreeMap<String, LogfeederFilterData>()); + logfeederDataMap.setId(Long.toString(System.currentTimeMillis())); + } + + addMissingFilters(logfeederDataMap); + + return logfeederDataMap; + } + + private void addMissingFilters(LogFeederDataMap logfeederDataMap) throws SolrServerException, IOException { + Set<String> logIds = HadoopServiceConfigHelper.getAllLogIds(); + if (logIds != null) { + List<String> logfeederDefaultLevels = solrUserConfig.getLogLevels(); + + boolean modified = false; + for (String logId : logIds) { + if (!logfeederDataMap.getFilter().containsKey(logId)) { + LogfeederFilterData logfeederFilterData = new LogfeederFilterData(); + logfeederFilterData.setLabel(logId); + logfeederFilterData.setDefaultLevels(logfeederDefaultLevels); + logfeederDataMap.getFilter().put(logId, logfeederFilterData); + modified = true; + } + } + + if (modified) { + saveUserFilter(logfeederDataMap); + } + } + } + + @Override + public SolrCollectionState getSolrCollectionState() { + return solrUserConfigState; + } + + @Override + public SolrPropsConfig getSolrPropsConfig() { + return solrUserConfig; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b0f1e340/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java new file mode 100644 index 0000000..63cc89e --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.dao; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; + +import org.apache.ambari.logsearch.conf.AuthPropsConfig; +import org.springframework.security.core.GrantedAuthority; +import org.springframework.stereotype.Repository; +import org.apache.ambari.logsearch.util.CommonUtil; +import org.apache.ambari.logsearch.util.FileUtil; +import org.apache.ambari.logsearch.util.JSONUtil; +import org.apache.ambari.logsearch.web.model.Privilege; +import org.apache.ambari.logsearch.web.model.Role; +import org.apache.ambari.logsearch.web.model.User; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.Predicate; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; + +@Repository +public class UserDao { + private static final Logger logger = Logger.getLogger(UserDao.class); + + private static final String USER_NAME = "username"; + private static final String PASSWORD = "password"; + private static final String ENC_PASSWORD = "en_password"; + private static final String NAME = "name"; + + @Inject + private AuthPropsConfig authPropsConfig; + + private ArrayList<HashMap<String, String>> userList = null; + + @SuppressWarnings("unchecked") + @PostConstruct + public void initialization() { + if (authPropsConfig.isAuthFileEnabled()) { + try { + String userPassJsonFileName = authPropsConfig.getCredentialsFile(); + logger.info("USER PASS JSON file NAME:" + userPassJsonFileName); + File jsonFile = FileUtil.getFileFromClasspath(userPassJsonFileName); + if (jsonFile == null || !jsonFile.exists()) { + logger.fatal("user_pass json file not found in classpath :" + userPassJsonFileName); + System.exit(1); + } + HashMap<String, Object> userInfos = JSONUtil.readJsonFromFile(jsonFile); + userList = (ArrayList<HashMap<String, String>>) userInfos.get("users"); + if (userList != null) { + boolean isUpdated = this.encryptAllPassword(); + userInfos.put("users", userList); + if (isUpdated) { + String jsonStr = JSONUtil.mapToJSON(userInfos); + JSONUtil.writeJSONInFile(jsonStr, jsonFile, true); + } + } else { + userList = new ArrayList<HashMap<String, String>>(); + } + + } catch (Exception exception) { + logger.error("Error while reading user prop file :" + exception.getMessage()); + userList = new ArrayList<HashMap<String, String>>(); + } + } else { + logger.info("File auth is disabled."); + } + } + + public User loadUserByUsername(String username) { + logger.debug(" loadUserByUsername username" + username); + HashMap<String, String> userInfo = findByusername(username); + if (userInfo == null) { + return null; + } + + User user = new User(); + user.setFirstName(StringUtils.defaultString(userInfo.get(NAME), "Unknown")); + user.setLastName(StringUtils.defaultString(userInfo.get(NAME), "Unknown")); + user.setUsername(StringUtils.defaultString(userInfo.get(USER_NAME), "")); + user.setPassword(StringUtils.defaultString(userInfo.get(ENC_PASSWORD), "")); + + Role r = new Role(); + r.setName("ROLE_USER"); + Privilege priv = new Privilege(); + priv.setName("READ_PRIVILEGE"); + r.setPrivileges(Arrays.asList(priv)); + user.setAuthorities(Arrays.asList((GrantedAuthority)r)); + + return user; + } + + private HashMap<String, String> findByusername(final String username) { + if (userList == null) { + return null; + } + @SuppressWarnings("unchecked") + HashMap<String, String> userInfo = (HashMap<String, String>) CollectionUtils.find(userList, + new Predicate() { + @Override + public boolean evaluate(Object args) { + HashMap<String, String> tmpUserInfo = (HashMap<String, String>) args; + String objUsername = tmpUserInfo.get(USER_NAME); + return (objUsername != null && username != null && username.equalsIgnoreCase(objUsername)); + } + }); + + return userInfo; + } + + private boolean encryptAllPassword() { + boolean isUpdated = false; + for (HashMap<String, String> user : userList) { + String encPassword = user.get(ENC_PASSWORD); + String username = user.get(USER_NAME); + String password = user.get(PASSWORD); + if (StringUtils.isNotBlank(password)) { + encPassword = CommonUtil.encryptPassword(username, password); + user.put(PASSWORD, ""); + user.put(ENC_PASSWORD, encPassword); + isUpdated = true; + } + if (StringUtils.isBlank(password) && StringUtils.isBlank(encPassword)) { + logger.error("Password is empty or null for username : " + username); + } + } + return isUpdated; + } +}
