AMBARI-18548. Declarative Logsearch/Logfeeder Component Metadata for Stack Component (oleewere)
Change-Id: Ic29344f7714b89b01affade0f0d3e1c3c58942dd Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e734a7bf Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e734a7bf Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e734a7bf Branch: refs/heads/branch-dev-patch-upgrade Commit: e734a7bf0989ef621065ef4ab83be1b3366c86bd Parents: cf61937 Author: oleewere <oleew...@gmail.com> Authored: Thu Oct 27 14:39:09 2016 +0200 Committer: oleewere <oleew...@gmail.com> Committed: Sat Nov 5 16:08:57 2016 +0100 ---------------------------------------------------------------------- .../AbstractLogRequestFacetQueryConverter.java | 1 + .../AuditComponentsRequestQueryConverter.java | 31 +- .../AuditServiceLoadRequestQueryConverter.java | 1 + .../BaseServiceLogRequestQueryConverter.java | 5 - .../ambari/logsearch/dao/SolrDaoBase.java | 1 + .../request/impl/AuditComponentRequest.java | 2 +- .../main/webapp/scripts/views/common/Header.js | 2 +- .../scripts/views/dashboard/MainLayoutView.js | 8 +- .../scripts/views/tabs/ComparisonLayoutView.js | 6 +- ...AuditComponentRequestQueryConverterTest.java | 4 +- ...BaseServiceLogRequestQueryConverterTest.java | 5 +- .../ServiceLogAnyGraphRequestConverterTest.java | 4 +- ...ComponentLevelRequestQueryConverterTest.java | 4 +- ...ComponentRequestFacetQueryConverterTest.java | 4 +- ...eLogLevelCountRequestQueryConverterTest.java | 4 +- ...ceLogTreeRequestFacetQueryConverterTest.java | 4 +- .../UserExportRequestQueryConverterTest.java | 4 +- .../configuration/accumulo-logsearch-conf.xml | 124 ++++ .../configuration/infra-logsearch-conf.xml | 80 +++ .../0.1.0/configuration/ams-logsearch-conf.xml | 201 +++++++ .../configuration/atlas-logsearch-conf.xml | 80 +++ .../configuration/falcon-logsearch-conf.xml | 80 +++ .../configuration/flume-logsearch-conf.xml | 80 +++ .../configuration/hbase-logsearch-conf.xml | 111 ++++ .../configuration/hdfs-logsearch-conf.xml | 248 ++++++++ .../configuration/hive-logsearch-conf.xml | 117 ++++ .../configuration/kafka-logsearch-conf.xml | 124 ++++ .../configuration/knox-logsearch-conf.xml | 93 +++ .../configuration/logfeeder-system_log-env.xml | 59 ++ .../0.5.0/package/scripts/logfeeder.py | 1 - .../0.5.0/package/scripts/logsearch.py | 1 - .../scripts/logsearch_config_aggregator.py | 67 +++ .../LOGSEARCH/0.5.0/package/scripts/params.py | 59 +- .../0.5.0/package/scripts/setup_logfeeder.py | 17 +- .../templates/HadoopServiceConfig.json.j2 | 582 +++---------------- .../templates/input.config-accumulo.json.j2 | 105 ---- .../templates/input.config-ambari.json.j2 | 2 +- .../package/templates/input.config-ams.json.j2 | 191 ------ .../templates/input.config-atlas.json.j2 | 55 -- .../templates/input.config-falcon.json.j2 | 55 -- .../templates/input.config-flume.json.j2 | 56 -- .../templates/input.config-hbase.json.j2 | 91 --- .../package/templates/input.config-hdfs.json.j2 | 251 -------- .../package/templates/input.config-hive.json.j2 | 97 ---- .../templates/input.config-infra.json.j2 | 56 -- .../templates/input.config-kafka.json.j2 | 105 ---- .../package/templates/input.config-knox.json.j2 | 68 --- .../templates/input.config-oozie.json.j2 | 56 -- .../templates/input.config-ranger.json.j2 | 122 ---- .../templates/input.config-spark.json.j2 | 72 --- .../templates/input.config-spark2.json.j2 | 66 --- .../templates/input.config-storm.json.j2 | 86 --- .../package/templates/input.config-yarn.json.j2 | 86 --- .../templates/input.config-zeppelin.json.j2 | 56 -- .../templates/input.config-zookeeper.json.j2 | 56 -- .../LOGSEARCH/0.5.0/properties/secure_log.json | 31 + .../0.5.0/properties/system_messages.json | 31 + .../configuration/oozie-logsearch-conf.xml | 80 +++ .../configuration/ranger-logsearch-conf.xml | 111 ++++ .../configuration/ranger-kms-logsearch-conf.xml | 80 +++ .../configuration/spark-logsearch-conf.xml | 98 ++++ .../configuration/spark2-logsearch-conf.xml | 92 +++ .../configuration/storm-logsearch-conf.xml | 110 ++++ .../mapred-logsearch-conf.xml | 80 +++ .../configuration/yarn-logsearch-conf.xml | 104 ++++ .../configuration/zeppelin-logsearch-conf.xml | 80 +++ .../configuration/zookeeper-logsearch-conf.xml | 76 +++ .../stacks/2.4/LOGSEARCH/test_logfeeder.py | 9 +- .../stacks/2.4/LOGSEARCH/test_logsearch.py | 1 - .../test/python/stacks/2.4/configs/default.json | 8 +- 70 files changed, 2597 insertions(+), 2340 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractLogRequestFacetQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractLogRequestFacetQueryConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractLogRequestFacetQueryConverter.java index 9dbab24..b377968 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractLogRequestFacetQueryConverter.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractLogRequestFacetQueryConverter.java @@ -48,6 +48,7 @@ public abstract class AbstractLogRequestFacetQueryConverter<SOURCE extends BaseL FacetOptions facetOptions = new FacetOptions(); facetOptions.setFacetMinCount(1); facetOptions.setFacetSort(getFacetSort()); + facetOptions.setFacetLimit(-1); appendFacetOptions(facetOptions, request); addIncludeFieldValues(facetQuery, StringEscapeUtils.unescapeXml(request.getIncludeQuery())); addExcludeFieldValues(facetQuery, StringEscapeUtils.unescapeXml(request.getExcludeQuery())); http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AuditComponentsRequestQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AuditComponentsRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AuditComponentsRequestQueryConverter.java index d81bdbf..bf77444 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AuditComponentsRequestQueryConverter.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AuditComponentsRequestQueryConverter.java @@ -18,45 +18,36 @@ */ package org.apache.ambari.logsearch.converter; -import org.apache.ambari.logsearch.common.LogSearchConstants; import org.apache.ambari.logsearch.common.LogType; import org.apache.ambari.logsearch.model.request.impl.AuditComponentRequest; -import org.apache.commons.lang3.StringUtils; -import org.springframework.data.domain.Sort; import org.springframework.data.solr.core.query.FacetOptions; -import org.springframework.data.solr.core.query.SimpleFacetQuery; import javax.inject.Named; import static org.apache.ambari.logsearch.solr.SolrConstants.AuditLogConstants.AUDIT_COMPONENT; +import static org.apache.ambari.logsearch.solr.SolrConstants.AuditLogConstants.AUDIT_EVTTIME; @Named -public class AuditComponentsRequestQueryConverter extends AbstractSearchRequestQueryConverter<AuditComponentRequest, SimpleFacetQuery> { +public class AuditComponentsRequestQueryConverter extends AbstractLogRequestFacetQueryConverter<AuditComponentRequest> { @Override - public SimpleFacetQuery extendSolrQuery(AuditComponentRequest request, SimpleFacetQuery query) { - FacetOptions facetOptions = new FacetOptions(); // TODO: check that date filtering is needed or not - facetOptions.addFacetOnField(AUDIT_COMPONENT); - facetOptions.setFacetSort(FacetOptions.FacetSort.INDEX); - facetOptions.setFacetLimit(-1); - query.setFacetOptions(facetOptions); - return query; + public FacetOptions.FacetSort getFacetSort() { + return FacetOptions.FacetSort.INDEX; } @Override - public Sort sort(AuditComponentRequest request) { - Sort.Direction direction = StringUtils.equals(request.getSortType(), LogSearchConstants.DESCENDING_ORDER) - ? Sort.Direction.DESC : Sort.Direction.ASC; - return new Sort(new Sort.Order(direction, AUDIT_COMPONENT)); + public String getDateTimeField() { + return AUDIT_EVTTIME; } @Override - public SimpleFacetQuery createQuery() { - return new SimpleFacetQuery(); + public LogType getLogType() { + return LogType.AUDIT; } @Override - public LogType getLogType() { - return LogType.AUDIT; + public void appendFacetOptions(FacetOptions facetOptions, AuditComponentRequest request) { + facetOptions.addFacetOnField(AUDIT_COMPONENT); + facetOptions.setFacetLimit(-1); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverter.java index fb1c763..d88ec6d 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverter.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverter.java @@ -48,5 +48,6 @@ public class AuditServiceLoadRequestQueryConverter extends AbstractLogRequestFac @Override public void appendFacetOptions(FacetOptions facetOptions, AuditServiceLoadRequest request) { facetOptions.addFacetOnField(AUDIT_COMPONENT); + facetOptions.setFacetLimit(10); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverter.java index 8270f2d..2601f72 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverter.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverter.java @@ -81,9 +81,4 @@ public class BaseServiceLogRequestQueryConverter extends AbstractServiceLogReque public LogType getLogType() { return LogType.SERVICE; } - - @Override - public void addComponentFilters(BaseServiceLogRequest request, SimpleQuery query) { - - } } http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java index 77eb51f..39b65ae 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java @@ -51,6 +51,7 @@ public abstract class SolrDaoBase { public QueryResponse process(SolrQuery solrQuery, String event) { SolrUtil.removeDoubleOrTripleEscapeFromFilters(solrQuery); + LOG.info("Solr query will be processed: " + solrQuery); if (getSolrClient() != null) { event = event == null ? solrQuery.get("event") : event; solrQuery.remove("event"); http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditComponentRequest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditComponentRequest.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditComponentRequest.java index 94cb255..636c492 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditComponentRequest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditComponentRequest.java @@ -21,5 +21,5 @@ package org.apache.ambari.logsearch.model.request.impl; import org.apache.ambari.logsearch.common.Marker; @Marker -public class AuditComponentRequest extends CommonSearchRequest { +public class AuditComponentRequest extends BaseLogRequest { } http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/Header.js ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/Header.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/Header.js index 1e469d5..deb0e28 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/Header.js +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/Header.js @@ -168,7 +168,7 @@ define(['require', require(['views/filter/CreateLogfeederFilterView'],function(CreateLogfeederFilter){ var view = new CreateLogfeederFilter({}); var options = { - title: "Log Feeder Log Levels", + title: "Log Feeder Log Levels Filter", content: view, viewType: 'Filter', resizable: false, http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/MainLayoutView.js ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/MainLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/MainLayoutView.js index aa79ead..2333c53 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/MainLayoutView.js +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/MainLayoutView.js @@ -174,8 +174,8 @@ define(['require', require(['views/tabs/ComparisonLayoutView'], function(ComparisonLayoutView){ var tabName = ""; _.each(view.componetList,function(object){ - if(object.host && object.component){ - tabName += (object.host + object.component).replace(/\./g,"_"); + if(object.host_name && object.component_name){ + tabName += (object.host_name + object.component_name).replace(/\./g,"_"); } }); if(_.isUndefined(that[tabName])){ @@ -511,10 +511,10 @@ define(['require', var component = dataValue.component || dataValue.node; var spanLength = this.$('.compare .panel-body span.hasNode'); if (spanLength.length != 0 && spanLength.length >= 1) { - this.componetList.push({ 'host': host, 'component': component, id: clickedId }); + this.componetList.push({ 'host_name': host, 'component_name': component, id: clickedId }); this.$('.compare .panel-body .hostCompList').append('<span class="hasNode" data-id="' + clickedId + '"><i class=" closeComponent fa fa-times-circle"></i>' + host.split(".")[0] + ' <i class="fa fa-angle-double-right"></i><br> ' + component + '</span>'); } else { - this.componetList.push({ 'host': host, 'component': component, id: clickedId }); + this.componetList.push({ 'host_name': host, 'component_name': component, id: clickedId }); this.$('.compare .panel-body .hostCompList').html('<span class="hasNode" data-id="' + clickedId + '"><i class=" closeComponent fa fa-times-circle"></i>' + host.split(".")[0] + ' <i class="fa fa-angle-double-right"></i><br> ' + component + '</span>'); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/ComparisonLayoutView.js ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/ComparisonLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/ComparisonLayoutView.js index 526e3c1..73cca56 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/ComparisonLayoutView.js +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/ComparisonLayoutView.js @@ -61,7 +61,7 @@ define(['require', if (that.componetList) { var $parent = that.ui.comparisonTab; _.each(that.componetList, function(object,i) { - var id = (object.host + '_' + object.component).replace(/\./g, "_"); + var id = (object.host_name + '_' + object.component_name).replace(/\./g, "_"); if(i % 2 == 0 && i > 0){ var $div = $("<div class='row comparisonTab'></div>"); that.ui.comparisonTab.parent().append($div); @@ -76,8 +76,8 @@ define(['require', region.show(new ComparisonView({ globalVent: that.globalVent, params: _.extend({},that.params, { - 'host': object.host, - 'component': object.component + 'host_name': object.host_name, + 'component_name': object.component_name }), datePickerPosition:(((i+1) % 2 == 0)?("left"):("right")) })); http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditComponentRequestQueryConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditComponentRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditComponentRequestQueryConverterTest.java index d6ecdd5..0c6e975 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditComponentRequestQueryConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditComponentRequestQueryConverterTest.java @@ -46,7 +46,7 @@ public class AuditComponentRequestQueryConverterTest extends AbstractRequestConv SimpleFacetQuery facetQuery = underTest.convert(request); SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(facetQuery); // THEN - assertEquals("?q=*%3A*&start=0&rows=25&sort=repo+asc&facet=true&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.field=repo", + assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B*+TO+*%5D&facet=true&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.field=repo", query.toQueryString()); } @@ -59,7 +59,7 @@ public class AuditComponentRequestQueryConverterTest extends AbstractRequestConv SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(facetQuery); // THEN assertNotNull(facetQuery); - assertEquals("?q=*%3A*&start=0&rows=99999&sort=repo+asc&facet=true&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.field=repo", + assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B*+TO+*%5D&facet=true&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.field=repo", query.toQueryString()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java index a75037b..3775a2d 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java @@ -52,8 +52,9 @@ public class BaseServiceLogRequestQueryConverterTest extends AbstractRequestConv SolrQuery solrQuery = defaultQueryParser.doConstructSolrQuery(query); SolrUtil.removeDoubleOrTripleEscapeFromFilters(solrQuery); // THEN - assertEquals("?q=*%3A*&start=0&rows=25&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*" + - "&fq=host%3Alogsearch.com&fq=path%3A%5C%2Fvar%5C%2Flog%5C%2Fmyfile%5C-%5C*%5C-hdfs.log&fq=type%3Acomponent&fq=level%3A%28FATAL+ERROR+WARN+UNKNOWN%29" + + assertEquals("?q=*%3A*&start=0&rows=25&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" + + "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&fq=host%3Alogsearch.com" + + "&fq=path%3A%5C%2Fvar%5C%2Flog%5C%2Fmyfile%5C-%5C*%5C-hdfs.log&fq=type%3Acomponent&fq=level%3A%28FATAL+ERROR+WARN+UNKNOWN%29" + "&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&sort=logtime+desc%2Cseq_num+desc", solrQuery.toQueryString()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java index 487ad11..63dc733 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java @@ -46,7 +46,7 @@ public class ServiceLogAnyGraphRequestConverterTest extends AbstractRequestConve // THEN assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3A*myincludemessage*" + "&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" + - "&facet=true&facet.mincount=1&facet.limit=10&facet.field=level", query.toQueryString()); + "&facet=true&facet.mincount=1&facet.limit=-1&facet.field=level", query.toQueryString()); } @Test @@ -56,7 +56,7 @@ public class ServiceLogAnyGraphRequestConverterTest extends AbstractRequestConve // WHEN SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request)); // THEN - assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B*+TO+*%5D&facet=true&facet.mincount=1&facet.limit=10&facet.field=level", + assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B*+TO+*%5D&facet=true&facet.mincount=1&facet.limit=-1&facet.field=level", query.toQueryString()); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java index 47f7a6d..2a7cc16 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java @@ -47,7 +47,7 @@ public class ServiceLogComponentLevelRequestQueryConverterTest extends AbstractR // THEN assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3A*myincludemessage*" + "&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" + - "&fq=level%3A%28WARN+ERROR+FATAL%29&facet=true&facet.mincount=1&facet.limit=10&facet.sort=index&facet.pivot=type%2Clevel", + "&fq=level%3A%28WARN+ERROR+FATAL%29&facet=true&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.pivot=type%2Clevel", query.toQueryString()); } @@ -60,6 +60,6 @@ public class ServiceLogComponentLevelRequestQueryConverterTest extends AbstractR SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request)); // THEN assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B*+TO+*%5D&fq=level%3A%28WARN+ERROR+FATAL%29&facet=true" + - "&facet.mincount=1&facet.limit=10&facet.sort=index&facet.pivot=type%2Clevel", query.toQueryString()); + "&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.pivot=type%2Clevel", query.toQueryString()); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java index d40bda0..a5ea683 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java @@ -47,7 +47,7 @@ public class ServiceLogComponentRequestFacetQueryConverterTest extends AbstractR // THEN assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D" + "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29" + - "&fq=-type%3A%28hst_agent+system_message%29&fq=level%3A%28WARN+ERROR+FATAL%29&facet=true&facet.mincount=1&facet.limit=10" + + "&fq=-type%3A%28hst_agent+system_message%29&fq=level%3A%28WARN+ERROR+FATAL%29&facet=true&facet.mincount=1&facet.limit=-1" + "&facet.sort=index&facet.pivot=type%2Chost%2Clevel&facet.pivot=type%2Clevel", query.toQueryString()); } @@ -61,7 +61,7 @@ public class ServiceLogComponentRequestFacetQueryConverterTest extends AbstractR SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request)); // THEN assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B*+TO+*%5D&fq=level%3A%28WARN+ERROR+FATAL%29" + - "&facet=true&facet.mincount=1&facet.limit=10&facet.sort=index&facet.pivot=type%2Chost%2Clevel&facet.pivot=type%2Clevel", + "&facet=true&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.pivot=type%2Chost%2Clevel&facet.pivot=type%2Clevel", query.toQueryString()); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java index a4a3256..b79ff91 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java @@ -45,7 +45,7 @@ public class ServiceLogLevelCountRequestQueryConverterTest extends AbstractReque // THEN assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D" + "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29" + - "&fq=-type%3A%28hst_agent+system_message%29&facet=true&facet.mincount=1&facet.limit=10&facet.field=level", + "&fq=-type%3A%28hst_agent+system_message%29&facet=true&facet.mincount=1&facet.limit=-1&facet.field=level", query.toQueryString()); } @@ -56,7 +56,7 @@ public class ServiceLogLevelCountRequestQueryConverterTest extends AbstractReque // WHEN SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request)); // THEN - assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B*+TO+*%5D&facet=true&facet.mincount=1&facet.limit=10&facet.field=level", + assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B*+TO+*%5D&facet=true&facet.mincount=1&facet.limit=-1&facet.field=level", query.toQueryString()); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java index 481bfeb..634ac19 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java @@ -46,7 +46,7 @@ public class ServiceLogTreeRequestFacetQueryConverterTest extends AbstractReques // THEN assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3A*myincludemessage*" + "&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29&fq=level%3A%28WARN+ERROR+FATAL%29" + - "&facet=true&facet.mincount=1&facet.limit=10&facet.sort=index&facet.pivot=host%2Ctype%2Clevel&facet.pivot=host%2Clevel", + "&facet=true&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.pivot=host%2Ctype%2Clevel&facet.pivot=host%2Clevel", query.toQueryString()); } @@ -59,7 +59,7 @@ public class ServiceLogTreeRequestFacetQueryConverterTest extends AbstractReques SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request)); // THEN assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B*+TO+*%5D&fq=level%3A%28WARN+ERROR+FATAL%29&facet=true&facet.mincount=1" + - "&facet.limit=10&facet.sort=index&facet.pivot=host%2Ctype%2Clevel&facet.pivot=host%2Clevel", + "&facet.limit=-1&facet.sort=index&facet.pivot=host%2Ctype%2Clevel&facet.pivot=host%2Clevel", query.toQueryString()); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java index 7728876..6644ca1 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java @@ -45,7 +45,7 @@ public class UserExportRequestQueryConverterTest extends AbstractRequestConverte // THEN assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D" + "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&fq=repo%3A%28logsearch_app+secure_log%29" + - "&fq=-repo%3A%28hst_agent+system_message%29&facet=true&facet.mincount=1&facet.limit=10&facet.pivot=reqUser%2Crepo&facet.pivot=resource%2Crepo", + "&fq=-repo%3A%28hst_agent+system_message%29&facet=true&facet.mincount=1&facet.limit=-1&facet.pivot=reqUser%2Crepo&facet.pivot=resource%2Crepo", query.toQueryString()); } @@ -56,7 +56,7 @@ public class UserExportRequestQueryConverterTest extends AbstractRequestConverte // WHEN SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request)); // THEN - assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B*+TO+*%5D&facet=true&facet.mincount=1&facet.limit=10" + + assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B*+TO+*%5D&facet=true&facet.mincount=1&facet.limit=-1" + "&facet.pivot=reqUser%2Crepo&facet.pivot=resource%2Crepo", query.toQueryString()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-logsearch-conf.xml new file mode 100644 index 0000000..ff4f695 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-logsearch-conf.xml @@ -0,0 +1,124 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>Accumulo</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>ACCUMULO_MASTER:accumulo_master;ACCUMULO_MONITOR:accumulo_monitor;ACCUMULO_GC:accumulo_gc;ACCUMULO_TRACER:accumulo_tracer;ACCUMULO_TSERVER:accumulo_tserver</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"accumulo_gc", + "rowtype":"service", + "path":"{{default('/configurations/accumulo-env/accumulo_log_dir', '/var/log/accumulo')}}/gc_*.log" + }, + { + "type":"accumulo_master", + "rowtype":"service", + "path":"{{default('/configurations/accumulo-env/accumulo_log_dir', '/var/log/accumulo')}}/master_*.log" + }, + { + "type":"accumulo_monitor", + "rowtype":"service", + "path":"{{default('/configurations/accumulo-env/accumulo_log_dir', '/var/log/accumulo')}}/monitor_*.log" + }, + { + "type":"accumulo_tracer", + "rowtype":"service", + "path":"{{default('/configurations/accumulo-env/accumulo_log_dir', '/var/log/accumulo')}}/tracer_*.log" + }, + { + "type":"accumulo_tserver", + "rowtype":"service", + "path":"{{default('/configurations/accumulo-env/accumulo_log_dir', '/var/log/accumulo')}}/tserver_*.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "accumulo_master" + ] + } + }, + "log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + }, + { + "filter":"grok", + "comment":"This one has one extra space after LEVEL", + "conditions":{ + "fields":{ + "type":[ + "accumulo_gc", + "accumulo_monitor", + "accumulo_tracer", + "accumulo_tserver" + ] + } + }, + "log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] + } + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/configuration/infra-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/configuration/infra-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/configuration/infra-logsearch-conf.xml new file mode 100644 index 0000000..1de8c46 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/configuration/infra-logsearch-conf.xml @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>Infra</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>INFRA_SOLR:infra_solr</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"infra_solr", + "rowtype":"service", + "path":"{{default('/configurations/infra-solr-env/infra_solr_log_dir', '/var/log/ambari-infra-solr')}}/solr.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "infra_solr" + ] + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] +} + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-logsearch-conf.xml new file mode 100644 index 0000000..72d44db --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-logsearch-conf.xml @@ -0,0 +1,201 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>AMS</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>METRICS_COLLECTOR:ams_collector,ams_hbase_master,ams_hbase_regionserver;METRICS_MONITOR:ams_monitor;METRICS_GRAFANA:ams_grafana</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"ams_hbase_master", + "rowtype":"service", + "path":"{{default('/configurations/ams-env/metrics_collector_log_dir', '/var/log/ambari-metrics-collector')}}/hbase-ams-master-*.log" + }, + { + "type":"ams_hbase_regionserver", + "rowtype":"service", + "path":"{{default('/configurations/ams-env/metrics_collector_log_dir', '/var/log/ambari-metrics-collector')}}/hbase-ams-regionserver-*.log" + }, + { + "type":"ams_collector", + "rowtype":"service", + "path":"{{default('/configurations/ams-env/metrics_collector_log_dir', '/var/log/ambari-metrics-collector')}}/ambari-metrics-collector.log" + }, + { + "type":"ams_monitor", + "rowtype":"service", + "path":"{{default('/configurations/ams-env/metrics_monitor_log_dir', '/var/log/ambari-metrics-monitor')}}/ambari-metrics-monitor.out" + }, + { + "type":"ams_grafana", + "rowtype":"service", + "path":"{{default('/configurations/ams-grafana-env/metrics_grafana_log_dir', '/var/log/ambari-metrics-grafana')}}/grafana.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ams_collector" + ] + } + }, + "log4j_format":"%d{ISO8601} %p %c: %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ams_hbase_master", + "ams_hbase_regionserver" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ams_grafana" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n", + "multiline_pattern":"^(%{DATESTAMP:logtime})", + "message_pattern":"(?m)^%{DATESTAMP:logtime}%{SPACE}\\[%{WORD:level}\\]%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy/MM/dd HH:mm:ss" + } + }, + "level":[ + { + "map_fieldvalue":{ + "pre_value":"I", + "post_value":"INFO" + } + }, + { + "map_fieldvalue":{ + "pre_value":"W", + "post_value":"WARN" + } + }, + { + "map_fieldvalue":{ + "pre_value":"D", + "post_value":"DEBUG" + } + }, + { + "map_fieldvalue":{ + "pre_value":"E", + "post_value":"ERROR" + } + }, + { + "map_fieldvalue":{ + "pre_value":"F", + "post_value":"FATAL" + } + } + ] + } + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ams_monitor" + ] + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{JAVAFILE:file}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + }, + "level":[ + { + "map_fieldvalue":{ + "pre_value":"WARNING", + "post_value":"WARN" + } + } + ] + } + ] + } + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-logsearch-conf.xml new file mode 100644 index 0000000..71a08fb --- /dev/null +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-logsearch-conf.xml @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>Atlas</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>ATLAS_SERVER:atlas_app</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"atlas_app", + "rowtype":"service", + "path":"{{default('/configurations/atlas-env/metadata_log_dir', '/var/log/atlas')}}/application.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "atlas_app" + ] + } + }, + "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] + } + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-logsearch-conf.xml new file mode 100644 index 0000000..6b43ba6 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-logsearch-conf.xml @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>Falcon</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>FALCON_SERVER:falcon_app</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"falcon_app", + "rowtype":"service", + "path":"{{default('/configurations/falcon-env/falcon_log_dir', '/var/log/falcon')}}/falcon.application.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "falcon_app" + ] + } + }, + "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] + } + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/configuration/flume-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/configuration/flume-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/configuration/flume-logsearch-conf.xml new file mode 100644 index 0000000..98e6db8 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/configuration/flume-logsearch-conf.xml @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>Flume</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>FLUME:flume_handler</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"flume_handler", + "rowtype":"service", + "path":"{{default('/configurations/flume-env/flume_log_dir', '/var/log/flume')}}/flume.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "flume_handler" + ] + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})", + "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}\\(%{JAVACLASS:class}\\.%{JAVAMETHOD:method}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"dd MMM yyyy HH:mm:ss,SSS" + } + } + } + } + ] + } + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-logsearch-conf.xml new file mode 100644 index 0000000..321ea4e --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-logsearch-conf.xml @@ -0,0 +1,111 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>HBase</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>HBASE_MASTER:hbase_master;HBASE_REGIONSERVER:hbase_regionserver;PHOENIX_QUERY_SERVER:hbase_phoenix_server</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"hbase_master", + "rowtype":"service", + "path":"{{default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')}}/hbase-*-master-*.log" + }, + { + "type":"hbase_regionserver", + "rowtype":"service", + "path":"{{default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')}}/hbase-*-regionserver-*.log" + }, + { + "type":"hbase_phoenix_server", + "rowtype":"service", + "path":"{{default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')}}/phoenix-*-server.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hbase_master", + "hbase_regionserver" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hbase_phoenix_server" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] + } + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-logsearch-conf.xml new file mode 100644 index 0000000..d85a028 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-logsearch-conf.xml @@ -0,0 +1,248 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>HDFS</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>NAMENODE:hdfs_namenode;DATANODE:hdfs_datanode;SECONDARY_NAMENODE:hdfs_secondarynamenode;JOURNALNODE:hdfs_journalnode;ZKFC:hdfs_zkfc;NFS_GATEWAY:hdfs_nfs3</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"hdfs_datanode", + "rowtype":"service", + "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-datanode-*.log" + }, + { + "type":"hdfs_namenode", + "rowtype":"service", + "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-namenode-*.log" + }, + { + "type":"hdfs_journalnode", + "rowtype":"service", + "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-journalnode-*.log" + }, + { + "type":"hdfs_secondarynamenode", + "rowtype":"service", + "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-secondarynamenode-*.log" + }, + { + "type":"hdfs_zkfc", + "rowtype":"service", + "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-zkfc-*.log" + }, + { + "type":"hdfs_nfs3", + "rowtype":"service", + "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-nfs3-*.log" + }, + { + "type":"hdfs_audit", + "rowtype":"audit", + "is_enabled":"true", + "add_fields":{ + "logType":"HDFSAudit", + "enforcer":"hadoop-acl", + "repoType":"1", + "repo":"hdfs" + }, + "path":"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hdfs-audit.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hdfs_datanode", + "hdfs_journalnode", + "hdfs_secondarynamenode", + "hdfs_namenode", + "hdfs_zkfc", + "hdfs_nfs3" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hdfs_audit" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "evtTime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + }, + { + "filter":"keyvalue", + "sort_order":1, + "conditions":{ + "fields":{ + "type":[ + "hdfs_audit" + ] + } + }, + "source_field":"log_message", + "value_split":"=", + "field_split":"\t", + "post_map_values":{ + "src":{ + "map_fieldname":{ + "new_fieldname":"resource" + } + }, + "ip":{ + "map_fieldname":{ + "new_fieldname":"cliIP" + } + }, + "allowed":[ + { + "map_fieldvalue":{ + "pre_value":"true", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"false", + "post_value":"0" + } + }, + { + "map_fieldname":{ + "new_fieldname":"result" + } + } + ], + "cmd":{ + "map_fieldname":{ + "new_fieldname":"action" + } + }, + "proto":{ + "map_fieldname":{ + "new_fieldname":"cliType" + } + }, + "callerContext":{ + "map_fieldname":{ + "new_fieldname":"req_caller_id" + } + } + } + }, + { + "filter":"grok", + "sort_order":2, + "source_field":"ugi", + "remove_source_field":"false", + "conditions":{ + "fields":{ + "type":[ + "hdfs_audit" + ] + } + }, + "message_pattern":"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}", + "post_map_values":{ + "user":{ + "map_fieldname":{ + "new_fieldname":"reqUser" + } + }, + "x_user":{ + "map_fieldname":{ + "new_fieldname":"reqUser" + } + }, + "p_user":{ + "map_fieldname":{ + "new_fieldname":"reqUser" + } + }, + "k_user":{ + "map_fieldname":{ + "new_fieldname":"proxyUsers" + } + }, + "p_authType":{ + "map_fieldname":{ + "new_fieldname":"authType" + } + }, + "k_authType":{ + "map_fieldname":{ + "new_fieldname":"proxyAuthType" + } + } + } + } + ] + } + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-logsearch-conf.xml new file mode 100644 index 0000000..c1b971c --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-logsearch-conf.xml @@ -0,0 +1,117 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>Hive</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>HIVE_METASTORE:hive_metastore;HIVE_SERVER:hive_hiveserver2;WEBHCAT_SERVER:webhcat_server</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"hive_hiveserver2", + "rowtype":"service", + "path":"{{default('/configurations/hive-env/hive_log_dir', '/var/log/hive')}}/hiveserver2.log" + }, + { + "type":"hive_metastore", + "rowtype":"service", + "path":"{{default('/configurations/hive-env/hive_log_dir', '/var/log/hive')}}/hivemetastore.log" + }, + { + "type": "webhcat_server", + "rowntype":"service", + "path":"{{default('configurations/hive-env/hcat_log_dir', '/var/log/webhcat')}}/webhcat.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hive_hiveserver2", + "hive_metastore" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "webhcat_server" + ] + } + }, + "log4j_format":" %-5p | %d{DATE} | %c | %m%n", + "multiline_pattern":"^(%{SPACE}%{LOGLEVEL:level}%{CUSTOM_SEPARATOR}%{CUSTOM_DATESTAMP:logtime})", + "message_pattern":"(?m)^%{SPACE}%{LOGLEVEL:level}%{CUSTOM_SEPARATOR}%{CUSTOM_DATESTAMP:logtime}%{CUSTOM_SEPARATOR}%{JAVACLASS:file}%{CUSTOM_SEPARATOR}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"dd MMM yyyy HH:mm:ss,SSS" + } + }, + "level":{ + "map_fieldvalue":{ + "pre_value":"WARNING", + "post_value":"WARN" + } + } + } + } + ] + } + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-logsearch-conf.xml new file mode 100644 index 0000000..29a8b36 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-logsearch-conf.xml @@ -0,0 +1,124 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>Kafka</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>KAFKA_BROKER:kafka_server,kafka_request,kafka_logcleaner,kafka_controller,kafka_statechange</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"kafka_controller", + "rowtype":"service", + "path":"{{default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')}}/controller.log" + }, + { + "type":"kafka_request", + "rowtype":"service", + "path":"{{default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')}}/kafka-request.log" + }, + { + "type":"kafka_logcleaner", + "rowtype":"service", + "path":"{{default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')}}/log-cleaner.log" + }, + { + "type":"kafka_server", + "rowtype":"service", + "path":"{{default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')}}/server.log" + }, + { + "type":"kafka_statechange", + "rowtype":"service", + "path":"{{kafka_log_dir}}/state-change.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "kafka_controller", + "kafka_request", + "kafka_logcleaner" + ] + } + }, + "log4j_format":"[%d] %p %m (%c)%n", + "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])", + "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + }, + { + "filter":"grok", + "comment":"Suppose to be same log4j pattern as other kafka processes, but some reason thread is not printed", + "conditions":{ + "fields":{ + "type":[ + "kafka_server", + "kafka_statechange" + ] + } + }, + "log4j_format":"[%d] %p %m (%c)%n", + "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])", + "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] + } + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration>
